From a3e795129dc367919d55dc84679178768de832cc Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:41:25 +0100 Subject: [PATCH 01/22] test(summary): an uncalled deletion must pair with nothing AnnotSV class 5 DEL over a recessive gene with GT ./. or no GT, plus a 1/1 SNV inside it, gives a biallelic hemizygous verdict today. A called deletion (0/1, ./1) must keep that verdict. Bead: C11 --- .../tests/rv_two_hits_uncalled_deletion.rs | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 rust/crates/og-summary/tests/rv_two_hits_uncalled_deletion.rs diff --git a/rust/crates/og-summary/tests/rv_two_hits_uncalled_deletion.rs b/rust/crates/og-summary/tests/rv_two_hits_uncalled_deletion.rs new file mode 100644 index 000000000..ffd5de7a5 --- /dev/null +++ b/rust/crates/og-summary/tests/rv_two_hits_uncalled_deletion.rs @@ -0,0 +1,109 @@ +//! A deletion the sample was never called for pairs with nothing. +//! +//! AnnotSV writes a row for every SV of its input, with the sample's own +//! GT. A merged or multi-sample input can hold a deletion whose GT for this +//! sample is `./.` or missing: the sample was not called for it. Such a +//! row is no copy loss, so a homozygous call inside its span must not read +//! as hemizygous on the one copy the deletion leaves. + +use indexmap::IndexMap; +use og_summary::collect::two_hits::{ + DeletionInputs, Spans, SvRow, Variant, Verdict, gene_deletions, two_hits, +}; + +/// AnnotSV class 5 DEL chr1:1000-5000 over the recessive gene G, with the +/// sample's GT. +fn annotsv_del(gt: Option<&str>) -> DeletionInputs { + DeletionInputs { + annotsv: Ok(Some(vec![SvRow { + sv: "1:1000-5000".into(), + sv_type: Some("DEL".into()), + gt: gt.map(str::to_owned), + ps: None, + acmg_class: Some("5".into()), + genes: vec!["G".into()], + caller: Some("manta".into()), + support: None, + common: false, + }])), + step35: Ok(None), + } +} + +/// A P/LP SNV at chr1:2000 in G, inside the deletion's span. +fn snv(gt: &str, copies: usize) -> Variant { + Variant { + gene: "G".into(), + key: ("chr1".into(), 2000, "A".into(), "T".into()), + chrom: "chr1".into(), + pos: 2000, + text: "chr1:2000 A>T".into(), + genotype: if copies >= 2 { "hom" } else { "het" }.into(), + copies, + ploidy: 2, + gt: gt.into(), + ps: None, + allele: None, + } +} + +fn spans() -> Spans { + IndexMap::from([("G".to_owned(), ("chr1".to_owned(), 500, 6000))]) +} + +fn pair(del_gt: Option<&str>, v: &Variant) -> og_summary::collect::two_hits::TwoHits { + let (dels, read, _) = gene_deletions(&annotsv_del(del_gt), &spans()); + assert_eq!(read, ["AnnotSV class 4 and 5 deletions (step 05)"]); + two_hits(std::slice::from_ref(v), &dels) +} + +#[test] +fn a_no_call_deletion_makes_no_homozygous_variant_biallelic() { + for gt in ["./.", ".|.", ".", "0/.", "./0"] { + let found = pair(Some(gt), &snv("1/1", 2)); + assert!( + found.entries.is_empty(), + "GT {gt}: an uncalled deletion paired: {:?}", + found.entries + ); + assert!(found.hemi.is_empty(), "GT {gt}: read as hemizygous"); + assert!(found.pairs.is_empty(), "GT {gt}: a pair was written"); + } +} + +#[test] +fn a_deletion_with_no_gt_makes_no_homozygous_variant_biallelic() { + let found = pair(None, &snv("1/1", 2)); + assert!(found.entries.is_empty(), "{:?}", found.entries); + assert!(found.hemi.is_empty()); +} + +#[test] +fn a_reference_genotype_still_pairs_with_nothing() { + for gt in ["0/0", "0|0", "0"] { + assert!(pair(Some(gt), &snv("1/1", 2)).entries.is_empty(), "GT {gt}"); + } +} + +#[test] +fn a_called_deletion_still_makes_a_homozygous_variant_hemizygous() { + for gt in ["0/1", "1/0", "./1", "1/1", "0|1"] { + let v = snv("1/1", 2); + let found = pair(Some(gt), &v); + let e = &found.entries["G"]; + assert_eq!(e.verdict, Verdict::Biallelic, "GT {gt}"); + assert_eq!(e.genotype, "hemizygous", "GT {gt}"); + assert!( + e.why.starts_with("the variant is called homozygous inside the deletion"), + "GT {gt}: {}", + e.why + ); + assert!(found.hemi.contains_key(&("G".to_owned(), v.key.clone()))); + } +} + +#[test] +fn a_called_deletion_still_pairs_with_a_het_variant() { + let found = pair(Some("0/1"), &snv("0/1", 1)); + assert_eq!(found.entries["G"].verdict, Verdict::PossiblyBiallelic); +} From 2a1427ded4326747cfb00c36a6a10d36784ce3f9 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:42:34 +0100 Subject: [PATCH 02/22] test(family): a pair somalier could not measure must not read ok Red on main: relatedness nan for declared partners and for declared first cousins gives ok, inf stops the step as a mismatch, and ibs0 over n gives ok. Bead: C18 --- .../og-steps/tests/rv_family_check_nan.rs | 218 ++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 rust/crates/og-steps/tests/rv_family_check_nan.rs diff --git a/rust/crates/og-steps/tests/rv_family_check_nan.rs b/rust/crates/og-steps/tests/rv_family_check_nan.rs new file mode 100644 index 000000000..eca880d1a --- /dev/null +++ b/rust/crates/og-steps/tests/rv_family_check_nan.rs @@ -0,0 +1,218 @@ +//! `og family check` on a pair somalier gave no measurement for. +//! +//! somalier relate writes `nan` (or `inf`) as the relatedness of a pair it +//! had nothing to divide by. Every comparison with NaN is false, so the +//! degree bins read it as unrelated, and a pair the pedigree declares +//! unrelated or distant came out `ok`. Such a pair, and one with counts +//! that cannot be counts (negative, or more opposite homozygotes than +//! sites), is `not_checked` with the reason and a `WARNING:` line, and the +//! step still exits 0. A finite relatedness keeps its verdict. +//! +//! The case that reads stderr runs `og family` in a child process: this +//! test binary itself, with [`CHILD`] set (see [`child_entry`]). + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +const CHILD: &str = "OG_TEST_RV_FAMILY_CHECK_NAN_CHILD"; + +const MEMBERS_HEAD: &str = "sample\tsex\tfather\tmother\tpartner\tin_pedigree\tsequenced\n"; +const PAIRS_HEAD: &str = "#sample_a\tsample_b\trelatedness\tibs0\tibs2\tn\n"; + +/// Two sequenced partners: declared unrelated, expected relatedness 0. +const PARTNERS: &str = "a\tmale\t\t\tb\tyes\tyes\nb\tfemale\t\t\ta\tyes\tyes\n"; + +/// Two sequenced first cousins (expected 0.125, third degree): the +/// grandparents, their two children and the children's partners are in +/// the pedigree, not sequenced. +const COUSINS: &str = "gf\tmale\t\t\tgm\tyes\tno\n\ +gm\tfemale\t\t\tgf\tyes\tno\n\ +p1\tmale\tgf\tgm\ts1\tyes\tno\n\ +s1\tfemale\t\t\tp1\tyes\tno\n\ +p2\tfemale\tgf\tgm\ts2\tyes\tno\n\ +s2\tmale\t\t\tp2\tyes\tno\n\ +c1\tmale\tp1\ts1\t\tyes\tyes\n\ +c2\tfemale\ts2\tp2\t\tyes\tyes\n"; + +/// In the child process only: runs `og family` with the arguments in +/// [`CHILD`] (separated by U+001F) and exits with its code, so the parent +/// reads its stderr. In a normal test run it does nothing. +#[test] +fn child_entry() { + let Ok(joined) = std::env::var(CHILD) else { + return; + }; + let args: Vec = joined.split('\u{1f}').map(str::to_owned).collect(); + let code = og_steps::family::cli(&args); + std::process::exit(i32::from(code)); +} + +fn scratch(name: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!( + "og-rv-family-check-nan-{}-{name}", + std::process::id() + )); + let _ = fs::remove_dir_all(&d); + fs::create_dir_all(&d).unwrap(); + d +} + +/// Writes the members and somalier's pairs, runs `og family check` in a +/// child process, and returns (exit code, stderr, the one row of +/// relationships.tsv as (column, value) pairs). +fn check(name: &str, members: &str, pair: &str) -> (i32, String, Vec<(String, String)>) { + let d = scratch(name); + let (m, p, out) = ( + d.join("members.tsv"), + d.join("pairs.tsv"), + d.join("relationships.tsv"), + ); + fs::write(&m, format!("{MEMBERS_HEAD}{members}")).unwrap(); + fs::write(&p, format!("{PAIRS_HEAD}{pair}\n")).unwrap(); + let args = [ + "check", + "--members", + path(&m), + "--pairs", + path(&p), + "--out", + path(&out), + ]; + let res = Command::new(std::env::current_exe().unwrap()) + .args(["child_entry", "--exact", "--nocapture", "--test-threads=1"]) + .env(CHILD, args.join("\u{1f}")) + .output() + .unwrap(); + let code = res.status.code().unwrap_or(-1); + let err = String::from_utf8_lossy(&res.stderr).into_owned(); + let text = fs::read_to_string(&out).unwrap_or_default(); + let mut lines = text.lines(); + let head: Vec<&str> = lines.next().unwrap_or_default().split('\t').collect(); + let rows: Vec<&str> = lines.collect(); + assert_eq!( + rows.len(), + 1, + "{name}: one pair, one row: {text}\nstderr: {err}" + ); + let row = head + .iter() + .zip(rows[0].split('\t')) + .map(|(k, v)| ((*k).to_owned(), v.to_owned())) + .collect(); + let _ = fs::remove_dir_all(&d); + (code, err, row) +} + +fn path(p: &Path) -> &str { + p.to_str().unwrap() +} + +fn get<'a>(row: &'a [(String, String)], k: &str) -> &'a str { + row.iter() + .find(|(c, _)| c == k) + .map_or_else(|| panic!("no column {k} in {row:?}"), |(_, v)| v.as_str()) +} + +/// One pair that must be not_checked, with exit 0 and a WARNING line that +/// carries `reason`. +fn assert_not_checked(name: &str, members: &str, pair: &str, reason: &str) { + let (code, err, row) = check(name, members, pair); + assert_eq!(code, 0, "{name}: exit code; stderr: {err}"); + assert_eq!(get(&row, "verdict"), "not_checked", "{name}: {row:?}"); + assert_eq!(get(&row, "reason"), reason, "{name}: {row:?}"); + assert!( + err.lines() + .any(|l| l.starts_with("WARNING: ") && l.ends_with(reason)), + "{name}: no WARNING line with the reason; stderr: {err}" + ); +} + +#[test] +fn nan_relatedness_of_declared_unrelated_pair_is_not_checked() { + assert_not_checked( + "nan-partners", + PARTNERS, + "a\tb\tnan\t10\t500\t1000", + "somalier gave relatedness 'nan' for this pair: not measured", + ); +} + +#[test] +fn inf_relatedness_of_declared_unrelated_pair_is_not_checked() { + assert_not_checked( + "inf-partners", + PARTNERS, + "a\tb\tinf\t10\t500\t1000", + "somalier gave relatedness 'inf' for this pair: not measured", + ); + assert_not_checked( + "minus-inf-partners", + PARTNERS, + "a\tb\t-inf\t10\t500\t1000", + "somalier gave relatedness '-inf' for this pair: not measured", + ); +} + +#[test] +fn nan_relatedness_of_declared_third_degree_pair_is_not_checked() { + let (code, _, row) = check("cousins-ok", COUSINS, "c1\tc2\t0.12\t5\t500\t5000"); + assert_eq!(code, 0); + assert_eq!(get(&row, "expected_relatedness"), "0.125", "{row:?}"); + assert_eq!(get(&row, "verdict"), "ok", "{row:?}"); + assert_not_checked( + "nan-cousins", + COUSINS, + "c1\tc2\tNaN\t5\t500\t5000", + "somalier gave relatedness 'NaN' for this pair: not measured", + ); +} + +#[test] +fn counts_that_cannot_be_counts_are_not_checked() { + let reason = |ibs0: i32, ibs2: i32, n: i32| { + format!( + "somalier gave ibs0 {ibs0}, ibs2 {ibs2} and {n} sites for this pair: not counts of \ + sites, not measured" + ) + }; + assert_not_checked( + "ibs0-over-n", + PARTNERS, + "a\tb\t0.01\t2000\t500\t1000", + &reason(2000, 500, 1000), + ); + assert_not_checked( + "negative-ibs2", + PARTNERS, + "a\tb\t0.01\t10\t-1\t1000", + &reason(10, -1, 1000), + ); + assert_not_checked( + "negative-n", + PARTNERS, + "a\tb\t0.01\t-5\t500\t-10", + &reason(-5, 500, -10), + ); +} + +#[test] +fn finite_relatedness_keeps_its_verdict() { + let (code, err, row) = check("finite-partners", PARTNERS, "a\tb\t0.01\t600\t100\t5000"); + assert_eq!(code, 0, "stderr: {err}"); + assert_eq!(get(&row, "verdict"), "ok", "{row:?}"); + assert_eq!(get(&row, "reason"), "unrelated, as declared", "{row:?}"); + assert!(!err.contains("WARNING"), "stderr: {err}"); + // Too few sites keeps its own reason. + let (code, _, row) = check("few-partners", PARTNERS, "a\tb\t0.01\t6\t10\t50"); + assert_eq!(code, 0); + assert_eq!(get(&row, "verdict"), "not_checked", "{row:?}"); + assert!( + get(&row, "reason").starts_with("somalier compared the two at 50 sites"), + "{row:?}" + ); + // A measured first degree for declared partners still stops the step. + let (code, _, row) = check("close-partners", PARTNERS, "a\tb\t0.5\t1\t900\t5000"); + assert_eq!(code, 1); + assert_eq!(get(&row, "verdict"), "mismatch", "{row:?}"); +} From 690cc4932582d7a09dea2a439df6312cc504340c Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:42:34 +0100 Subject: [PATCH 03/22] fix(family): a pair somalier gives no relatedness for reads not checked instead of ok og family check read relatedness nan or inf as a number; every comparison with NaN is false, so the degree bins called it unrelated and a pair declared unrelated or distant passed as ok. Such a pair, and one with counts that cannot be counts, is now not_checked with the reason and a WARNING line. Bead: C18 --- CHANGELOG.md | 1 + docs/multi-sample.md | 2 +- rust/crates/og-steps/src/family/check.rs | 27 +++++++++++++++++++++++- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05c3388e1..b5c103c31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- `og family check` (step 38 and the Nextflow `FAMILY_CHECK` process) no longer reports `ok` for a pair somalier could not measure. somalier writes `nan` as the relatedness when it had nothing to divide by. Every comparison with NaN is false, so the degree bins read it as unrelated, and a pair declared unrelated or distant passed as "unrelated, as declared". Such a pair, one with `inf`, and one with counts that cannot be counts (negative, or more opposite homozygotes than sites) now get the verdict `not_checked`, a reason that names the value ("somalier gave relatedness 'nan' for this pair: not measured") and a `WARNING:` line. The step still exits 0 for them. Before, `inf` stopped the step as a mismatch ("same person"). The columns of `relationships.tsv` do not change; `not_checked` was already one of its verdicts. - The self-test of `scripts/ci/check-images.sh` copies the whole tree again once the list of tracked paths passes 128 KiB. It piped that list through `xargs` to `tar`, and GNU `xargs` split a list that long over two `tar` runs; the reading `tar` stopped after the first archive, so Guard's `images` job failed on a broken pipe (exit 123). One `tar` now reads the list itself (`tar --null -T -`). Nothing the check reports changes. - The phased family VCF is indexed by the Rust `og` binary. WhatsHap's step (step 38 and the Nextflow WHATSHAP_PED process) wrote `family.phased.vcf.gz.tbi` with a Python one-liner, `pysam.tabix_index`. It now runs `og vcf index -f`. The `.tbi` keeps its name and format, and its bytes, once inflated, are the ones htslib 1.24 writes (the htslib of the pysam and bcftools the pipeline pins): the same bins in the same order, the same chunks and the same linear index. Only the deflate stream can differ, as two deflate libraries differ. On a 5-million-record three-sample VCF it took 0.75 to 0.91 s and 31 MB at peak, where htslib's own `tabix` took 0.81 to 0.93 s and 19 MB. - As in htslib 1.24, a record ends at the largest of INFO/END, the REF length, the SVLEN of a ``, ``, `` or `` ALT, and a sample's LEN on a gVCF block. A line with no position to read, unsorted records, a sequence that starts again, a record past 2^29 and an empty file stop the index with a message, and nothing is written. Without `-f` it never writes over an index, even one that appears while it runs. diff --git a/docs/multi-sample.md b/docs/multi-sample.md index c8c60b3d9..77fbbfda4 100644 --- a/docs/multi-sample.md +++ b/docs/multi-sample.md @@ -70,7 +70,7 @@ It reads the `family.vcf.gz` and `pedigree.tsv` the earlier run left in `$GENOME ### What it does 1. **Joint call.** GLnexus (`GLNEXUS_IMAGE`, its DeepVariantWGS preset) merges the members' DeepVariant gVCFs into one VCF, `family.vcf.gz`. A site where one person has a variant gets the others' genotypes from their reference blocks, so "not called" and "matches the reference" are told apart. The container gets 16 GB (`GLNEXUS_MEM` in step 38, the `GLNEXUS` process in `conf/base.config`), and GLnexus's own `--mem-gbytes` budget is half of it. That budget sizes only GLnexus's database cache and write buffers; the alleles it discovers and the sites it merges are held outside it. So a budget near the limit leaves no room for them. Whether 16 GB is enough for three whole genomes under the half rule has not been measured yet; when the task is killed for memory, Nextflow retries it once with 32 GB (capped at `--max_memory`), and step 38 takes a larger `GLNEXUS_MEM`. -2. **Relationship check.** somalier extracts each member's genotypes at its sites from the gVCF and `somalier relate` measures the relatedness of every pair: about the share of the genome two people inherited from common ancestors (1 the same person, 0.5 parent and child or full siblings, 0.25 second degree, 0 unrelated). The pedigree gives the expected value. A pair must fall in the same degree as declared (bins at 0.707, 0.354, 0.177 and 0.088); a declared parent and child must also be opposite homozygotes at under 1% of the sites, which siblings are not; a pair declared unrelated or distant must not measure first degree. A pair that disagrees stops the step with exit 1, before any phasing, and `relationships.tsv` says which pair and why: two samples were swapped, or a pedigree line is wrong. A pair compared at fewer than 1,000 sites is reported as not checked. +2. **Relationship check.** somalier extracts each member's genotypes at its sites from the gVCF and `somalier relate` measures the relatedness of every pair: about the share of the genome two people inherited from common ancestors (1 the same person, 0.5 parent and child or full siblings, 0.25 second degree, 0 unrelated). The pedigree gives the expected value. A pair must fall in the same degree as declared (bins at 0.707, 0.354, 0.177 and 0.088); a declared parent and child must also be opposite homozygotes at under 1% of the sites, which siblings are not; a pair declared unrelated or distant must not measure first degree. A pair that disagrees stops the step with exit 1, before any phasing, and `relationships.tsv` says which pair and why: two samples were swapped, or a pedigree line is wrong. A pair compared at fewer than 1,000 sites is reported as not checked. So is a pair somalier gave no relatedness for (`nan` or `inf`), or counts that cannot be counts (negative, or more opposite homozygotes than sites): the reason names the value, a `WARNING:` line says it, and the step goes on. 3. **Phasing by inheritance.** For each child whose two parents are sequenced, WhatsHap (`WHATSHAP_IMAGE`, `whatshap phase --ped`) phases the family by inheritance alone; no reads are given. A site where the child and both parents are heterozygous stays unphased, because no genotype tells it. 4. **The report.** For each child with a sequenced parent, `_inheritance.tsv.gz` gives, at every site of the joint VCF where the child and a sequenced parent have a genotype, the allele from the father and the allele from the mother when the genotypes tell (`how` is `genotype`), or `ambiguous` (both parents heterozygous like the child) or `inconsistent` (a genotype error, a deletion or a new mutation). On chrX and chrY outside the pseudoautosomal regions a son's X comes from his mother and his Y from his father; the child's sex must be known there. chrM is left out. With one parent sequenced, the allele the other parent passed on is known wherever the sequenced parent is homozygous or differs from the child: that is the half of the unsequenced parent's genome the child carries, and `family_report.md` gives its share of the sites. The half the child did not inherit is not seen. diff --git a/rust/crates/og-steps/src/family/check.rs b/rust/crates/og-steps/src/family/check.rs index 509547e8c..204f0d5c9 100644 --- a/rust/crates/og-steps/src/family/check.rs +++ b/rust/crates/og-steps/src/family/check.rs @@ -81,6 +81,27 @@ fn verdict( ("ok", format!("{got}, as declared")) } +/// Why somalier's numbers for one pair are no measurement, or None. +/// somalier writes `nan` (or `inf`) when it had nothing to divide by; every +/// comparison with NaN is false, so [`verdict`] would bin it as unrelated +/// and call a pair declared unrelated or distant ok. A negative count, or +/// more opposite homozygotes than sites, is no count somalier can make. +fn unmeasured(rel_text: &str, rel: f64, ibs0: i128, ibs2: i128, n: i128) -> Option { + if ibs0 < 0 || ibs2 < 0 || n < 0 || ibs0 > n { + return Some(format!( + "somalier gave ibs0 {ibs0}, ibs2 {ibs2} and {n} sites for this pair: not counts \ + of sites, not measured" + )); + } + if !rel.is_finite() { + return Some(format!( + "somalier gave relatedness '{}' for this pair: not measured", + rel_text.trim() + )); + } + None +} + pub fn run(a: &Args) -> Result<()> { let members_path = a.one("--members").unwrap_or_default(); let pairs_path = a.one("--pairs").unwrap_or_default(); @@ -130,7 +151,11 @@ pub fn run(a: &Args) -> Result<()> { let (rel, ibs0, ibs2, n) = (float("relatedness")?, int("ibs0")?, int("ibs2")?, int("n")?); let rate = if n != 0 { ibs0 as f64 / n as f64 } else { 0.0 }; - let (v, why) = verdict(&d, e, rel, rate, n); + let rel_text = cell(r, "relatedness", pairs_path)?; + let (v, why) = match unmeasured(rel_text, rel, ibs0, ibs2, n) { + Some(why) => ("not_checked", why), + None => verdict(&d, e, rel, rate, n), + }; if v == "mismatch" { bad.push(format!("{x} and {y}: {why}")); } else if v != "ok" { From 8ece22226c256950697caab50034276a67067b5a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:43:45 +0100 Subject: [PATCH 04/22] fix(summary): pair only a deletion with a called ALT allele gene_deletions() skipped an AnnotSV deletion only when its GT was 0/0. A GT of ./., 0/. or none passed, became a deletion inside its span, and a 1/1 variant there read as hemizygous and biallelic, although the sample never carried the deletion. Now a row pairs only when some allele of its GT is called and not 0 (allele_copies, the count the small variants use). Bead: C11 --- CHANGELOG.md | 1 + docs/05-annotsv.md | 2 +- rust/crates/og-summary/src/collect/two_hits.rs | 9 ++++++--- .../og-summary/tests/rv_two_hits_uncalled_deletion.rs | 3 ++- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05c3388e1..f738d1db2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- A deletion the sample was never called for no longer makes a homozygous variant look biallelic. The two-hit rule of `og summary collect` dropped an AnnotSV deletion only when its `GT` was `0/0`. A row with `GT` `./.`, `0/.` or no `GT` still counted as a copy loss, so a `1/1` variant inside its span read as hemizygous, "biallelic" with the deletion, in the carrier list and the secondary-findings card. Now only a deletion with a called ALT allele in the sample pairs. A table without FORMAT and sample columns has no `GT`, so its deletions pair with nothing. No field, column or file name changes. - The self-test of `scripts/ci/check-images.sh` copies the whole tree again once the list of tracked paths passes 128 KiB. It piped that list through `xargs` to `tar`, and GNU `xargs` split a list that long over two `tar` runs; the reading `tar` stopped after the first archive, so Guard's `images` job failed on a broken pipe (exit 123). One `tar` now reads the list itself (`tar --null -T -`). Nothing the check reports changes. - The phased family VCF is indexed by the Rust `og` binary. WhatsHap's step (step 38 and the Nextflow WHATSHAP_PED process) wrote `family.phased.vcf.gz.tbi` with a Python one-liner, `pysam.tabix_index`. It now runs `og vcf index -f`. The `.tbi` keeps its name and format, and its bytes, once inflated, are the ones htslib 1.24 writes (the htslib of the pysam and bcftools the pipeline pins): the same bins in the same order, the same chunks and the same linear index. Only the deflate stream can differ, as two deflate libraries differ. On a 5-million-record three-sample VCF it took 0.75 to 0.91 s and 31 MB at peak, where htslib's own `tabix` took 0.81 to 0.93 s and 19 MB. - As in htslib 1.24, a record ends at the largest of INFO/END, the REF length, the SVLEN of a ``, ``, `` or `` ALT, and a sample's LEN on a gVCF block. A line with no position to read, unsorted records, a sequence that starts again, a record past 2^29 and an empty file stop the index with a message, and nothing is written. Without `-f` it never writes over an index, even one that appears while it runs. diff --git a/docs/05-annotsv.md b/docs/05-annotsv.md index eb61026c3..e59f45025 100644 --- a/docs/05-annotsv.md +++ b/docs/05-annotsv.md @@ -112,7 +112,7 @@ AnnotSV gives class 4 or 5 to some calls that are not a rare deletion: a call on | `AnnotSV_class` | AnnotSV's own class (4 or 5). `ACMG_class` is the class after the checks below: 3 when one of them moved the call, else AnnotSV's | | `Depth_support` | for a deletion or duplication, whether its read depth shows the change (below); `.` for other types | | `Rule`, `Rule_detail` | each check that fired (`no_depth_support`, `tandem_repeat`, `ig_tr_locus`, `mhc`, `gene_alias`) and why, in words | -| `FORMAT`, `` | the last two columns, as AnnotSV's input has them: the caller's FORMAT keys and the sample's values, with `GT` and, when the caller phased the call, `PS`. The reports pair a deletion with a small variant of the same gene by that phase (in trans when both sit in one phase set on different copies), and a deletion genotyped `0/0` pairs with nothing. An input without FORMAT, or a table with no class 4 or 5 SV, writes neither. A sample named like a column of this table stops the step with an error, since its `GT` and `PS` would be lost | +| `FORMAT`, `` | the last two columns, as AnnotSV's input has them: the caller's FORMAT keys and the sample's values, with `GT` and, when the caller phased the call, `PS`. The reports pair a deletion with a small variant of the same gene by that phase (in trans when both sit in one phase set on different copies), and a deletion with no called ALT in the sample (`0/0`, `./.`, `0/.` or no `GT`) pairs with nothing. An input without FORMAT, or a table with no class 4 or 5 SV, writes neither. A sample named like a column of this table stops the step with an error, since its `GT` and `PS` would be lost | ### Calls the table moves to class 3 diff --git a/rust/crates/og-summary/src/collect/two_hits.rs b/rust/crates/og-summary/src/collect/two_hits.rs index b022c8412..de93590e5 100644 --- a/rust/crates/og-summary/src/collect/two_hits.rs +++ b/rust/crates/og-summary/src/collect/two_hits.rs @@ -747,9 +747,12 @@ pub fn gene_deletions( if x.sv_type.as_deref() != Some("DEL") || x.common || !(isdigit(a) && isdigit(b)) { continue; } - // A call genotyped 0/0 is no deletion in this sample. - if let Some(gt) = x.gt.as_deref() - && gt.replace('|', "/").split('/').all(|a| a == "0") + // Only a called ALT is a deletion in this sample: a GT of + // 0/0, ./., 0/. or none is no copy loss, so it pairs with + // nothing. + if x.gt + .as_deref() + .is_none_or(|gt| super::clinvar::allele_copies(gt).0 == 0) { continue; } diff --git a/rust/crates/og-summary/tests/rv_two_hits_uncalled_deletion.rs b/rust/crates/og-summary/tests/rv_two_hits_uncalled_deletion.rs index ffd5de7a5..7d5e18ed1 100644 --- a/rust/crates/og-summary/tests/rv_two_hits_uncalled_deletion.rs +++ b/rust/crates/og-summary/tests/rv_two_hits_uncalled_deletion.rs @@ -94,7 +94,8 @@ fn a_called_deletion_still_makes_a_homozygous_variant_hemizygous() { assert_eq!(e.verdict, Verdict::Biallelic, "GT {gt}"); assert_eq!(e.genotype, "hemizygous", "GT {gt}"); assert!( - e.why.starts_with("the variant is called homozygous inside the deletion"), + e.why + .starts_with("the variant is called homozygous inside the deletion"), "GT {gt}: {}", e.why ); From 9d0f111425ddf28b4298e954d88f56cdcc1fc7d4 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:44:30 +0100 Subject: [PATCH 05/22] fix(clinvar): fail step 06 when the first bcftools of a container pipeline fails The three two-command pipelines in the bcftools container ran under sh, which has no pipefail, so a failed bcftools view or annotate went unnoticed when the next command exited 0. They now run in bash with pipefail, as the join-adjacent pipeline of the same step already does. Bead: PL02 --- scripts/06-clinvar-screen.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/06-clinvar-screen.sh b/scripts/06-clinvar-screen.sh index b6b3c8654..505c51659 100755 --- a/scripts/06-clinvar-screen.sh +++ b/scripts/06-clinvar-screen.sh @@ -128,12 +128,14 @@ SAMPLE_DROPPED=$(comm -23 <(printf '%s\n' "$SAMPLE_CONTIGS") <(printf '%s\n' "$S # Step 1: normalise ClinVar once (split multiallelics, left-align), rebuilt when the # source is newer. isec matches on identical REF/ALT, so both sides must be split # and left-aligned the same way. +# Each pipeline below runs in bash with pipefail: under sh a failed first +# bcftools would go unnoticed when the second one exits 0. if [ ! -f "$CLINVAR_NORM" ] || [ ! -f "${CLINVAR_NORM}.tbi" ] || [ "$CLINVAR" -nt "$CLINVAR_NORM" ]; then echo "Normalising ClinVar into ${CLINVAR_NORM} ..." rm -f "${CLINVAR_NORM}.part.vcf.gz" "${CLINVAR_NORM}.part.vcf.gz.tbi" # The normalised copy is shared by every sample, so clinvar/ is writable here. run_in --rw "$(dirname "$CLINVAR_NORM")" --cpus 2 --memory 2g "$BCFTOOLS_IMAGE" \ - sh -c "set -e; bcftools view -T '$(cpath "$CLINVAR_TARGETS")' -Ou '$(cpath "$CLINVAR")' \ + bash -o pipefail -c "set -e; bcftools view -T '$(cpath "$CLINVAR_TARGETS")' -Ou '$(cpath "$CLINVAR")' \ | bcftools norm -m -any -c w -f '$(cpath "$REF")' -Oz -o '$(cpath "${CLINVAR_NORM}.part.vcf.gz")' -" bcftools_run --rw "$(dirname "$CLINVAR_NORM")" bcftools index -t "$(cpath "${CLINVAR_NORM}.part.vcf.gz")" mv "${CLINVAR_NORM}.part.vcf.gz" "$CLINVAR_NORM" @@ -170,7 +172,7 @@ fi PASS_VCF="${OUTPUT_DIR}/${SAMPLE}_pass.vcf.gz" NORM_LOG="${OUTPUT_DIR}/${SAMPLE}_norm.log" if ! run_in --cpus 2 --memory 2g "$BCFTOOLS_IMAGE" \ - sh -c "set -e; bcftools view -f '${FILTER}' -T '$(cpath "$SAMPLE_TARGETS")' -Ou '$(cpath "$VCF")' \ + bash -o pipefail -c "set -e; bcftools view -f '${FILTER}' -T '$(cpath "$SAMPLE_TARGETS")' -Ou '$(cpath "$VCF")' \ | bcftools norm -m -any --multi-overlaps . -c w -f '$(cpath "$REF")' -Oz -o '$(cpath "$PASS_VCF")' - bcftools index -f -t '$(cpath "$PASS_VCF")'" 2> "$NORM_LOG"; then cat "$NORM_LOG" >&2 @@ -234,7 +236,7 @@ case "$CLINVAR_COLS" in esac SHARED="${OUTPUT_DIR}/${SAMPLE}_shared.vcf.gz" run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 2 --memory 2g "$BCFTOOLS_IMAGE" \ - sh -c "set -e + bash -o pipefail -c "set -e bcftools isec --no-version -n=2 -w1 -Oz -o '$(cpath "$SHARED")' '$(cpath "$CANDIDATES")' '$(cpath "$CLINVAR_NORM")' bcftools index -f -t '$(cpath "$SHARED")' bcftools annotate --no-version -a '$(cpath "$CLINVAR_NORM")' --pair-logic exact \ From da1e8bdb55517737d849e8cc5865a407860540cc Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:44:30 +0100 Subject: [PATCH 06/22] fix(clinvar): build step 06's shared normalised ClinVar once under a lock Every sample shares clinvar/clinvar_pathogenic_chr.norm.vcf.gz, but two screens that found it stale both wrote the same .part file, so one could fail on a missing file or publish a .tbi of another write than the VCF. One run now builds it under clinvar/.norm.lock (lock_acquire, which works without flock) while the others wait and test it again; the index goes in place before the VCF. Bead: PL01 --- scripts/06-clinvar-screen.sh | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/scripts/06-clinvar-screen.sh b/scripts/06-clinvar-screen.sh index 505c51659..ea153f125 100755 --- a/scripts/06-clinvar-screen.sh +++ b/scripts/06-clinvar-screen.sh @@ -128,18 +128,31 @@ SAMPLE_DROPPED=$(comm -23 <(printf '%s\n' "$SAMPLE_CONTIGS") <(printf '%s\n' "$S # Step 1: normalise ClinVar once (split multiallelics, left-align), rebuilt when the # source is newer. isec matches on identical REF/ALT, so both sides must be split # and left-aligned the same way. +# Every sample shares the normalised copy, so one run builds it while the others +# wait on clinvar/.norm.lock (lock_acquire, also without flock), then test it +# again and find it built. The build goes to a .part file only the lock holder +# writes; the index goes in place before the VCF, so a VCF that is newer than +# the source always has the index of the same write beside it. # Each pipeline below runs in bash with pipefail: under sh a failed first # bcftools would go unnoticed when the second one exits 0. -if [ ! -f "$CLINVAR_NORM" ] || [ ! -f "${CLINVAR_NORM}.tbi" ] || [ "$CLINVAR" -nt "$CLINVAR_NORM" ]; then - echo "Normalising ClinVar into ${CLINVAR_NORM} ..." - rm -f "${CLINVAR_NORM}.part.vcf.gz" "${CLINVAR_NORM}.part.vcf.gz.tbi" - # The normalised copy is shared by every sample, so clinvar/ is writable here. - run_in --rw "$(dirname "$CLINVAR_NORM")" --cpus 2 --memory 2g "$BCFTOOLS_IMAGE" \ - bash -o pipefail -c "set -e; bcftools view -T '$(cpath "$CLINVAR_TARGETS")' -Ou '$(cpath "$CLINVAR")' \ - | bcftools norm -m -any -c w -f '$(cpath "$REF")' -Oz -o '$(cpath "${CLINVAR_NORM}.part.vcf.gz")' -" - bcftools_run --rw "$(dirname "$CLINVAR_NORM")" bcftools index -t "$(cpath "${CLINVAR_NORM}.part.vcf.gz")" - mv "${CLINVAR_NORM}.part.vcf.gz" "$CLINVAR_NORM" - mv "${CLINVAR_NORM}.part.vcf.gz.tbi" "${CLINVAR_NORM}.tbi" +norm_stale() { [ ! -f "$CLINVAR_NORM" ] || [ ! -f "${CLINVAR_NORM}.tbi" ] || [ "$CLINVAR" -nt "$CLINVAR_NORM" ]; } +if norm_stale; then + NORM_LOCK="${GENOME_DIR}/clinvar/.norm.lock" + lock_acquire "$NORM_LOCK" + trap 'lock_release "$NORM_LOCK"' EXIT + if norm_stale; then + echo "Normalising ClinVar into ${CLINVAR_NORM} ..." + rm -f "${CLINVAR_NORM}.part.vcf.gz" "${CLINVAR_NORM}.part.vcf.gz.tbi" + # The normalised copy is shared by every sample, so clinvar/ is writable here. + run_in --rw "$(dirname "$CLINVAR_NORM")" --cpus 2 --memory 2g "$BCFTOOLS_IMAGE" \ + bash -o pipefail -c "set -e; bcftools view -T '$(cpath "$CLINVAR_TARGETS")' -Ou '$(cpath "$CLINVAR")' \ + | bcftools norm -m -any -c w -f '$(cpath "$REF")' -Oz -o '$(cpath "${CLINVAR_NORM}.part.vcf.gz")' -" + bcftools_run --rw "$(dirname "$CLINVAR_NORM")" bcftools index -t "$(cpath "${CLINVAR_NORM}.part.vcf.gz")" + mv "${CLINVAR_NORM}.part.vcf.gz.tbi" "${CLINVAR_NORM}.tbi" + mv "${CLINVAR_NORM}.part.vcf.gz" "$CLINVAR_NORM" + fi + lock_release "$NORM_LOCK" + trap - EXIT fi # Step 2: filter the sample. Callers that never write PASS (FILTER '.') would lose From dbd6c4c2f371a8586ebe6954d480e153f88d34cf Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:44:30 +0100 Subject: [PATCH 07/22] test(clinvar): fake-docker case for step 06 pipeline failures and concurrent screens Runs each container command of step 06 for real with a fake bcftools and og: a failed first stage of each nested pipeline must fail the step and leave no hits, and two screens at once against a stale normalised ClinVar must build it once, with the VCF and .tbi of one write. Bead: PL01 Bead: PL02 --- .../fake-docker/rv-step06-screen-integrity.sh | 183 ++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 tests/fake-docker/rv-step06-screen-integrity.sh diff --git a/tests/fake-docker/rv-step06-screen-integrity.sh b/tests/fake-docker/rv-step06-screen-integrity.sh new file mode 100644 index 000000000..61b80e702 --- /dev/null +++ b/tests/fake-docker/rv-step06-screen-integrity.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +# Step 06 (scripts/06-clinvar-screen.sh) never publishes a partial or +# mismatched ClinVar screen: +# - a failed first bcftools of a nested pipeline (view | norm when ClinVar +# is normalised, view | norm when the sample is filtered, annotate | view +# when the hits are written) fails the step and leaves no hits file. The +# container's sh has no pipefail, so each pipeline must run in bash with +# pipefail; +# - two samples screened at the same time against a normalised ClinVar +# older than its source build it once, and the published VCF and its +# .tbi come from that one write. +# The docker run hook runs each container command for real, with /genome +# mapped to GENOME_DIR and a fake bcftools and og first on PATH. The fake +# tools write plain text and stamp every write with an id, which an index +# copies, so a VCF and a .tbi of two different writes can be told apart. +# shellcheck source=../../scripts/ci/fake-docker/lib.sh +. "${REPO_ROOT:?}/scripts/ci/fake-docker/lib.sh" + +export GENOME_DIR="${CASE_WORK}/genome" +C="${GENOME_DIR}/clinvar" +NORM="${C}/clinvar_pathogenic_chr.norm.vcf.gz" +seed_reference "$GENOME_DIR" +seed_clinvar "$GENOME_DIR" +seed_sample "$GENOME_DIR" sample1 +seed_sample "$GENOME_DIR" sample2 + +TOOLS="${CASE_WORK}/tools" +mkdir -p "$TOOLS" +export FAKE_TOOL_LOG="${CASE_WORK}/tools.log" + +# bcftools: writes a header and a write id to -o (or stdout), indexes by +# copying the data file's id into FILE.tbi, and fails like the real one on a +# missing input. FAKE_FAIL (a regex over the arguments) makes the matching +# call write a header and exit 1. FAKE_NORM_SLEEP slows the norm that +# writes the normalised ClinVar, to hold two runs in that window together. +cat > "${TOOLS}/bcftools" <<'TOOL' +#!/usr/bin/env bash +set -uo pipefail +args="$*" +printf '%s\n' "bcftools ${args}" >> "$FAKE_TOOL_LOG" +if [ -n "${FAKE_FAIL:-}" ] && [[ "$args" =~ $FAKE_FAIL ]]; then + printf '##fileformat=VCFv4.2\n' + echo "fake bcftools: failing on purpose: ${args}" >&2 + exit 1 +fi +sub=$1 +shift +out="" last="" h=false n=false s=false +while [ $# -gt 0 ]; do + case "$1" in + -o) out=$2; shift 2; continue ;; + -h) h=true ;; + -n) n=true ;; + -s) s=true ;; + esac + last=$1 + shift +done +# No file after the options (or -): bcftools reads stdin. +case "$last" in -*|"") last=-; cat > /dev/null ;; esac +if [ "$last" != - ] && [ ! -e "$last" ]; then echo "fake bcftools: ${last}: No such file" >&2; exit 1; fi +case "$sub" in + index) + if $s; then printf 'chr1\t248956422\t3\n' + elif $n; then echo 3 + else grep -m1 '^##write=' "$last" > "${last}.tbi" || { echo "fake bcftools: ${last} has no write id" >&2; exit 1; } + fi + exit 0 ;; + view) + if $h; then printf '##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tsample\n'; exit 0; fi + case " ${args} " in *" -H "*) printf 'chr1\t100\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\n'; exit 0 ;; esac ;; + query) [ -z "$out" ] || : > "$out"; exit 0 ;; +esac +if [ "$sub" = norm ] && [[ "$out" == *clinvar_pathogenic_chr.norm* ]]; then + /bin/sleep "${FAKE_NORM_SLEEP:-0}" +fi +body=$(printf '##fileformat=VCFv4.2\n##write=%s.%s.%s\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n' "$sub" "$$" "$RANDOM") +if [ -n "$out" ]; then printf '%s\n' "$body" > "$out"; else printf '%s\n' "$body"; fi +TOOL + +# og: the subcommands step 06 calls, writing what the step reads next. +cat > "${TOOLS}/og" <<'TOOL' +#!/usr/bin/env bash +set -uo pipefail +printf '%s\n' "og $*" >> "$FAKE_TOOL_LOG" +case "$1 $2" in + "clinvar join-adjacent") + if [ "$4" = tag ]; then cat "$5"; else cat; fi ;; + "clinvar deep-intronic") + cat > /dev/null + printf 'chrom\tpos\tref\talt\tgenotype\n' ;; + "cdna check"|"cdna scan") + while [ $# -gt 0 ]; do + if [ "$1" = --out ]; then printf 'header\n' > "$2"; fi + shift + done ;; + *) echo "fake og: no answer for: $*" >&2; exit 1 ;; +esac +TOOL +chmod +x "${TOOLS}/bcftools" "${TOOLS}/og" + +# The docker run hook: each container path under /genome becomes the host +# path, /pgp-bin/og the fake og; the command runs from the sample's clinvar +# folder (step 06 runs og cdna with -w there and a relative --out). +cat > "${CASE_WORK}/hook-run" <<'HOOK' +#!/usr/bin/env bash +set -uo pipefail +shift # the image +cmd=() +for a in "$@"; do + a=${a//\/pgp-bin\/og/og} + cmd+=("${a//\/genome\//${GENOME_DIR}/}") +done +while IFS= read -r v; do + case "$v" in + *:/genome/clinvar*) ;; + *:/genome/*) [ -d "${v%%:*}/clinvar" ] && cd "${v%%:*}/clinvar" ;; + esac +done <<<"${FAKE_DOCKER_VOLUMES:-}" +PATH="${FAKE_TOOLS}:${PATH}" exec "${cmd[@]}" +HOOK +chmod +x "${CASE_WORK}/hook-run" +export FAKE_DOCKER_RUN_HOOK="${CASE_WORK}/hook-run" FAKE_TOOLS="$TOOLS" + +write_id() { grep -m1 '^##write=' "$1" 2>/dev/null || echo none; } + +# --- A full screen with the fake tools succeeds ------------------------------------- +# The positive control: the cases below fail only because of the failure they plant. +run_rc clean "${SCRIPTS}/06-clinvar-screen.sh" sample1 +expect_rc clean 0 +[ -f "${GENOME_DIR}/sample1/clinvar/sample1_clinvar_hits.vcf" ] || fail "a clean screen wrote no hits file" +[ "$(write_id "$NORM")" = "$(write_id "${NORM}.tbi")" ] || fail "a clean screen published a .tbi of another write than the normalised ClinVar" +[ ! -e "${C}/.norm.lock" ] || fail "a clean screen left clinvar/.norm.lock behind" + +# --- A failed first stage of each nested pipeline fails the step --------------------- +# name|regex of the first stage's arguments +for stage in \ + "clinvar-view|^view -T .*/clinvar/clinvar_pathogenic_chr\.vcf\.gz$" \ + "sample-view|^view -f .*/sample1\.vcf\.gz$" \ + "hits-annotate|^annotate "; do + name=${stage%%|*} + rm -rf "${GENOME_DIR}/sample1/clinvar" "$NORM" "${NORM}.tbi" + : > "$FAKE_TOOL_LOG" + FAKE_FAIL=${stage#*|} run_rc "fail-${name}" "${SCRIPTS}/06-clinvar-screen.sh" sample1 + grep -q 'failing on purpose' "${CASE_WORK}/fail-${name}.out" || fail "${name}: the planted failure never ran" + [ "$RC" -ne 0 ] || fail "${name}: step 06 exited 0 after its first bcftools failed (a partial screen)" + for f in sample1_clinvar_hits.vcf sample1_clinvar_hits.tsv; do + [ ! -e "${GENOME_DIR}/sample1/clinvar/${f}" ] || fail "${name}: step 06 left ${f} after a failed bcftools" + done + [ ! -e "${C}/.norm.lock" ] || fail "${name}: a failed screen left clinvar/.norm.lock behind" + echo "${name}: a failed first bcftools fails the step and leaves no hits: PASS" +done + +# --- Two samples at once against a normalised ClinVar older than its source ---------- +printf '##write=old\n' > "$NORM" +printf '##write=old\n' > "${NORM}.tbi" +touch -t 202001010000 "$NORM" "${NORM}.tbi" +rm -rf "${GENOME_DIR}"/sample[12]/clinvar +: > "$FAKE_TOOL_LOG" +export FAKE_NORM_SLEEP=2 +rc1=0 rc2=0 +"${SCRIPTS}/06-clinvar-screen.sh" sample1 > "${CASE_WORK}/both-1.out" 2>&1 & +p1=$! +"${SCRIPTS}/06-clinvar-screen.sh" sample2 > "${CASE_WORK}/both-2.out" 2>&1 & +p2=$! +wait "$p1" || rc1=$? +wait "$p2" || rc2=$? +unset FAKE_NORM_SLEEP +cat "${CASE_WORK}/both-1.out" "${CASE_WORK}/both-2.out" +[ "$rc1" -eq 0 ] && [ "$rc2" -eq 0 ] || fail "two screens at once exited ${rc1} and ${rc2}, want 0 and 0" +builds=$(grep -c '^bcftools norm .*clinvar_pathogenic_chr\.norm' "$FAKE_TOOL_LOG" || true) +[ "$builds" -eq 1 ] || fail "two screens at once normalised ClinVar ${builds} times, want once" +vid=$(write_id "$NORM") +tid=$(write_id "${NORM}.tbi") +echo "published VCF ${vid}, .tbi ${tid}" +[ "$vid" != '##write=old' ] && [ "$vid" != none ] || fail "the normalised ClinVar was not rebuilt (${vid})" +[ "$vid" = "$tid" ] || fail "the published .tbi (${tid}) is from another write than the VCF (${vid})" +for s in sample1 sample2; do + [ -f "${GENOME_DIR}/${s}/clinvar/${s}_clinvar_hits.vcf" ] || fail "${s}: no hits file after two screens at once" +done +[ -z "$(find "$C" -name '*.part*')" ] || fail "two screens at once left part files: $(find "$C" -name '*.part*')" +[ ! -e "${C}/.norm.lock" ] || fail "two screens at once left clinvar/.norm.lock behind" +echo "two screens at once build the normalised ClinVar once, VCF and .tbi of one write: PASS" From 36e32058ae3fa71dc50bffdaaf16ed05b8142ad8 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:44:30 +0100 Subject: [PATCH 08/22] docs(clinvar): say how step 06 builds its shared normalised ClinVar Bead: PL01 Bead: PL02 --- CHANGELOG.md | 4 ++++ docs/06-clinvar-screen.md | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05c3388e1..a55719ba7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Step 06 (`scripts/06-clinvar-screen.sh`) no longer ships a ClinVar screen that is silently incomplete. + - Its three two-command pipelines in the bcftools container (ClinVar's normalisation, the sample's PASS filter and the copy of ClinVar's fields onto the hits) ran under `sh`, which has no `pipefail`. A failed first `bcftools` went unnoticed when the second one exited 0, and the step finished with part of the records. They now run in `bash -o pipefail`, so the step stops with an error and writes no hits file. + - Two samples screened at the same time after a new install or a ClinVar refresh both rebuilt the shared `clinvar/clinvar_pathogenic_chr.norm.vcf.gz` into one `.part` file. One run could fail, or publish a `.tbi` of another write than the VCF. Now one run builds it under `clinvar/.norm.lock` while the others wait and then reuse it, and the index goes in place before the VCF. + No output file or option changes. The Nextflow module is not affected: it normalises in its own task folder and runs with `pipefail`. - The self-test of `scripts/ci/check-images.sh` copies the whole tree again once the list of tracked paths passes 128 KiB. It piped that list through `xargs` to `tar`, and GNU `xargs` split a list that long over two `tar` runs; the reading `tar` stopped after the first archive, so Guard's `images` job failed on a broken pipe (exit 123). One `tar` now reads the list itself (`tar --null -T -`). Nothing the check reports changes. - The phased family VCF is indexed by the Rust `og` binary. WhatsHap's step (step 38 and the Nextflow WHATSHAP_PED process) wrote `family.phased.vcf.gz.tbi` with a Python one-liner, `pysam.tabix_index`. It now runs `og vcf index -f`. The `.tbi` keeps its name and format, and its bytes, once inflated, are the ones htslib 1.24 writes (the htslib of the pysam and bcftools the pipeline pins): the same bins in the same order, the same chunks and the same linear index. Only the deflate stream can differ, as two deflate libraries differ. On a 5-million-record three-sample VCF it took 0.75 to 0.91 s and 31 MB at peak, where htslib's own `tabix` took 0.81 to 0.93 s and 19 MB. - As in htslib 1.24, a record ends at the largest of INFO/END, the REF length, the SVLEN of a ``, ``, `` or `` ALT, and a sample's LEN on a gVCF block. A line with no position to read, unsorted records, a sequence that starts again, a record past 2^29 and an empty file stop the index with a message, and nothing is written. Without `-f` it never writes over an index, even one that appears while it runs. diff --git a/docs/06-clinvar-screen.md b/docs/06-clinvar-screen.md index a8534eafe..c92da1458 100644 --- a/docs/06-clinvar-screen.md +++ b/docs/06-clinvar-screen.md @@ -49,7 +49,7 @@ In the Nextflow pipeline the same floor is `ext.min_gq` and `ext.min_dp` of `CLI ### What the Script Does 1. Lists the contigs each file holds records on (`bcftools index -s`) and keeps those the reference has too: ClinVar's contigs that are in the reference, and the sample's contigs among those. `bcftools norm` stops at the first record whose contig the reference lacks, such as a provider's unplaced scaffold or one of the `NT_` contigs the full ClinVar file has, and a record on a contig the other file lacks cannot match anyway. The step prints how many of the sample's records it leaves out and on which contigs. When no contig is shared it stops with an error instead of reporting zero hits, and says which file is named the other way (`1, 2, ...` against `chr1, chr2, ...`). -2. Builds `clinvar/clinvar_pathogenic_chr.norm.vcf.gz` beside the ClinVar file the first time, and again whenever the source file is newer: ClinVar's records on the reference's contigs, multiallelic records split (`bcftools norm -m -any`) and indels left-aligned against the reference. Every sample then reuses it. The Nextflow module reuses this file when it sits beside the `--clinvar` file and is newer; otherwise it normalises ClinVar inside the task. +2. Builds `clinvar/clinvar_pathogenic_chr.norm.vcf.gz` beside the ClinVar file the first time, and again whenever the source file is newer: ClinVar's records on the reference's contigs, multiallelic records split (`bcftools norm -m -any`) and indels left-aligned against the reference. Every sample then reuses it. Two samples screened at the same time build it once: the first takes `clinvar/.norm.lock` (a folder holding its process id, also without `flock`), the others wait, then find it built. The build goes to a `.part` file, and its index goes in place before the VCF, so the VCF and its `.tbi` always come from the same build. The Nextflow module reuses this file when it sits beside the `--clinvar` file and is newer; otherwise it normalises ClinVar inside the task. 3. Filters the sample VCF to PASS records. If the VCF has no PASS record at all (callers that leave FILTER as `.`), it uses `-f .,PASS` instead and prints a notice saying so. If no record is left, the step stops with an error. 4. Splits and left-aligns the sample the same way. `bcftools isec` matches only identical REF/ALT, so a pathogenic allele inside a multiallelic record (genotype `1/2`) is found only after this split. The split writes the other alt allele as missing, not as the reference (`bcftools norm --multi-overlaps .`): a `1/2` record gives `1/.` and `./1`, where the default would give `1/0` and `0/1`. Both copies of such a call carry an alt allele, so neither part may read as het. A poly-T tract is where this shows most: a call of 5 T on one copy and 9 T on the other, against a reference of 7, is one `1/2` record, and its 5 T allele is a ClinVar hit. The reports word such a hit "one copy, other copy not called ref (1/.)", never het; the ACMG and carrier lists count it as one copy of two. This step does not size the tract: a short-read call of a homopolymer length can be off by one or two bases, so check the reads. A sample record whose REF is not the reference's base at that position is kept as written (`bcftools norm -c w`), with a `REF_MISMATCH` warning in the log. Such a record cannot match a ClinVar allele, so the summary of item 11 counts these records, and prints a notice to check that the VCF was called on this reference when there are any. 5. Builds the combined alleles of neighbouring records. A caller can write one complex allele, a delins, as two records a few bases apart on the same haplotype, for example a deletion and an insertion. Matched one by one, the parts can hit two single-submitter ClinVar records and miss the better reviewed record ClinVar holds for the whole allele. `bcftools norm -m +any` joins only records at the same position, so it cannot build that allele. `og clinvar join-adjacent` pairs two records when the second starts at most 10 bases after the first ends (records that overlap are never paired) and their genotypes allow both ALT alleles on one haplotype: phased with the same genotype in the same phase set (`FORMAT/PS`), or unphased with the same genotype (`0/1` and `1/0` count as the same het call; an unphased call with a missing allele such as `./1` is never paired). A phased call with a missing allele, such as `1|.`, has its ALT on a known haplotype: it pairs with a record in the same phase set whose ALT sits on that haplotype (`1|0` or `1|.`), and the combined record keeps `.` on the haplotype where either part is missing. That `.` means the copy was not called there; when the part comes from a split multiallelic record, it holds the other alt allele. Two unphased het calls can also sit on different haplotypes, and the genotype cannot tell which, so the notes of a combined allele and its parts built from them end with "unphased, so the parts may sit on different haplotypes". The combined record runs from the first part's start to the second part's end; `og clinvar join-adjacent` takes the bases between the parts from the reference (an uncompressed FASTA with its `.fai`), and `bcftools norm` trims and left-aligns it the way ClinVar was. The gene is not checked here, because the sample VCF has no gene annotation yet: a combined allele only counts when ClinVar holds a record for it. The step prints how many combined alleles it built. Only pairs are built: an allele split into three or more records is matched by its pairs only. From a95f854921361be2dbee0d18f53c04a74eed69e4 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:44:35 +0100 Subject: [PATCH 09/22] fix(family): pair each ALT of a 1/2 call in compound_het.tsv The compound-het block ran only for a child call with the reference allele, and looked up one ALT. GLnexus writes sites with two ALTs as one record, so a 1/2 child never reached the pairs. It now runs for any call with two different alleles and adds each ALT that is a candidate with its own origin: a 1/2 from a 0/1 father and a 0/2 mother is one pair in trans. A 0/1 call gives the same pairs as before. Bead: C02 --- rust/crates/og-steps/src/family/report.rs | 18 +- .../og-steps/tests/rv_family_trio_calls.rs | 372 ++++++++++++++++++ 2 files changed, 385 insertions(+), 5 deletions(-) create mode 100644 rust/crates/og-steps/tests/rv_family_trio_calls.rs diff --git a/rust/crates/og-steps/src/family/report.rs b/rust/crates/og-steps/src/family/report.rs index 0a9cd8905..6359e0ef5 100644 --- a/rust/crates/og-steps/src/family/report.rs +++ b/rust/crates/og-steps/src/family/report.rs @@ -483,11 +483,19 @@ impl Pass<'_> { ch.de_novo.push(row); } } - // compound heterozygote candidates - if het && let Some(table) = self.cands.get(&ch.name) { - let ai = if cg[0] == 0 { cg[1] } else { cg[0] }; - let alt = allele(cols, ai, path, line)?; - if let Some(genes) = table.get(&(chrom.to_vec(), pos, cols[3].to_vec(), alt.to_vec())) { + // compound heterozygote candidates: each ALT of a call with two + // different alleles, 0/1 or 1/2 (GLnexus leaves multiallelic records + // unsplit, the candidate VCF has one record per ALT) + if cg.len() == 2 + && cset.len() == 2 + && let Some(table) = self.cands.get(&ch.name) + { + for &ai in cset.iter().filter(|&&a| a != 0) { + let alt = allele(cols, ai, path, line)?; + let Some(genes) = table.get(&(chrom.to_vec(), pos, cols[3].to_vec(), alt.to_vec())) + else { + continue; + }; let first = if cols[3] == alt { 0 } else { diff --git a/rust/crates/og-steps/tests/rv_family_trio_calls.rs b/rust/crates/og-steps/tests/rv_family_trio_calls.rs new file mode 100644 index 000000000..2c1214441 --- /dev/null +++ b/rust/crates/og-steps/tests/rv_family_trio_calls.rs @@ -0,0 +1,372 @@ +//! `og family report` on small hand-made trios: the calls the trio report +//! missed or trusted too much. +//! +//! 1. Compound heterozygotes: a child call with two different ALTs (1/2, +//! as GLnexus writes an unsplit multiallelic record) is heterozygous, and +//! each ALT that is a candidate is paired with its origin, so a 1/2 from a +//! 0/1 father and a 0/2 mother is one pair in trans. A 0/1 child gives the +//! same pairs as before. +//! 2. A son's sex chromosomes outside the PARs: his Y comes from his father +//! and his X from his mother, so a de novo call there needs only that +//! parent's call, and the depth and quality floors apply only to him and +//! that parent. +//! 3. Parental reads: a parent called 0/0 with reads on the new allele (a +//! mosaic parent) keeps a de novo candidate out of `pass`. + +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; + +fn par_bed() -> String { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../../assets/par_grch38.bed") + .display() + .to_string() +} + +fn scratch(name: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("og-family-trio-{}-{name}", std::process::id())); + let _ = fs::remove_dir_all(&d); + fs::create_dir_all(&d).unwrap(); + d +} + +/// A trio: `kid` (sex `kid_sex`), father `dad`, mother `mum`, all three +/// sequenced. +fn members(d: &Path, kid_sex: &str) -> String { + let p = d.join("members.tsv"); + fs::write( + &p, + format!( + "sample\tsex\tfather\tmother\tpartner\tin_pedigree\tsequenced\n\ + kid\t{kid_sex}\tdad\tmum\t\tyes\tyes\n\ + dad\tmale\t\t\tmum\tyes\tyes\n\ + mum\tfemale\t\t\tdad\tyes\tyes\n" + ), + ) + .unwrap(); + p.display().to_string() +} + +/// A joint VCF with the columns kid, dad, mum and FORMAT GT:AD:DP:GQ; each +/// record is `(chrom, pos, ref, alt, [kid, dad, mum])`. +fn joint_vcf(d: &Path, records: &[(&str, i64, &str, &str, [&str; 3])]) -> String { + let mut out = String::from( + "##fileformat=VCFv4.2\n##contig=\n\ + ##contig=\n##contig=\n\ + ##contig=\n\ + ##FORMAT=\n\ + ##FORMAT=\n\ + ##FORMAT=\n\ + ##FORMAT=\n\ + #CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tkid\tdad\tmum\n", + ); + for (chrom, pos, r, a, s) in records { + out += &format!( + "{chrom}\t{pos}\t.\t{r}\t{a}\t50\tPASS\t.\tGT:AD:DP:GQ\t{}\t{}\t{}\n", + s[0], s[1], s[2] + ); + } + let p = d.join("family.vcf"); + fs::write(&p, out).unwrap(); + p.display().to_string() +} + +/// The child's candidate VCF as step 23 leaves it: split per ALT, the gene +/// in INFO/GENE. Each record is `(chrom, pos, ref, alt, gene)`. +fn candidates(d: &Path, records: &[(&str, i64, &str, &str, &str)]) -> String { + let mut out = + String::from("##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n"); + for (chrom, pos, r, a, g) in records { + out += &format!("{chrom}\t{pos}\t.\t{r}\t{a}\t50\tPASS\tGENE={g}\n"); + } + let p = d.join("kid.candidates.vcf"); + fs::write(&p, out).unwrap(); + p.display().to_string() +} + +/// Runs `og family report` in `d` and returns the output folder. +fn report(d: &Path, members: &str, vcf: &str, cands: Option<&str>) -> PathBuf { + let rel = d.join("relationships.tsv"); + fs::write( + &rel, + "sample_a\tsample_b\tdeclared\texpected_relatedness\trelatedness\tibs0_rate\tsites\tverdict\treason\n", + ) + .unwrap(); + let out = d.join("out"); + let par = par_bed(); + let cand = cands.map(|c| format!("kid={c}")); + let mut args = vec![ + "report", + "--members", + members, + "--relationships", + rel.to_str().unwrap(), + "--vcf", + vcf, + "--par-bed", + &par, + "--out-dir", + out.to_str().unwrap(), + ]; + if let Some(c) = &cand { + args.extend(["--candidates", c.as_str()]); + } + let args: Vec = args.iter().map(|s| (*s).to_owned()).collect(); + assert_eq!(og_steps::family::cli(&args), 0, "og family report exits 0"); + out +} + +/// The rows of a TSV as maps from the header. +fn read_tsv(path: &Path) -> Vec> { + let text = fs::read_to_string(path).unwrap(); + let mut lines = text.lines(); + let head: Vec<&str> = lines.next().unwrap().split('\t').collect(); + lines + .map(|l| { + head.iter() + .zip(l.split('\t')) + .map(|(h, v)| ((*h).to_owned(), v.to_owned())) + .collect() + }) + .collect() +} + +fn pairs(out: &Path) -> Vec> { + read_tsv(&out.join("compound_het.tsv")) + .into_iter() + .map(|r| { + [ + "child", + "gene", + "variant_a", + "variant_b", + "a_from", + "b_from", + "phase", + ] + .iter() + .map(|k| r[*k].clone()) + .collect() + }) + .collect() +} + +/// The de novo rows at `chrom:pos`, as (child_gt, quality). +fn de_novo_at(out: &Path, chrom: &str, pos: i64) -> Vec<(String, String)> { + read_tsv(&out.join("de_novo.tsv")) + .into_iter() + .filter(|r| r["chrom"] == chrom && r["pos"] == pos.to_string()) + .map(|r| (r["child_gt"].clone(), r["quality"].clone())) + .collect() +} + +const GOOD_REF: &str = "0/0:30,0:30:99"; + +// ---- 1. compound heterozygotes ------------------------------------------ + +#[test] +fn a_two_alt_call_from_both_parents_is_one_pair_in_trans() { + let d = scratch("two-alt"); + let m = members(&d, "female"); + let vcf = joint_vcf( + &d, + &[( + "chr1", + 1_000_000, + "C", + "A,G", + [ + "1/2:0,15,15:30:99", + "0/1:15,15,0:30:99", + "0/2:15,0,15:30:99", + ], + )], + ); + let c = candidates( + &d, + &[ + ("chr1", 1_000_000, "C", "A", "G1"), + ("chr1", 1_000_000, "C", "G", "G1"), + ], + ); + let out = report(&d, &m, &vcf, Some(&c)); + assert_eq!( + pairs(&out), + vec![vec![ + "kid".to_owned(), + "G1".into(), + "chr1:1000000:C>A".into(), + "chr1:1000000:C>G".into(), + "father".into(), + "mother".into(), + "in trans".into(), + ]], + "a 1/2 child of a 0/1 father and a 0/2 mother: one pair in trans" + ); + let md = fs::read_to_string(out.join("family_report.md")).unwrap(); + assert!( + md.contains( + "in one gene: 1; in trans (one from each parent, both copies of the gene hit) 1" + ), + "the report counts the pair: {md}" + ); +} + +#[test] +fn a_two_alt_call_pairs_with_a_ref_alt_call_in_the_gene() { + let d = scratch("two-alt-plus-one"); + let m = members(&d, "male"); + let vcf = joint_vcf( + &d, + &[ + ( + "chr1", + 1_000_000, + "C", + "A,G", + [ + "1/2:0,15,15:30:99", + "0/1:15,15,0:30:99", + "0/2:15,0,15:30:99", + ], + ), + ( + "chr1", + 1_000_500, + "T", + "C", + ["0/1:15,15:30:99", "0/1:15,15:30:99", GOOD_REF], + ), + ], + ); + // only the G of the 1/2 is a candidate, with the 0/1 site from the father + let c = candidates( + &d, + &[ + ("chr1", 1_000_000, "C", "G", "G1"), + ("chr1", 1_000_500, "T", "C", "G1"), + ], + ); + let out = report(&d, &m, &vcf, Some(&c)); + assert_eq!( + pairs(&out), + vec![vec![ + "kid".to_owned(), + "G1".into(), + "chr1:1000000:C>G".into(), + "chr1:1000500:T>C".into(), + "mother".into(), + "father".into(), + "in trans".into(), + ]], + "the G of a 1/2 (from the mother) and a 0/1 from the father: one pair in trans" + ); +} + +#[test] +fn ref_alt_calls_pair_as_before() { + let d = scratch("ref-alt"); + let m = members(&d, "female"); + let vcf = joint_vcf( + &d, + &[ + // from the father + ( + "chr1", + 2_000_000, + "A", + "G", + ["0/1:15,15:30:99", "0/1:15,15:30:99", GOOD_REF], + ), + // from the mother, written ALT first + ( + "chr1", + 2_000_100, + "C", + "T", + ["1/0:15,15:30:99", GOOD_REF, "0/1:15,15:30:99"], + ), + // from the father again + ( + "chr1", + 2_000_200, + "G", + "A", + ["0/1:15,15:30:99", "1/1:0,30:30:99", GOOD_REF], + ), + // both parents het: origin unknown + ( + "chr2", + 3_000_000, + "T", + "G", + ["0/1:15,15:30:99", "0/1:15,15:30:99", "0/1:15,15:30:99"], + ), + ], + ); + let c = candidates( + &d, + &[ + ("chr1", 2_000_000, "A", "G", "G2"), + ("chr1", 2_000_100, "C", "T", "G2"), + ("chr1", 2_000_200, "G", "A", "G2"), + ("chr2", 3_000_000, "T", "G", "G2"), + ], + ); + let out = report(&d, &m, &vcf, Some(&c)); + let row = |a: &str, b: &str, fa: &str, fb: &str, ph: &str| { + ["kid", "G2", a, b, fa, fb, ph] + .iter() + .map(|s| (*s).to_owned()) + .collect::>() + }; + assert_eq!( + pairs(&out), + vec![ + row( + "chr1:2000000:A>G", + "chr1:2000100:C>T", + "father", + "mother", + "in trans" + ), + row( + "chr1:2000000:A>G", + "chr1:2000200:G>A", + "father", + "father", + "in cis" + ), + row( + "chr1:2000000:A>G", + "chr2:3000000:T>G", + "father", + ".", + "unknown" + ), + row( + "chr1:2000100:C>T", + "chr1:2000200:G>A", + "mother", + "father", + "in trans" + ), + row( + "chr1:2000100:C>T", + "chr2:3000000:T>G", + "mother", + ".", + "unknown" + ), + row( + "chr1:2000200:G>A", + "chr2:3000000:T>G", + "father", + ".", + "unknown" + ), + ], + "0/1 children pair as before" + ); +} From 4c20d16a8a2a80c9a8d464f389e2c0bbc83f526a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:44:50 +0100 Subject: [PATCH 10/22] fix(family): report a son's X and Y de novo calls and check parental reads A de novo row needed a call from both parents, so a son's chrY with the mother not called (the usual case) and his chrX with the father not called gave none, though only the father passes on the Y and only the mother the X. The transmitting parents are now worked out once: de novo needs their calls only, and the DP and GQ floors of pass apply to the child and those parents only. pass read AD for the child only, so a parent called 0/0 with reads on the new allele (a mosaic parent) still passed. Each transmitting parent's AD is now read: 2 or more reads, or 5% or more of its reads, on the new allele makes the row low. family_report.md says so. Bead: C20 Bead: C06 --- rust/crates/og-steps/src/family/report.rs | 124 ++++++++--- .../og-steps/tests/rv_family_trio_calls.rs | 196 ++++++++++++++++++ 2 files changed, 289 insertions(+), 31 deletions(-) diff --git a/rust/crates/og-steps/src/family/report.rs b/rust/crates/og-steps/src/family/report.rs index 6359e0ef5..3af701b1e 100644 --- a/rust/crates/og-steps/src/family/report.rs +++ b/rust/crates/og-steps/src/family/report.rs @@ -44,10 +44,25 @@ pub const SPEC: Spec = Spec { required: &["--members", "--relationships", "--vcf", "--out-dir"], }; -/// De novo candidates: every one of the three must have these. +/// De novo candidates: the child and each parent who passes the site on +/// must have these. const MIN_GQ: i128 = 20; const MIN_DP: i128 = 10; const HET_AB: (f64, f64) = (0.25, 0.75); +/// De novo candidates: a parent who passes the site on with this many reads, +/// or this share of its reads, on the new allele keeps the row from `pass`. +const PARENT_ALT_READS: i128 = 2; +const PARENT_ALT_FRACTION: f64 = 0.05; + +/// The parents who pass a site on to the child: both on the autosomes and +/// on a daughter's X, the mother on a son's X and the father on his Y +/// (outside the pseudoautosomal regions). +#[derive(Clone, Copy)] +struct Need { + father: bool, + mother: bool, +} + /// Bytes of inheritance rows handed to a child's compressing thread at a time. const CHUNK: usize = 1 << 18; /// The level of the inheritance files (the content is what is compared). @@ -454,32 +469,47 @@ impl Pass<'_> { b[usize::from(cg[0] != f)] += 1; } } - // de novo: an allele the child carries that neither parent has - if how == How::Inconsistent && f_known && m_known { - let (f, m) = ( - fset.as_deref().unwrap_or(&[]), - mset.as_deref().unwrap_or(&[]), - ); - // on a male's single X only the mother's alleles count, on his Y the father's; - // a female's two X come one from each parent, so both parents' alleles count - let seen: Vec = if sexchr && is_x { - if ch.sex.as_deref() == Some("male") { - m.to_vec() - } else { - [f, m].concat() - } - } else if sexchr { - f.to_vec() - } else { - [f, m].concat() - }; + // de novo: an allele the child carries that no parent who passes + // this site on has. A male's single X comes from his mother and his + // Y from his father (a female never reaches here on Y); a female's + // two X and the autosomes come one from each parent. Only those + // parents need a call here: the other one usually has none. + let need = if sexchr && !is_x { + Need { + father: true, + mother: false, + } + } else if sexchr && ch.sex.as_deref() == Some("male") { + Need { + father: false, + mother: true, + } + } else { + Need { + father: true, + mother: true, + } + }; + if how == How::Inconsistent + && ch.f_seq + && ch.m_seq + && (!need.father || f_known) + && (!need.mother || m_known) + { + let mut seen: Vec = Vec::new(); + if need.father { + seen.extend(fset.as_deref().unwrap_or(&[])); + } + if need.mother { + seen.extend(mset.as_deref().unwrap_or(&[])); + } let novel: Vec = cset .iter() .copied() .filter(|a| *a != 0 && !seen.contains(a)) .collect(); if !novel.is_empty() { - let row = de_novo_row(ch, cols, format, gi, &novel, &cset, calls, path)?; + let row = de_novo_row(ch, cols, format, gi, &novel, &cset, need, calls, path)?; ch.de_novo.push(row); } } @@ -530,6 +560,7 @@ fn de_novo_row( gi: usize, novel: &[u32], cset: &[u32], + need: Need, calls: &mut Calls<'_>, path: &str, ) -> std::result::Result, Die> { @@ -545,7 +576,21 @@ fn de_novo_row( None => Ok(None), } } + /// The AD counts of a sample, `None` when it has none. + fn reads( + calls: &mut Calls<'_>, + format: &[u8], + col: usize, + ) -> std::result::Result>, Die> { + Ok(field(calls, format, col, b"AD")?.map(|ad| { + ad.split(|&c| c == b',') + .map(|x| to_int(Some(x)).unwrap_or(0)) + .collect() + })) + } let three = [ch.col, ch.fcol, ch.mcol]; + // the child and the parents who pass this site on + let counts = [true, need.father, need.mother]; let mut dp = [None; 3]; let mut gq = [None; 3]; for (k, &c) in three.iter().enumerate() { @@ -553,22 +598,35 @@ fn de_novo_row( gq[k] = to_int(field(calls, format, c, b"GQ")?); } let mut ab = None; - if let Some(ad) = field(calls, format, ch.col, b"AD")? { - let n: Vec = ad - .split(|&c| c == b',') - .map(|x| to_int(Some(x)).unwrap_or(0)) - .collect(); + if let Some(n) = reads(calls, format, ch.col)? { let tot: i128 = n.iter().sum(); if tot != 0 { ab = n.get(novel[0] as usize).map(|&v| v as f64 / tot as f64); } } let het = cset.len() == 2; - let mut good = gq.iter().all(|x| x.is_some_and(|v| v >= MIN_GQ)) - && dp.iter().all(|x| x.is_some_and(|v| v >= MIN_DP)); + let mut good = (0..3) + .filter(|&k| counts[k]) + .all(|k| gq[k].is_some_and(|v| v >= MIN_GQ) && dp[k].is_some_and(|v| v >= MIN_DP)); if het && ab.is_none_or(|ab| !(HET_AB.0 <= ab && ab <= HET_AB.1)) { good = false; } + // a parent called without the new allele whose reads still show it + // (a mosaic parent, or a missed call): not a clean de novo + for k in (1..3).filter(|&k| counts[k]) { + if let Some(n) = reads(calls, format, three[k])? { + let alt: i128 = novel + .iter() + .map(|&a| n.get(a as usize).copied().unwrap_or(0)) + .sum(); + let tot: i128 = n.iter().sum(); + if alt >= PARENT_ALT_READS + || (tot > 0 && alt as f64 / tot as f64 >= PARENT_ALT_FRACTION) + { + good = false; + } + } + } let mut alts = Vec::new(); for &i in novel { alts.push(text(allele(cols, i, path, line)?)); @@ -972,10 +1030,14 @@ fn markdown( String::new(), format!( "De novo candidates (an allele neither parent has): {}, {good} of them with GQ >= {MIN_GQ} and \ - depth >= {MIN_DP} in all three and, for a heterozygous call, an allele balance of {} to {} \ - (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that \ - matters needs confirming by another method.", + depth >= {MIN_DP} in the child and in each parent who passes the site on (on a son's X \ + outside the pseudoautosomal regions the mother, on his Y the father), fewer than \ + {PARENT_ALT_READS} reads and under {}% of the reads on the new allele in each such \ + parent and, for a heterozygous call, an allele balance of {} to {} (`de_novo.tsv`). \ + Most candidates of a genome are genotype errors; a candidate that matters needs \ + confirming by another method.", ch.de_novo.len(), + fixed(100.0 * PARENT_ALT_FRACTION, 0), fixed(HET_AB.0, 2), fixed(HET_AB.1, 2) ), diff --git a/rust/crates/og-steps/tests/rv_family_trio_calls.rs b/rust/crates/og-steps/tests/rv_family_trio_calls.rs index 2c1214441..67159f880 100644 --- a/rust/crates/og-steps/tests/rv_family_trio_calls.rs +++ b/rust/crates/og-steps/tests/rv_family_trio_calls.rs @@ -370,3 +370,199 @@ fn ref_alt_calls_pair_as_before() { "0/1 children pair as before" ); } + +// ---- 2. a son's X and Y outside the PARs ----------------------------------- + +#[test] +fn a_sons_y_de_novo_needs_only_the_father() { + let d = scratch("son-y"); + let m = members(&d, "male"); + let vcf = joint_vcf( + &d, + &[ + // the mother has no call on Y, as always + ( + "chrY", + 15_000_000, + "A", + "T", + ["1:0,30:30:99", "0:30,0:30:99", "./.:.:.:."], + ), + // the father has no call either: nothing tells + ( + "chrY", + 15_000_100, + "C", + "G", + ["1:0,30:30:99", "./.:.:.:.", "./.:.:.:."], + ), + // the father's allele: inherited, no row + ( + "chrY", + 15_000_200, + "G", + "T", + ["1:0,30:30:99", "1:0,30:30:99", "./.:.:.:."], + ), + ], + ); + let out = report(&d, &m, &vcf, None); + assert_eq!( + de_novo_at(&out, "chrY", 15_000_000), + vec![("1".to_owned(), "pass".to_owned())], + "a son's Y allele his father does not have, the mother with no call: one pass row" + ); + assert!( + de_novo_at(&out, "chrY", 15_000_100).is_empty(), + "no father call on Y: no row" + ); + assert!( + de_novo_at(&out, "chrY", 15_000_200).is_empty(), + "the father's allele on Y: no row" + ); +} + +#[test] +fn a_sons_x_de_novo_needs_only_the_mother() { + let d = scratch("son-x"); + let m = members(&d, "male"); + let vcf = joint_vcf( + &d, + &[ + // the father has no call on X + ( + "chrX", + 20_000_000, + "A", + "T", + ["1:0,30:30:99", "./.:.:.:.", GOOD_REF], + ), + // the father's call is weak: it does not count on a son's X + ( + "chrX", + 20_000_100, + "C", + "G", + ["1:0,30:30:99", "0:3,0:3:5", GOOD_REF], + ), + // the mother's call is weak: low + ( + "chrX", + 20_000_200, + "G", + "T", + ["1:0,30:30:99", "0:30,0:30:99", "0/0:5,0:5:10"], + ), + // the father carries the allele, the mother does not: still new on a son's X + ( + "chrX", + 20_000_300, + "T", + "C", + ["1:0,30:30:99", "1:0,30:30:99", GOOD_REF], + ), + ], + ); + let out = report(&d, &m, &vcf, None); + assert_eq!( + de_novo_at(&out, "chrX", 20_000_000), + vec![("1".to_owned(), "pass".to_owned())], + "a son's X allele his mother does not have, the father with no call: one pass row" + ); + assert_eq!( + de_novo_at(&out, "chrX", 20_000_100), + vec![("1".to_owned(), "pass".to_owned())], + "the father's depth and quality do not count on a son's X" + ); + assert_eq!( + de_novo_at(&out, "chrX", 20_000_200), + vec![("1".to_owned(), "low".to_owned())], + "the mother's depth and quality still count" + ); + assert_eq!( + de_novo_at(&out, "chrX", 20_000_300), + vec![("1".to_owned(), "pass".to_owned())], + "a father's X allele is not a son's" + ); +} + +// ---- 3. parental reads on the new allele ----------------------------------- + +#[test] +fn a_parents_reads_on_the_new_allele_keep_it_out_of_pass() { + let d = scratch("parent-reads"); + let m = members(&d, "female"); + let kid = "0/1:15,15:30:99"; + let vcf = joint_vcf( + &d, + &[ + // a mosaic father: 3 of 30 reads on the new allele + ( + "chr2", + 5_000_000, + "A", + "G", + [kid, "0/0:27,3:30:40", GOOD_REF], + ), + // the same trio, no reads on it + ( + "chr2", + 5_000_100, + "A", + "G", + [kid, "0/0:30,0:30:40", GOOD_REF], + ), + // one read of 15 (7%) in the mother + ( + "chr2", + 5_000_200, + "C", + "T", + [kid, GOOD_REF, "0/0:14,1:15:40"], + ), + // one read of 30 (3%) in the mother + ( + "chr2", + 5_000_300, + "C", + "T", + [kid, GOOD_REF, "0/0:29,1:30:40"], + ), + // two reads on a new allele of a 1/2 child, in the mother + ( + "chr2", + 5_000_400, + "C", + "A,T", + ["1/2:0,15,15:30:99", "0/1:15,15,0:30:99", "0/0:58,0,2:60:40"], + ), + ], + ); + let out = report(&d, &m, &vcf, None); + let at = |p| de_novo_at(&out, "chr2", p); + assert_eq!( + at(5_000_000), + vec![("0/1".to_owned(), "low".to_owned())], + "a father 0/0 with AD=27,3: not pass" + ); + assert_eq!( + at(5_000_100), + vec![("0/1".to_owned(), "pass".to_owned())], + "a father 0/0 with AD=30,0: pass" + ); + assert_eq!( + at(5_000_200), + vec![("0/1".to_owned(), "low".to_owned())], + "one read of 15 in the mother (5% or more): not pass" + ); + assert_eq!( + at(5_000_300), + vec![("0/1".to_owned(), "pass".to_owned())], + "one read of 30 in the mother (under 5%): pass" + ); + assert_eq!( + at(5_000_400), + vec![("1/2".to_owned(), "low".to_owned())], + "two reads on the new ALT of a 1/2 in the mother: not pass" + ); +} From 48febd88a8e19f4a92888c02d5abee3cf43127a2 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:44:56 +0100 Subject: [PATCH 11/22] docs(family): say which calls make a de novo pass and that 1/2 pairs count docs/multi-sample.md and the changelog now say that a son's X and Y candidates need only the transmitting parent, that pass checks each such parent's reads on the new allele, and that a call with two ALTs is heterozygous for compound_het.tsv. Bead: C02 Bead: C20 Bead: C06 --- CHANGELOG.md | 5 +++++ docs/multi-sample.md | 10 ++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05c3388e1..68f9189a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Step 38's trio report (`og family report`) finds three kinds of call it missed or trusted too much. Each output keeps its name and columns; only which rows appear and their `quality` change. + - A child call with two different ALTs (`1/2`, as GLnexus writes a site with two ALTs) now counts as heterozygous for `compound_het.tsv`. Before, only a call with the reference allele did, so a `1/2` from a `0/1` father and a `0/2` mother, with both ALTs candidates in one gene, gave no row. Each ALT that is a candidate is now paired with its own origin, so that call is one pair in trans. A `0/1` call gives the same pairs as before. + - A son's de novo candidate on chrX or chrY outside the pseudoautosomal regions needs only the parent who passes that chromosome on: the mother for his X, the father for his Y. Before, both parents needed a call, so a son's Y with the mother not called (the usual case) and his X with the father not called gave no row. The depth and GQ floors of `pass` now apply to the child and those parents only. + - A de novo candidate is `low`, not `pass`, when a parent who passes the site on has 2 or more reads, or 5% or more of its reads, on the new allele (from its `AD`). Before, only the child's reads were read, so a parent called `0/0` with reads on the allele, as a mosaic parent has, still gave `pass`. + - The self-test of `scripts/ci/check-images.sh` copies the whole tree again once the list of tracked paths passes 128 KiB. It piped that list through `xargs` to `tar`, and GNU `xargs` split a list that long over two `tar` runs; the reading `tar` stopped after the first archive, so Guard's `images` job failed on a broken pipe (exit 123). One `tar` now reads the list itself (`tar --null -T -`). Nothing the check reports changes. - The phased family VCF is indexed by the Rust `og` binary. WhatsHap's step (step 38 and the Nextflow WHATSHAP_PED process) wrote `family.phased.vcf.gz.tbi` with a Python one-liner, `pysam.tabix_index`. It now runs `og vcf index -f`. The `.tbi` keeps its name and format, and its bytes, once inflated, are the ones htslib 1.24 writes (the htslib of the pysam and bcftools the pipeline pins): the same bins in the same order, the same chunks and the same linear index. Only the deflate stream can differ, as two deflate libraries differ. On a 5-million-record three-sample VCF it took 0.75 to 0.91 s and 31 MB at peak, where htslib's own `tabix` took 0.81 to 0.93 s and 19 MB. - As in htslib 1.24, a record ends at the largest of INFO/END, the REF length, the SVLEN of a ``, ``, `` or `` ALT, and a sample's LEN on a gVCF block. A line with no position to read, unsorted records, a sequence that starts again, a record past 2^29 and an empty file stop the index with a message, and nothing is written. Without `-f` it never writes over an index, even one that appears while it runs. diff --git a/docs/multi-sample.md b/docs/multi-sample.md index c8c60b3d9..20b09c210 100644 --- a/docs/multi-sample.md +++ b/docs/multi-sample.md @@ -74,9 +74,15 @@ It reads the `family.vcf.gz` and `pedigree.tsv` the earlier run left in `$GENOME 3. **Phasing by inheritance.** For each child whose two parents are sequenced, WhatsHap (`WHATSHAP_IMAGE`, `whatshap phase --ped`) phases the family by inheritance alone; no reads are given. A site where the child and both parents are heterozygous stays unphased, because no genotype tells it. 4. **The report.** For each child with a sequenced parent, `_inheritance.tsv.gz` gives, at every site of the joint VCF where the child and a sequenced parent have a genotype, the allele from the father and the allele from the mother when the genotypes tell (`how` is `genotype`), or `ambiguous` (both parents heterozygous like the child) or `inconsistent` (a genotype error, a deletion or a new mutation). On chrX and chrY outside the pseudoautosomal regions a son's X comes from his mother and his Y from his father; the child's sex must be known there. chrM is left out. With one parent sequenced, the allele the other parent passed on is known wherever the sequenced parent is homozygous or differs from the child: that is the half of the unsequenced parent's genome the child carries, and `family_report.md` gives its share of the sites. The half the child did not inherit is not seen. - De novo candidates (`de_novo.tsv`, a child allele neither parent has) are called only when both parents are sequenced. A candidate is `pass` with GQ 20 or more and depth 10 or more in all three and, when the child is heterozygous, 25% to 75% of the child's reads on the new allele; otherwise `low`. Most candidates of a whole genome are genotype errors: confirm one that matters by another method. + De novo candidates (`de_novo.tsv`, a child allele neither parent has) are called only when both parents are sequenced. On chrX and chrY outside the pseudoautosomal regions a son's candidate needs only the parent who passes that chromosome on: the mother for his X, the father for his Y. The other parent may have no call there. A candidate is `pass` when all of these hold; otherwise it is `low`: - When step 23 (the clinical filter) ran for a child, every pair of its candidate variants the child carries heterozygous in one gene is listed in `compound_het.tsv`: `in trans` when one came from each parent (both copies of the gene are hit), `in cis` when both came from one parent, `unknown` when the genotypes do not tell. + - GQ 20 or more and depth 10 or more in the child and in each parent who passes the site on (both parents, except on a son's X and Y); + - in each such parent, fewer than 2 reads and under 5% of the reads on the new allele (the parent's `AD`; a parent with no `AD` is not checked). A parent called homozygous reference with reads on the new allele can be mosaic for it; + - when the child is heterozygous, 25% to 75% of the child's reads on the new allele. + + Most candidates of a whole genome are genotype errors: confirm one that matters by another method. + + When step 23 (the clinical filter) ran for a child, every pair of its candidate variants the child carries heterozygous in one gene is listed in `compound_het.tsv`. A call with two different ALTs (`1/2`, as GLnexus writes a site with two ALTs) is heterozygous too: each of its ALTs that is a candidate counts on its own, so a `1/2` with one ALT from each parent is a pair in trans. The pairs are `in trans` when one came from each parent (both copies of the gene are hit), `in cis` when both came from one parent, `unknown` when the genotypes do not tell. 5. **The chromosome map.** For each child with a sequenced parent, `og family report` maps the stretches of each chromosome the child got from that parent. The joint VCF must be sorted by position: an unsorted one stops the report with exit 2, naming the two records. It reads the joint VCF's genotypes at the sites where the parent is heterozygous and the genotypes give the allele passed on. When one of that parent's own parents is sequenced too, each such site also says which of the parent's two copies was passed on, the grandfather's or the grandmother's, and a switch from one to the other along a chromosome is a crossover. Without a sequenced grandparent the stretches say only "from this parent". With one parent sequenced, the child's other copy is written over the same stretches as the copy that did not come from the sequenced parent. From 15c8fd482e9c9197579e18309af5f592dbee37d4 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:04:29 +0100 Subject: [PATCH 12/22] fix(clinvar): stop step 06 when the stale normalised ClinVar's folder is read-only lock_acquire waits for as long as mkdir fails, so a clinvar/ folder the user cannot write would hold the step forever where it used to fail on the write. The step now says so and exits 1; the fake-docker case checks it as an ordinary user. Bead: PL01 --- docs/06-clinvar-screen.md | 2 +- scripts/06-clinvar-screen.sh | 6 ++++++ .../fake-docker/rv-step06-screen-integrity.sh | 18 ++++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/docs/06-clinvar-screen.md b/docs/06-clinvar-screen.md index c92da1458..e3143e77d 100644 --- a/docs/06-clinvar-screen.md +++ b/docs/06-clinvar-screen.md @@ -49,7 +49,7 @@ In the Nextflow pipeline the same floor is `ext.min_gq` and `ext.min_dp` of `CLI ### What the Script Does 1. Lists the contigs each file holds records on (`bcftools index -s`) and keeps those the reference has too: ClinVar's contigs that are in the reference, and the sample's contigs among those. `bcftools norm` stops at the first record whose contig the reference lacks, such as a provider's unplaced scaffold or one of the `NT_` contigs the full ClinVar file has, and a record on a contig the other file lacks cannot match anyway. The step prints how many of the sample's records it leaves out and on which contigs. When no contig is shared it stops with an error instead of reporting zero hits, and says which file is named the other way (`1, 2, ...` against `chr1, chr2, ...`). -2. Builds `clinvar/clinvar_pathogenic_chr.norm.vcf.gz` beside the ClinVar file the first time, and again whenever the source file is newer: ClinVar's records on the reference's contigs, multiallelic records split (`bcftools norm -m -any`) and indels left-aligned against the reference. Every sample then reuses it. Two samples screened at the same time build it once: the first takes `clinvar/.norm.lock` (a folder holding its process id, also without `flock`), the others wait, then find it built. The build goes to a `.part` file, and its index goes in place before the VCF, so the VCF and its `.tbi` always come from the same build. The Nextflow module reuses this file when it sits beside the `--clinvar` file and is newer; otherwise it normalises ClinVar inside the task. +2. Builds `clinvar/clinvar_pathogenic_chr.norm.vcf.gz` beside the ClinVar file the first time, and again whenever the source file is newer: ClinVar's records on the reference's contigs, multiallelic records split (`bcftools norm -m -any`) and indels left-aligned against the reference. Every sample then reuses it. Two samples screened at the same time build it once: the first takes `clinvar/.norm.lock` (a folder holding its process id, also without `flock`), the others wait, then find it built. The build goes to a `.part` file, and its index goes in place before the VCF, so the VCF and its `.tbi` always come from the same build. When the copy must be rebuilt and `clinvar/` is not writable by you, the step stops with an error. The Nextflow module reuses this file when it sits beside the `--clinvar` file and is newer; otherwise it normalises ClinVar inside the task. 3. Filters the sample VCF to PASS records. If the VCF has no PASS record at all (callers that leave FILTER as `.`), it uses `-f .,PASS` instead and prints a notice saying so. If no record is left, the step stops with an error. 4. Splits and left-aligns the sample the same way. `bcftools isec` matches only identical REF/ALT, so a pathogenic allele inside a multiallelic record (genotype `1/2`) is found only after this split. The split writes the other alt allele as missing, not as the reference (`bcftools norm --multi-overlaps .`): a `1/2` record gives `1/.` and `./1`, where the default would give `1/0` and `0/1`. Both copies of such a call carry an alt allele, so neither part may read as het. A poly-T tract is where this shows most: a call of 5 T on one copy and 9 T on the other, against a reference of 7, is one `1/2` record, and its 5 T allele is a ClinVar hit. The reports word such a hit "one copy, other copy not called ref (1/.)", never het; the ACMG and carrier lists count it as one copy of two. This step does not size the tract: a short-read call of a homopolymer length can be off by one or two bases, so check the reads. A sample record whose REF is not the reference's base at that position is kept as written (`bcftools norm -c w`), with a `REF_MISMATCH` warning in the log. Such a record cannot match a ClinVar allele, so the summary of item 11 counts these records, and prints a notice to check that the VCF was called on this reference when there are any. 5. Builds the combined alleles of neighbouring records. A caller can write one complex allele, a delins, as two records a few bases apart on the same haplotype, for example a deletion and an insertion. Matched one by one, the parts can hit two single-submitter ClinVar records and miss the better reviewed record ClinVar holds for the whole allele. `bcftools norm -m +any` joins only records at the same position, so it cannot build that allele. `og clinvar join-adjacent` pairs two records when the second starts at most 10 bases after the first ends (records that overlap are never paired) and their genotypes allow both ALT alleles on one haplotype: phased with the same genotype in the same phase set (`FORMAT/PS`), or unphased with the same genotype (`0/1` and `1/0` count as the same het call; an unphased call with a missing allele such as `./1` is never paired). A phased call with a missing allele, such as `1|.`, has its ALT on a known haplotype: it pairs with a record in the same phase set whose ALT sits on that haplotype (`1|0` or `1|.`), and the combined record keeps `.` on the haplotype where either part is missing. That `.` means the copy was not called there; when the part comes from a split multiallelic record, it holds the other alt allele. Two unphased het calls can also sit on different haplotypes, and the genotype cannot tell which, so the notes of a combined allele and its parts built from them end with "unphased, so the parts may sit on different haplotypes". The combined record runs from the first part's start to the second part's end; `og clinvar join-adjacent` takes the bases between the parts from the reference (an uncompressed FASTA with its `.fai`), and `bcftools norm` trims and left-aligns it the way ClinVar was. The gene is not checked here, because the sample VCF has no gene annotation yet: a combined allele only counts when ClinVar holds a record for it. The step prints how many combined alleles it built. Only pairs are built: an allele split into three or more records is matched by its pairs only. diff --git a/scripts/06-clinvar-screen.sh b/scripts/06-clinvar-screen.sh index ea153f125..ecd549004 100755 --- a/scripts/06-clinvar-screen.sh +++ b/scripts/06-clinvar-screen.sh @@ -138,6 +138,12 @@ SAMPLE_DROPPED=$(comm -23 <(printf '%s\n' "$SAMPLE_CONTIGS") <(printf '%s\n' "$S norm_stale() { [ ! -f "$CLINVAR_NORM" ] || [ ! -f "${CLINVAR_NORM}.tbi" ] || [ "$CLINVAR" -nt "$CLINVAR_NORM" ]; } if norm_stale; then NORM_LOCK="${GENOME_DIR}/clinvar/.norm.lock" + # lock_acquire waits for as long as mkdir fails, so a folder it cannot write + # would make it wait forever. + if [ ! -w "${GENOME_DIR}/clinvar" ]; then + echo "ERROR: ${CLINVAR_NORM} must be rebuilt, but ${GENOME_DIR}/clinvar is not writable by you." >&2 + exit 1 + fi lock_acquire "$NORM_LOCK" trap 'lock_release "$NORM_LOCK"' EXIT if norm_stale; then diff --git a/tests/fake-docker/rv-step06-screen-integrity.sh b/tests/fake-docker/rv-step06-screen-integrity.sh index 61b80e702..ca5bf3eab 100644 --- a/tests/fake-docker/rv-step06-screen-integrity.sh +++ b/tests/fake-docker/rv-step06-screen-integrity.sh @@ -181,3 +181,21 @@ done [ -z "$(find "$C" -name '*.part*')" ] || fail "two screens at once left part files: $(find "$C" -name '*.part*')" [ ! -e "${C}/.norm.lock" ] || fail "two screens at once left clinvar/.norm.lock behind" echo "two screens at once build the normalised ClinVar once, VCF and .tbi of one write: PASS" + +# --- A stale normalised ClinVar in a folder the user cannot write ---------------------- +# The lock is a folder made in clinvar/, so the step must stop with an error +# there, not wait for a lock it can never take. root writes anywhere, so +# this part needs an ordinary user, as on the CI runner. +if [ "$(id -u)" -eq 0 ]; then + echo "read-only clinvar/: not checked as root, which can write any folder" +else + touch -t 202001010000 "$NORM" "${NORM}.tbi" + rm -rf "${GENOME_DIR}/sample1/clinvar" + chmod a-w "$C" + run_rc readonly-clinvar timeout 60 "${SCRIPTS}/06-clinvar-screen.sh" sample1 + chmod u+w "$C" + [ "$RC" -ne 124 ] || fail "step 06 was still waiting after 60 s for a lock in a clinvar/ folder it cannot write" + [ "$RC" -ne 0 ] || fail "step 06 exited 0 with a stale normalised ClinVar it cannot rebuild" + output_has readonly-clinvar 'clinvar is not writable' + echo "a stale normalised ClinVar in a read-only folder stops the step with an error: PASS" +fi From d973ef61df2d4a7ebddf9b2842304d75496eb11f Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:00:55 +0100 Subject: [PATCH 13/22] fix(family): keep the too-few-sites reason under 1,000 sites The parity case family/check-few (300 sites, ibs0 600) read 'not counts of sites' where the oracle says too few to tell. Under MIN_SITES the sites reason stays, unless a count is negative. Bead: C18 --- CHANGELOG.md | 2 +- docs/multi-sample.md | 2 +- rust/crates/og-steps/src/family/check.rs | 17 +++++++++++--- .../og-steps/tests/rv_family_check_nan.rs | 23 ++++++++++++------- 4 files changed, 31 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5c103c31..36eda042a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- `og family check` (step 38 and the Nextflow `FAMILY_CHECK` process) no longer reports `ok` for a pair somalier could not measure. somalier writes `nan` as the relatedness when it had nothing to divide by. Every comparison with NaN is false, so the degree bins read it as unrelated, and a pair declared unrelated or distant passed as "unrelated, as declared". Such a pair, one with `inf`, and one with counts that cannot be counts (negative, or more opposite homozygotes than sites) now get the verdict `not_checked`, a reason that names the value ("somalier gave relatedness 'nan' for this pair: not measured") and a `WARNING:` line. The step still exits 0 for them. Before, `inf` stopped the step as a mismatch ("same person"). The columns of `relationships.tsv` do not change; `not_checked` was already one of its verdicts. +- `og family check` (step 38 and the Nextflow `FAMILY_CHECK` process) no longer reports `ok` for a pair somalier could not measure. somalier writes `nan` as the relatedness when it had nothing to divide by. Every comparison with NaN is false, so the degree bins read it as unrelated, and a pair declared unrelated or distant passed as "unrelated, as declared". Such a pair, one with `inf`, and one with counts that cannot be counts (negative, or more opposite homozygotes than sites) now get the verdict `not_checked`, a reason that names the value ("somalier gave relatedness 'nan' for this pair: not measured") and a `WARNING:` line. The step still exits 0 for them. Under 1,000 sites the reason stays "too few to tell", unless a count is negative. Before, `inf` stopped the step as a mismatch ("same person"). The columns of `relationships.tsv` do not change; `not_checked` was already one of its verdicts. - The self-test of `scripts/ci/check-images.sh` copies the whole tree again once the list of tracked paths passes 128 KiB. It piped that list through `xargs` to `tar`, and GNU `xargs` split a list that long over two `tar` runs; the reading `tar` stopped after the first archive, so Guard's `images` job failed on a broken pipe (exit 123). One `tar` now reads the list itself (`tar --null -T -`). Nothing the check reports changes. - The phased family VCF is indexed by the Rust `og` binary. WhatsHap's step (step 38 and the Nextflow WHATSHAP_PED process) wrote `family.phased.vcf.gz.tbi` with a Python one-liner, `pysam.tabix_index`. It now runs `og vcf index -f`. The `.tbi` keeps its name and format, and its bytes, once inflated, are the ones htslib 1.24 writes (the htslib of the pysam and bcftools the pipeline pins): the same bins in the same order, the same chunks and the same linear index. Only the deflate stream can differ, as two deflate libraries differ. On a 5-million-record three-sample VCF it took 0.75 to 0.91 s and 31 MB at peak, where htslib's own `tabix` took 0.81 to 0.93 s and 19 MB. - As in htslib 1.24, a record ends at the largest of INFO/END, the REF length, the SVLEN of a ``, ``, `` or `` ALT, and a sample's LEN on a gVCF block. A line with no position to read, unsorted records, a sequence that starts again, a record past 2^29 and an empty file stop the index with a message, and nothing is written. Without `-f` it never writes over an index, even one that appears while it runs. diff --git a/docs/multi-sample.md b/docs/multi-sample.md index 77fbbfda4..4aecf0690 100644 --- a/docs/multi-sample.md +++ b/docs/multi-sample.md @@ -70,7 +70,7 @@ It reads the `family.vcf.gz` and `pedigree.tsv` the earlier run left in `$GENOME ### What it does 1. **Joint call.** GLnexus (`GLNEXUS_IMAGE`, its DeepVariantWGS preset) merges the members' DeepVariant gVCFs into one VCF, `family.vcf.gz`. A site where one person has a variant gets the others' genotypes from their reference blocks, so "not called" and "matches the reference" are told apart. The container gets 16 GB (`GLNEXUS_MEM` in step 38, the `GLNEXUS` process in `conf/base.config`), and GLnexus's own `--mem-gbytes` budget is half of it. That budget sizes only GLnexus's database cache and write buffers; the alleles it discovers and the sites it merges are held outside it. So a budget near the limit leaves no room for them. Whether 16 GB is enough for three whole genomes under the half rule has not been measured yet; when the task is killed for memory, Nextflow retries it once with 32 GB (capped at `--max_memory`), and step 38 takes a larger `GLNEXUS_MEM`. -2. **Relationship check.** somalier extracts each member's genotypes at its sites from the gVCF and `somalier relate` measures the relatedness of every pair: about the share of the genome two people inherited from common ancestors (1 the same person, 0.5 parent and child or full siblings, 0.25 second degree, 0 unrelated). The pedigree gives the expected value. A pair must fall in the same degree as declared (bins at 0.707, 0.354, 0.177 and 0.088); a declared parent and child must also be opposite homozygotes at under 1% of the sites, which siblings are not; a pair declared unrelated or distant must not measure first degree. A pair that disagrees stops the step with exit 1, before any phasing, and `relationships.tsv` says which pair and why: two samples were swapped, or a pedigree line is wrong. A pair compared at fewer than 1,000 sites is reported as not checked. So is a pair somalier gave no relatedness for (`nan` or `inf`), or counts that cannot be counts (negative, or more opposite homozygotes than sites): the reason names the value, a `WARNING:` line says it, and the step goes on. +2. **Relationship check.** somalier extracts each member's genotypes at its sites from the gVCF and `somalier relate` measures the relatedness of every pair: about the share of the genome two people inherited from common ancestors (1 the same person, 0.5 parent and child or full siblings, 0.25 second degree, 0 unrelated). The pedigree gives the expected value. A pair must fall in the same degree as declared (bins at 0.707, 0.354, 0.177 and 0.088); a declared parent and child must also be opposite homozygotes at under 1% of the sites, which siblings are not; a pair declared unrelated or distant must not measure first degree. A pair that disagrees stops the step with exit 1, before any phasing, and `relationships.tsv` says which pair and why: two samples were swapped, or a pedigree line is wrong. A pair compared at fewer than 1,000 sites is reported as not checked. So is a pair somalier gave no relatedness for (`nan` or `inf`), or counts that cannot be counts (negative, or more opposite homozygotes than sites): the reason names the value, a `WARNING:` line says it, and the step goes on. Under 1,000 sites the reason stays the number of sites, unless a count is negative. 3. **Phasing by inheritance.** For each child whose two parents are sequenced, WhatsHap (`WHATSHAP_IMAGE`, `whatshap phase --ped`) phases the family by inheritance alone; no reads are given. A site where the child and both parents are heterozygous stays unphased, because no genotype tells it. 4. **The report.** For each child with a sequenced parent, `_inheritance.tsv.gz` gives, at every site of the joint VCF where the child and a sequenced parent have a genotype, the allele from the father and the allele from the mother when the genotypes tell (`how` is `genotype`), or `ambiguous` (both parents heterozygous like the child) or `inconsistent` (a genotype error, a deletion or a new mutation). On chrX and chrY outside the pseudoautosomal regions a son's X comes from his mother and his Y from his father; the child's sex must be known there. chrM is left out. With one parent sequenced, the allele the other parent passed on is known wherever the sequenced parent is homozygous or differs from the child: that is the half of the unsequenced parent's genome the child carries, and `family_report.md` gives its share of the sites. The half the child did not inherit is not seen. diff --git a/rust/crates/og-steps/src/family/check.rs b/rust/crates/og-steps/src/family/check.rs index 204f0d5c9..c88ddd441 100644 --- a/rust/crates/og-steps/src/family/check.rs +++ b/rust/crates/og-steps/src/family/check.rs @@ -86,12 +86,23 @@ fn verdict( /// comparison with NaN is false, so [`verdict`] would bin it as unrelated /// and call a pair declared unrelated or distant ok. A negative count, or /// more opposite homozygotes than sites, is no count somalier can make. +/// Under [`MIN_SITES`] sites [`verdict`] already says too few to tell, the +/// more useful reason, so only a negative count comes before it. fn unmeasured(rel_text: &str, rel: f64, ibs0: i128, ibs2: i128, n: i128) -> Option { - if ibs0 < 0 || ibs2 < 0 || n < 0 || ibs0 > n { - return Some(format!( + let counts = || { + format!( "somalier gave ibs0 {ibs0}, ibs2 {ibs2} and {n} sites for this pair: not counts \ of sites, not measured" - )); + ) + }; + if ibs0 < 0 || ibs2 < 0 || n < 0 { + return Some(counts()); + } + if n < MIN_SITES { + return None; + } + if ibs0 > n { + return Some(counts()); } if !rel.is_finite() { return Some(format!( diff --git a/rust/crates/og-steps/tests/rv_family_check_nan.rs b/rust/crates/og-steps/tests/rv_family_check_nan.rs index eca880d1a..c99634384 100644 --- a/rust/crates/og-steps/tests/rv_family_check_nan.rs +++ b/rust/crates/og-steps/tests/rv_family_check_nan.rs @@ -203,14 +203,21 @@ fn finite_relatedness_keeps_its_verdict() { assert_eq!(get(&row, "verdict"), "ok", "{row:?}"); assert_eq!(get(&row, "reason"), "unrelated, as declared", "{row:?}"); assert!(!err.contains("WARNING"), "stderr: {err}"); - // Too few sites keeps its own reason. - let (code, _, row) = check("few-partners", PARTNERS, "a\tb\t0.01\t6\t10\t50"); - assert_eq!(code, 0); - assert_eq!(get(&row, "verdict"), "not_checked", "{row:?}"); - assert!( - get(&row, "reason").starts_with("somalier compared the two at 50 sites"), - "{row:?}" - ); + // Too few sites keeps its own reason, also beside a nan relatedness or + // more opposite homozygotes than sites (the parity case family/check-few). + for (name, pair) in [ + ("few-partners", "a\tb\t0.01\t6\t10\t50"), + ("few-nan-partners", "a\tb\tnan\t6\t10\t50"), + ("few-ibs0-over-n-partners", "a\tb\t0.01\t600\t10\t50"), + ] { + let (code, _, row) = check(name, PARTNERS, pair); + assert_eq!(code, 0, "{name}"); + assert_eq!(get(&row, "verdict"), "not_checked", "{name}: {row:?}"); + assert!( + get(&row, "reason").starts_with("somalier compared the two at 50 sites"), + "{name}: {row:?}" + ); + } // A measured first degree for declared partners still stops the step. let (code, _, row) = check("close-partners", PARTNERS, "a\tb\t0.5\t1\t900\t5000"); assert_eq!(code, 1); From 2a7fbdedbb24b1e0c564691c7a127d51321f4aed Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:46:41 +0100 Subject: [PATCH 14/22] fix(family): pair the two ALTs of one 1/2 call in trans whatever the parents show --- CHANGELOG.md | 2 +- docs/multi-sample.md | 2 +- rust/crates/og-steps/src/family/report.rs | 20 +++++--- .../og-steps/tests/rv_family_trio_calls.rs | 48 ++++++++++++++++++- 4 files changed, 62 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68f9189a5..acaa3acca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased - Step 38's trio report (`og family report`) finds three kinds of call it missed or trusted too much. Each output keeps its name and columns; only which rows appear and their `quality` change. - - A child call with two different ALTs (`1/2`, as GLnexus writes a site with two ALTs) now counts as heterozygous for `compound_het.tsv`. Before, only a call with the reference allele did, so a `1/2` from a `0/1` father and a `0/2` mother, with both ALTs candidates in one gene, gave no row. Each ALT that is a candidate is now paired with its own origin, so that call is one pair in trans. A `0/1` call gives the same pairs as before. + - A child call with two different ALTs (`1/2`, as GLnexus writes a site with two ALTs) now counts as heterozygous for `compound_het.tsv`. Before, only a call with the reference allele did, so a `1/2` from a `0/1` father and a `0/2` mother, with both ALTs candidates in one gene, gave no row. Each ALT that is a candidate is now paired with its own origin. The two ALTs of one `1/2` call are on different copies, so they are one pair in trans even when the parents do not tell which ALT came from whom. A `0/1` call gives the same pairs as before. - A son's de novo candidate on chrX or chrY outside the pseudoautosomal regions needs only the parent who passes that chromosome on: the mother for his X, the father for his Y. Before, both parents needed a call, so a son's Y with the mother not called (the usual case) and his X with the father not called gave no row. The depth and GQ floors of `pass` now apply to the child and those parents only. - A de novo candidate is `low`, not `pass`, when a parent who passes the site on has 2 or more reads, or 5% or more of its reads, on the new allele (from its `AD`). Before, only the child's reads were read, so a parent called `0/0` with reads on the allele, as a mosaic parent has, still gave `pass`. diff --git a/docs/multi-sample.md b/docs/multi-sample.md index 20b09c210..67322f34d 100644 --- a/docs/multi-sample.md +++ b/docs/multi-sample.md @@ -82,7 +82,7 @@ It reads the `family.vcf.gz` and `pedigree.tsv` the earlier run left in `$GENOME Most candidates of a whole genome are genotype errors: confirm one that matters by another method. - When step 23 (the clinical filter) ran for a child, every pair of its candidate variants the child carries heterozygous in one gene is listed in `compound_het.tsv`. A call with two different ALTs (`1/2`, as GLnexus writes a site with two ALTs) is heterozygous too: each of its ALTs that is a candidate counts on its own, so a `1/2` with one ALT from each parent is a pair in trans. The pairs are `in trans` when one came from each parent (both copies of the gene are hit), `in cis` when both came from one parent, `unknown` when the genotypes do not tell. + When step 23 (the clinical filter) ran for a child, every pair of its candidate variants the child carries heterozygous in one gene is listed in `compound_het.tsv`. A call with two different ALTs (`1/2`, as GLnexus writes a site with two ALTs) is heterozygous too: each of its ALTs that is a candidate counts on its own. Its two ALTs sit on different copies, so they are a pair in trans even when the parents do not tell which came from whom. The pairs are `in trans` when one came from each parent (both copies of the gene are hit), `in cis` when both came from one parent, `unknown` when the genotypes do not tell. 5. **The chromosome map.** For each child with a sequenced parent, `og family report` maps the stretches of each chromosome the child got from that parent. The joint VCF must be sorted by position: an unsorted one stops the report with exit 2, naming the two records. It reads the joint VCF's genotypes at the sites where the parent is heterozygous and the genotypes give the allele passed on. When one of that parent's own parents is sequenced too, each such site also says which of the parent's two copies was passed on, the grandfather's or the grandmother's, and a switch from one to the other along a chromosome is a crossover. Without a sequenced grandparent the stretches say only "from this parent". With one parent sequenced, the child's other copy is written over the same stretches as the copy that did not come from the sequenced parent. diff --git a/rust/crates/og-steps/src/family/report.rs b/rust/crates/og-steps/src/family/report.rs index 3af701b1e..b80b04e4f 100644 --- a/rust/crates/og-steps/src/family/report.rs +++ b/rust/crates/og-steps/src/family/report.rs @@ -272,7 +272,9 @@ struct Child { mcol: usize, n: Counts, de_novo: Vec>, - cands: Vec<(Genes, String, Option<&'static str>)>, + /// compound heterozygote candidates: genes, variant, origin, and the + /// VCF line it came from (the two ALTs of one 1/2 call share it) + cands: Vec<(Genes, String, Option<&'static str>, u64)>, /// WhatsHap: per (chrom, PS) block, sites where the first haplotype /// holds the father's allele, and sites where it holds the mother's. blocks: HashMap<(Vec, Vec), [u64; 2]>, @@ -545,6 +547,7 @@ impl Pass<'_> { genes.clone(), format!("{}:{pos}:{}>{}", text(chrom), text(cols[3]), text(alt)), origin, + line, )); } } @@ -830,17 +833,22 @@ pub fn run(a: &Args) -> Result<()> { write_file(&out_dir.join("de_novo.tsv"), dn.as_bytes())?; let mut comp: Vec<[String; 7]> = Vec::new(); for ch in &children { - let mut by_gene: IndexMap<&str, Vec<(&str, Option<&str>)>> = IndexMap::new(); - for (genes, var, origin) in &ch.cands { + // variant, origin, VCF line + type Cand<'a> = (&'a str, Option<&'a str>, u64); + let mut by_gene: IndexMap<&str, Vec>> = IndexMap::new(); + for (genes, var, origin, site) in &ch.cands { for g in genes.iter() { - by_gene.entry(g).or_default().push((var, *origin)); + by_gene.entry(g).or_default().push((var, *origin, *site)); } } by_gene.sort_keys(); for (g, list) in &by_gene { - for (i, (v1, o1)) in list.iter().enumerate() { - for (v2, o2) in &list[i + 1..] { + for (i, (v1, o1, s1)) in list.iter().enumerate() { + for (v2, o2, s2) in &list[i + 1..] { let phase = match (o1, o2) { + // the two ALTs of one diploid 1/2 call sit on the + // two copies, whoever passed them on + _ if s1 == s2 => "in trans", (Some(a), Some(b)) if a == b => "in cis", (Some(_), Some(_)) => "in trans", _ => "unknown", diff --git a/rust/crates/og-steps/tests/rv_family_trio_calls.rs b/rust/crates/og-steps/tests/rv_family_trio_calls.rs index 67159f880..d507af91a 100644 --- a/rust/crates/og-steps/tests/rv_family_trio_calls.rs +++ b/rust/crates/og-steps/tests/rv_family_trio_calls.rs @@ -4,8 +4,9 @@ //! 1. Compound heterozygotes: a child call with two different ALTs (1/2, //! as GLnexus writes an unsplit multiallelic record) is heterozygous, and //! each ALT that is a candidate is paired with its origin, so a 1/2 from a -//! 0/1 father and a 0/2 mother is one pair in trans. A 0/1 child gives the -//! same pairs as before. +//! 0/1 father and a 0/2 mother is one pair in trans. The two ALTs of one +//! 1/2 call are in trans even when the parents do not show where each +//! came from. A 0/1 child gives the same pairs as before. //! 2. A son's sex chromosomes outside the PARs: his Y comes from his father //! and his X from his mother, so a de novo call there needs only that //! parent's call, and the depth and quality floors apply only to him and @@ -213,6 +214,49 @@ fn a_two_alt_call_from_both_parents_is_one_pair_in_trans() { ); } +#[test] +fn a_two_alt_call_is_in_trans_even_when_the_parents_do_not_tell() { + // both parents 1/2: the origin of each ALT is unknown, but the two ALTs + // of one diploid call are on different copies by definition + let d = scratch("two-alt-unknown-origin"); + let m = members(&d, "female"); + let vcf = joint_vcf( + &d, + &[( + "chr1", + 1_000_000, + "C", + "A,G", + [ + "1/2:0,15,15:30:99", + "1/2:0,15,15:30:99", + "1/2:0,15,15:30:99", + ], + )], + ); + let c = candidates( + &d, + &[ + ("chr1", 1_000_000, "C", "A", "G1"), + ("chr1", 1_000_000, "C", "G", "G1"), + ], + ); + let out = report(&d, &m, &vcf, Some(&c)); + assert_eq!( + pairs(&out), + vec![vec![ + "kid".to_owned(), + "G1".into(), + "chr1:1000000:C>A".into(), + "chr1:1000000:C>G".into(), + ".".into(), + ".".into(), + "in trans".into(), + ]], + "a 1/2 child of two 1/2 parents: one pair in trans, origins unknown" + ); +} + #[test] fn a_two_alt_call_pairs_with_a_ref_alt_call_in_the_gene() { let d = scratch("two-alt-plus-one"); From b48fdb0fafb5681a44c86bd7d12af5c9a01e1571 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:46:41 +0100 Subject: [PATCH 15/22] test(parity): golden family report outputs for the son's Y row and parental-read checks --- .../report-3gen-duo/out/family_report.md | 53 +++++++++++++++++++ .../golden/family/report-3gen-duo/stdout | 53 +++++++++++++++++++ .../family/report-3gen/out/family_report.md | 51 ++++++++++++++++++ tests/parity/golden/family/report-3gen/stdout | 51 ++++++++++++++++++ .../family/report-bench/out/family_report.md | 52 ++++++++++++++++++ .../parity/golden/family/report-bench/stdout | 52 ++++++++++++++++++ .../family/report-daughter-x/out/de_novo.tsv | 2 + .../report-daughter-x/out/family_report.md | 37 +++++++++++++ .../golden/family/report-daughter-x/stdout | 37 +++++++++++++ .../family/report-low/out/family_report.md | 37 +++++++++++++ tests/parity/golden/family/report-low/stdout | 37 +++++++++++++ .../family/report-phased/out/family_report.md | 39 ++++++++++++++ .../parity/golden/family/report-phased/stdout | 39 ++++++++++++++ .../report-trio-gz-no-par/out/de_novo.tsv | 6 +++ .../out/family_report.md | 38 +++++++++++++ .../family/report-trio-gz-no-par/stdout | 38 +++++++++++++ .../golden/family/report-trio/out/de_novo.tsv | 6 +++ .../family/report-trio/out/family_report.md | 38 +++++++++++++ tests/parity/golden/family/report-trio/stdout | 38 +++++++++++++ 19 files changed, 704 insertions(+) create mode 100644 tests/parity/golden/family/report-3gen-duo/out/family_report.md create mode 100644 tests/parity/golden/family/report-3gen-duo/stdout create mode 100644 tests/parity/golden/family/report-3gen/out/family_report.md create mode 100644 tests/parity/golden/family/report-3gen/stdout create mode 100644 tests/parity/golden/family/report-bench/out/family_report.md create mode 100644 tests/parity/golden/family/report-bench/stdout create mode 100644 tests/parity/golden/family/report-daughter-x/out/de_novo.tsv create mode 100644 tests/parity/golden/family/report-daughter-x/out/family_report.md create mode 100644 tests/parity/golden/family/report-daughter-x/stdout create mode 100644 tests/parity/golden/family/report-low/out/family_report.md create mode 100644 tests/parity/golden/family/report-low/stdout create mode 100644 tests/parity/golden/family/report-phased/out/family_report.md create mode 100644 tests/parity/golden/family/report-phased/stdout create mode 100644 tests/parity/golden/family/report-trio-gz-no-par/out/de_novo.tsv create mode 100644 tests/parity/golden/family/report-trio-gz-no-par/out/family_report.md create mode 100644 tests/parity/golden/family/report-trio-gz-no-par/stdout create mode 100644 tests/parity/golden/family/report-trio/out/de_novo.tsv create mode 100644 tests/parity/golden/family/report-trio/out/family_report.md create mode 100644 tests/parity/golden/family/report-trio/stdout diff --git a/tests/parity/golden/family/report-3gen-duo/out/family_report.md b/tests/parity/golden/family/report-3gen-duo/out/family_report.md new file mode 100644 index 000000000..a21df2733 --- /dev/null +++ b/tests/parity/golden/family/report-3gen-duo/out/family_report.md @@ -0,0 +1,53 @@ +# Family analysis + +Sequenced: dad, gma, gpa, kid. In the pedigree, not sequenced: mum. + +## Relationships + +Relatedness is twice the kinship coefficient: about the share of the genome the two have inherited from common ancestors (1 the same person, 0.5 parent and child or full siblings, 0.25 second degree, 0 unrelated). somalier measures it at its sites; the pedigree gives the expected value. + +| Pair | Declared | Expected | Measured | Opposite homozygotes | Sites | Verdict | +|---|---|---|---|---|---|---| + +## dad + +Father: gpa (sequenced). Mother: gma (sequenced). + +Sites of the joint VCF where dad has a genotype and a sequenced parent does too: 35000. `dad_inheritance.tsv.gz` lists them. + +- the allele from the father is known at 26293 (75.1%) +- the allele from the mother is known at 31293 (89.4%) +- both parents heterozygous like the child, so no origin from the genotypes: 3707 (10.6%) +- inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 0 (0.0%) + +De novo candidates (an allele neither parent has): 0, 0 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. + +## kid + +Father: dad (sequenced). Mother: mum (not sequenced). + +Sites of the joint VCF where kid has a genotype and a sequenced parent does too: 30000. `kid_inheritance.tsv.gz` lists them. + +- the allele from the father is known at 22455 (74.8%) +- the allele from the mother is known at 22455 (74.8%) +- both parents heterozygous like the child, so no origin from the genotypes: 7545 (25.1%) +- inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 0 (0.0%) + +The mother is not sequenced. At 22455 of these sites (74.8%) the genotypes give the allele the mother passed on to kid: that is the half of the mother's genome kid carries. The other half was not passed on and is not seen here. + +No de novo candidates are called: that needs both parents sequenced. + +## Compound heterozygotes + +No candidate variants were given (the clinical filter, step 23, did not run for a child), so no pair is reported. + +## Chromosome map + +For each child, the stretches of each chromosome that came from each parent, from the sites where that parent is heterozygous and the genotypes give the allele passed on. When a parent's own parent is sequenced too, each stretch also says which of the parent's two copies it is (the grandfather's or the grandmother's), and a change between them is a crossover. `_segments.tsv` lists them, `_dnapainter.csv` is the same list in DNA Painter's import format (GRCh38 positions), `_segments.svg` draws it. + +- dad: from the father (gpa): 3 stretches over 60.0 Mb; no grandparent on this side is sequenced, so the two copies of the father are not told apart +- dad: from the mother (gma): 4 stretches over 69.9 Mb; no grandparent on this side is sequenced, so the two copies of the mother are not told apart +- kid: from the father (dad): 4 stretches over 59.9 Mb, told apart as the copy from gma or gpa; 1 crossover, 0 discordant sites absorbed +- kid: from the mother (mum, not sequenced): 3 stretches over 59.9 Mb, the copy that did not come from the sequenced parent + +A stretch starts and ends at its first and last informative site: a crossover lies somewhere between two stretches, not at a base. Short reads give no long-range phase, so without a sequenced grandparent the map says only which parent a stretch came from. diff --git a/tests/parity/golden/family/report-3gen-duo/stdout b/tests/parity/golden/family/report-3gen-duo/stdout new file mode 100644 index 000000000..a21df2733 --- /dev/null +++ b/tests/parity/golden/family/report-3gen-duo/stdout @@ -0,0 +1,53 @@ +# Family analysis + +Sequenced: dad, gma, gpa, kid. In the pedigree, not sequenced: mum. + +## Relationships + +Relatedness is twice the kinship coefficient: about the share of the genome the two have inherited from common ancestors (1 the same person, 0.5 parent and child or full siblings, 0.25 second degree, 0 unrelated). somalier measures it at its sites; the pedigree gives the expected value. + +| Pair | Declared | Expected | Measured | Opposite homozygotes | Sites | Verdict | +|---|---|---|---|---|---|---| + +## dad + +Father: gpa (sequenced). Mother: gma (sequenced). + +Sites of the joint VCF where dad has a genotype and a sequenced parent does too: 35000. `dad_inheritance.tsv.gz` lists them. + +- the allele from the father is known at 26293 (75.1%) +- the allele from the mother is known at 31293 (89.4%) +- both parents heterozygous like the child, so no origin from the genotypes: 3707 (10.6%) +- inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 0 (0.0%) + +De novo candidates (an allele neither parent has): 0, 0 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. + +## kid + +Father: dad (sequenced). Mother: mum (not sequenced). + +Sites of the joint VCF where kid has a genotype and a sequenced parent does too: 30000. `kid_inheritance.tsv.gz` lists them. + +- the allele from the father is known at 22455 (74.8%) +- the allele from the mother is known at 22455 (74.8%) +- both parents heterozygous like the child, so no origin from the genotypes: 7545 (25.1%) +- inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 0 (0.0%) + +The mother is not sequenced. At 22455 of these sites (74.8%) the genotypes give the allele the mother passed on to kid: that is the half of the mother's genome kid carries. The other half was not passed on and is not seen here. + +No de novo candidates are called: that needs both parents sequenced. + +## Compound heterozygotes + +No candidate variants were given (the clinical filter, step 23, did not run for a child), so no pair is reported. + +## Chromosome map + +For each child, the stretches of each chromosome that came from each parent, from the sites where that parent is heterozygous and the genotypes give the allele passed on. When a parent's own parent is sequenced too, each stretch also says which of the parent's two copies it is (the grandfather's or the grandmother's), and a change between them is a crossover. `_segments.tsv` lists them, `_dnapainter.csv` is the same list in DNA Painter's import format (GRCh38 positions), `_segments.svg` draws it. + +- dad: from the father (gpa): 3 stretches over 60.0 Mb; no grandparent on this side is sequenced, so the two copies of the father are not told apart +- dad: from the mother (gma): 4 stretches over 69.9 Mb; no grandparent on this side is sequenced, so the two copies of the mother are not told apart +- kid: from the father (dad): 4 stretches over 59.9 Mb, told apart as the copy from gma or gpa; 1 crossover, 0 discordant sites absorbed +- kid: from the mother (mum, not sequenced): 3 stretches over 59.9 Mb, the copy that did not come from the sequenced parent + +A stretch starts and ends at its first and last informative site: a crossover lies somewhere between two stretches, not at a base. Short reads give no long-range phase, so without a sequenced grandparent the map says only which parent a stretch came from. diff --git a/tests/parity/golden/family/report-3gen/out/family_report.md b/tests/parity/golden/family/report-3gen/out/family_report.md new file mode 100644 index 000000000..642cc4774 --- /dev/null +++ b/tests/parity/golden/family/report-3gen/out/family_report.md @@ -0,0 +1,51 @@ +# Family analysis + +Sequenced: dad, gma, gpa, kid, mum. In the pedigree, not sequenced: none. + +## Relationships + +Relatedness is twice the kinship coefficient: about the share of the genome the two have inherited from common ancestors (1 the same person, 0.5 parent and child or full siblings, 0.25 second degree, 0 unrelated). somalier measures it at its sites; the pedigree gives the expected value. + +| Pair | Declared | Expected | Measured | Opposite homozygotes | Sites | Verdict | +|---|---|---|---|---|---|---| + +## dad + +Father: gpa (sequenced). Mother: gma (sequenced). + +Sites of the joint VCF where dad has a genotype and a sequenced parent does too: 35000. `dad_inheritance.tsv.gz` lists them. + +- the allele from the father is known at 26293 (75.1%) +- the allele from the mother is known at 31293 (89.4%) +- both parents heterozygous like the child, so no origin from the genotypes: 3707 (10.6%) +- inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 0 (0.0%) + +De novo candidates (an allele neither parent has): 0, 0 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. + +## kid + +Father: dad (sequenced). Mother: mum (sequenced). + +Sites of the joint VCF where kid has a genotype and a sequenced parent does too: 35000. `kid_inheritance.tsv.gz` lists them. + +- the allele from the father is known at 26192 (74.8%) +- the allele from the mother is known at 31192 (89.1%) +- both parents heterozygous like the child, so no origin from the genotypes: 3808 (10.9%) +- inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 0 (0.0%) + +De novo candidates (an allele neither parent has): 0, 0 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. + +## Compound heterozygotes + +No candidate variants were given (the clinical filter, step 23, did not run for a child), so no pair is reported. + +## Chromosome map + +For each child, the stretches of each chromosome that came from each parent, from the sites where that parent is heterozygous and the genotypes give the allele passed on. When a parent's own parent is sequenced too, each stretch also says which of the parent's two copies it is (the grandfather's or the grandmother's), and a change between them is a crossover. `_segments.tsv` lists them, `_dnapainter.csv` is the same list in DNA Painter's import format (GRCh38 positions), `_segments.svg` draws it. + +- dad: from the father (gpa): 3 stretches over 60.0 Mb; no grandparent on this side is sequenced, so the two copies of the father are not told apart +- dad: from the mother (gma): 4 stretches over 69.9 Mb; no grandparent on this side is sequenced, so the two copies of the mother are not told apart +- kid: from the father (dad): 4 stretches over 59.9 Mb, told apart as the copy from gma or gpa; 1 crossover, 3 discordant sites absorbed +- kid: from the mother (mum): 4 stretches over 70.0 Mb; no grandparent on this side is sequenced, so the two copies of the mother are not told apart + +A stretch starts and ends at its first and last informative site: a crossover lies somewhere between two stretches, not at a base. Short reads give no long-range phase, so without a sequenced grandparent the map says only which parent a stretch came from. diff --git a/tests/parity/golden/family/report-3gen/stdout b/tests/parity/golden/family/report-3gen/stdout new file mode 100644 index 000000000..642cc4774 --- /dev/null +++ b/tests/parity/golden/family/report-3gen/stdout @@ -0,0 +1,51 @@ +# Family analysis + +Sequenced: dad, gma, gpa, kid, mum. In the pedigree, not sequenced: none. + +## Relationships + +Relatedness is twice the kinship coefficient: about the share of the genome the two have inherited from common ancestors (1 the same person, 0.5 parent and child or full siblings, 0.25 second degree, 0 unrelated). somalier measures it at its sites; the pedigree gives the expected value. + +| Pair | Declared | Expected | Measured | Opposite homozygotes | Sites | Verdict | +|---|---|---|---|---|---|---| + +## dad + +Father: gpa (sequenced). Mother: gma (sequenced). + +Sites of the joint VCF where dad has a genotype and a sequenced parent does too: 35000. `dad_inheritance.tsv.gz` lists them. + +- the allele from the father is known at 26293 (75.1%) +- the allele from the mother is known at 31293 (89.4%) +- both parents heterozygous like the child, so no origin from the genotypes: 3707 (10.6%) +- inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 0 (0.0%) + +De novo candidates (an allele neither parent has): 0, 0 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. + +## kid + +Father: dad (sequenced). Mother: mum (sequenced). + +Sites of the joint VCF where kid has a genotype and a sequenced parent does too: 35000. `kid_inheritance.tsv.gz` lists them. + +- the allele from the father is known at 26192 (74.8%) +- the allele from the mother is known at 31192 (89.1%) +- both parents heterozygous like the child, so no origin from the genotypes: 3808 (10.9%) +- inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 0 (0.0%) + +De novo candidates (an allele neither parent has): 0, 0 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. + +## Compound heterozygotes + +No candidate variants were given (the clinical filter, step 23, did not run for a child), so no pair is reported. + +## Chromosome map + +For each child, the stretches of each chromosome that came from each parent, from the sites where that parent is heterozygous and the genotypes give the allele passed on. When a parent's own parent is sequenced too, each stretch also says which of the parent's two copies it is (the grandfather's or the grandmother's), and a change between them is a crossover. `_segments.tsv` lists them, `_dnapainter.csv` is the same list in DNA Painter's import format (GRCh38 positions), `_segments.svg` draws it. + +- dad: from the father (gpa): 3 stretches over 60.0 Mb; no grandparent on this side is sequenced, so the two copies of the father are not told apart +- dad: from the mother (gma): 4 stretches over 69.9 Mb; no grandparent on this side is sequenced, so the two copies of the mother are not told apart +- kid: from the father (dad): 4 stretches over 59.9 Mb, told apart as the copy from gma or gpa; 1 crossover, 3 discordant sites absorbed +- kid: from the mother (mum): 4 stretches over 70.0 Mb; no grandparent on this side is sequenced, so the two copies of the mother are not told apart + +A stretch starts and ends at its first and last informative site: a crossover lies somewhere between two stretches, not at a base. Short reads give no long-range phase, so without a sequenced grandparent the map says only which parent a stretch came from. diff --git a/tests/parity/golden/family/report-bench/out/family_report.md b/tests/parity/golden/family/report-bench/out/family_report.md new file mode 100644 index 000000000..0d102a174 --- /dev/null +++ b/tests/parity/golden/family/report-bench/out/family_report.md @@ -0,0 +1,52 @@ +# Family analysis + +Sequenced: DAD, GF, GM, KID, MOM. In the pedigree, not sequenced: none. + +## Relationships + +Relatedness is twice the kinship coefficient: about the share of the genome the two have inherited from common ancestors (1 the same person, 0.5 parent and child or full siblings, 0.25 second degree, 0 unrelated). somalier measures it at its sites; the pedigree gives the expected value. + +| Pair | Declared | Expected | Measured | Opposite homozygotes | Sites | Verdict | +|---|---|---|---|---|---|---| + +## DAD + +Father: GF (sequenced). Mother: GM (sequenced). + +Sites of the joint VCF where DAD has a genotype and a sequenced parent does too: 199936. `DAD_inheritance.tsv.gz` lists them. + +- the allele from the father is known at 176839 (88.4%) +- the allele from the mother is known at 176890 (88.5%) +- both parents heterozygous like the child, so no origin from the genotypes: 23046 (11.5%) +- inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 0 (0.0%) +- left out on chrX or chrY outside the pseudoautosomal regions (sex unknown, or a heterozygous call on a single X or Y): 49 + +De novo candidates (an allele neither parent has): 0, 0 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. + +## KID + +Father: DAD (sequenced). Mother: MOM (sequenced). + +Sites of the joint VCF where KID has a genotype and a sequenced parent does too: 199985. `KID_inheritance.tsv.gz` lists them. + +- the allele from the father is known at 176935 (88.5%) +- the allele from the mother is known at 176935 (88.5%) +- both parents heterozygous like the child, so no origin from the genotypes: 23050 (11.5%) +- inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 0 (0.0%) + +De novo candidates (an allele neither parent has): 0, 0 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. + +## Compound heterozygotes + +No candidate variants were given (the clinical filter, step 23, did not run for a child), so no pair is reported. + +## Chromosome map + +For each child, the stretches of each chromosome that came from each parent, from the sites where that parent is heterozygous and the genotypes give the allele passed on. When a parent's own parent is sequenced too, each stretch also says which of the parent's two copies it is (the grandfather's or the grandmother's), and a change between them is a crossover. `_segments.tsv` lists them, `_dnapainter.csv` is the same list in DNA Painter's import format (GRCh38 positions), `_segments.svg` draws it. + +- DAD: from the father (GF): 22 stretches over 19.1 Mb; no grandparent on this side is sequenced, so the two copies of the father are not told apart +- DAD: from the mother (GM): 23 stretches over 19.1 Mb; no grandparent on this side is sequenced, so the two copies of the mother are not told apart +- KID: from the father (DAD): 22 stretches over 19.1 Mb, told apart as the copy from GF or GM; 0 crossovers, 25582 discordant sites absorbed +- KID: from the mother (MOM): 22 stretches over 19.1 Mb; no grandparent on this side is sequenced, so the two copies of the mother are not told apart + +A stretch starts and ends at its first and last informative site: a crossover lies somewhere between two stretches, not at a base. Short reads give no long-range phase, so without a sequenced grandparent the map says only which parent a stretch came from. diff --git a/tests/parity/golden/family/report-bench/stdout b/tests/parity/golden/family/report-bench/stdout new file mode 100644 index 000000000..0d102a174 --- /dev/null +++ b/tests/parity/golden/family/report-bench/stdout @@ -0,0 +1,52 @@ +# Family analysis + +Sequenced: DAD, GF, GM, KID, MOM. In the pedigree, not sequenced: none. + +## Relationships + +Relatedness is twice the kinship coefficient: about the share of the genome the two have inherited from common ancestors (1 the same person, 0.5 parent and child or full siblings, 0.25 second degree, 0 unrelated). somalier measures it at its sites; the pedigree gives the expected value. + +| Pair | Declared | Expected | Measured | Opposite homozygotes | Sites | Verdict | +|---|---|---|---|---|---|---| + +## DAD + +Father: GF (sequenced). Mother: GM (sequenced). + +Sites of the joint VCF where DAD has a genotype and a sequenced parent does too: 199936. `DAD_inheritance.tsv.gz` lists them. + +- the allele from the father is known at 176839 (88.4%) +- the allele from the mother is known at 176890 (88.5%) +- both parents heterozygous like the child, so no origin from the genotypes: 23046 (11.5%) +- inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 0 (0.0%) +- left out on chrX or chrY outside the pseudoautosomal regions (sex unknown, or a heterozygous call on a single X or Y): 49 + +De novo candidates (an allele neither parent has): 0, 0 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. + +## KID + +Father: DAD (sequenced). Mother: MOM (sequenced). + +Sites of the joint VCF where KID has a genotype and a sequenced parent does too: 199985. `KID_inheritance.tsv.gz` lists them. + +- the allele from the father is known at 176935 (88.5%) +- the allele from the mother is known at 176935 (88.5%) +- both parents heterozygous like the child, so no origin from the genotypes: 23050 (11.5%) +- inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 0 (0.0%) + +De novo candidates (an allele neither parent has): 0, 0 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. + +## Compound heterozygotes + +No candidate variants were given (the clinical filter, step 23, did not run for a child), so no pair is reported. + +## Chromosome map + +For each child, the stretches of each chromosome that came from each parent, from the sites where that parent is heterozygous and the genotypes give the allele passed on. When a parent's own parent is sequenced too, each stretch also says which of the parent's two copies it is (the grandfather's or the grandmother's), and a change between them is a crossover. `_segments.tsv` lists them, `_dnapainter.csv` is the same list in DNA Painter's import format (GRCh38 positions), `_segments.svg` draws it. + +- DAD: from the father (GF): 22 stretches over 19.1 Mb; no grandparent on this side is sequenced, so the two copies of the father are not told apart +- DAD: from the mother (GM): 23 stretches over 19.1 Mb; no grandparent on this side is sequenced, so the two copies of the mother are not told apart +- KID: from the father (DAD): 22 stretches over 19.1 Mb, told apart as the copy from GF or GM; 0 crossovers, 25582 discordant sites absorbed +- KID: from the mother (MOM): 22 stretches over 19.1 Mb; no grandparent on this side is sequenced, so the two copies of the mother are not told apart + +A stretch starts and ends at its first and last informative site: a crossover lies somewhere between two stretches, not at a base. Short reads give no long-range phase, so without a sequenced grandparent the map says only which parent a stretch came from. diff --git a/tests/parity/golden/family/report-daughter-x/out/de_novo.tsv b/tests/parity/golden/family/report-daughter-x/out/de_novo.tsv new file mode 100644 index 000000000..ca2d84556 --- /dev/null +++ b/tests/parity/golden/family/report-daughter-x/out/de_novo.tsv @@ -0,0 +1,2 @@ +child chrom pos ref alt child_gt father_gt mother_gt child_dp child_gq child_ab father_dp father_gq mother_dp mother_gq quality +kid chrX 5000100 A G 0/1 0/0 0/0 30 50 0.53 30 50 30 50 low diff --git a/tests/parity/golden/family/report-daughter-x/out/family_report.md b/tests/parity/golden/family/report-daughter-x/out/family_report.md new file mode 100644 index 000000000..f47fec79e --- /dev/null +++ b/tests/parity/golden/family/report-daughter-x/out/family_report.md @@ -0,0 +1,37 @@ +# Family analysis + +Sequenced: dad, kid, mum. In the pedigree, not sequenced: grandpa. + +## Relationships + +Relatedness is twice the kinship coefficient: about the share of the genome the two have inherited from common ancestors (1 the same person, 0.5 parent and child or full siblings, 0.25 second degree, 0 unrelated). somalier measures it at its sites; the pedigree gives the expected value. + +| Pair | Declared | Expected | Measured | Opposite homozygotes | Sites | Verdict | +|---|---|---|---|---|---|---| +| dad and kid | parent-child | 0.500 | 0.490 | 0.0% | 5000 | ok: first degree, as declared | +| dad and mum | partners | 0.000 | 0.010 | 12.0% | 5000 | ok: unrelated, as declared | +| kid and mum | parent-child | 0.500 | 0.510 | 0.0% | 5000 | ok: first degree, as declared | + +## kid + +Father: dad (sequenced). Mother: mum (sequenced). + +Sites of the joint VCF where kid has a genotype and a sequenced parent does too: 3. `kid_inheritance.tsv.gz` lists them. + +- the allele from the father is known at 1 (33.3%) +- the allele from the mother is known at 1 (33.3%) +- both parents heterozygous like the child, so no origin from the genotypes: 0 (0.0%) +- inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 2 (66.7%) + +De novo candidates (an allele neither parent has): 1, 0 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. + +## Compound heterozygotes + +No candidate variants were given (the clinical filter, step 23, did not run for a child), so no pair is reported. + +## Chromosome map + +For each child, the stretches of each chromosome that came from each parent, from the sites where that parent is heterozygous and the genotypes give the allele passed on. When a parent's own parent is sequenced too, each stretch also says which of the parent's two copies it is (the grandfather's or the grandmother's), and a change between them is a crossover. `_segments.tsv` lists them, `_dnapainter.csv` is the same list in DNA Painter's import format (GRCh38 positions), `_segments.svg` draws it. + + +A stretch starts and ends at its first and last informative site: a crossover lies somewhere between two stretches, not at a base. Short reads give no long-range phase, so without a sequenced grandparent the map says only which parent a stretch came from. diff --git a/tests/parity/golden/family/report-daughter-x/stdout b/tests/parity/golden/family/report-daughter-x/stdout new file mode 100644 index 000000000..f47fec79e --- /dev/null +++ b/tests/parity/golden/family/report-daughter-x/stdout @@ -0,0 +1,37 @@ +# Family analysis + +Sequenced: dad, kid, mum. In the pedigree, not sequenced: grandpa. + +## Relationships + +Relatedness is twice the kinship coefficient: about the share of the genome the two have inherited from common ancestors (1 the same person, 0.5 parent and child or full siblings, 0.25 second degree, 0 unrelated). somalier measures it at its sites; the pedigree gives the expected value. + +| Pair | Declared | Expected | Measured | Opposite homozygotes | Sites | Verdict | +|---|---|---|---|---|---|---| +| dad and kid | parent-child | 0.500 | 0.490 | 0.0% | 5000 | ok: first degree, as declared | +| dad and mum | partners | 0.000 | 0.010 | 12.0% | 5000 | ok: unrelated, as declared | +| kid and mum | parent-child | 0.500 | 0.510 | 0.0% | 5000 | ok: first degree, as declared | + +## kid + +Father: dad (sequenced). Mother: mum (sequenced). + +Sites of the joint VCF where kid has a genotype and a sequenced parent does too: 3. `kid_inheritance.tsv.gz` lists them. + +- the allele from the father is known at 1 (33.3%) +- the allele from the mother is known at 1 (33.3%) +- both parents heterozygous like the child, so no origin from the genotypes: 0 (0.0%) +- inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 2 (66.7%) + +De novo candidates (an allele neither parent has): 1, 0 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. + +## Compound heterozygotes + +No candidate variants were given (the clinical filter, step 23, did not run for a child), so no pair is reported. + +## Chromosome map + +For each child, the stretches of each chromosome that came from each parent, from the sites where that parent is heterozygous and the genotypes give the allele passed on. When a parent's own parent is sequenced too, each stretch also says which of the parent's two copies it is (the grandfather's or the grandmother's), and a change between them is a crossover. `_segments.tsv` lists them, `_dnapainter.csv` is the same list in DNA Painter's import format (GRCh38 positions), `_segments.svg` draws it. + + +A stretch starts and ends at its first and last informative site: a crossover lies somewhere between two stretches, not at a base. Short reads give no long-range phase, so without a sequenced grandparent the map says only which parent a stretch came from. diff --git a/tests/parity/golden/family/report-low/out/family_report.md b/tests/parity/golden/family/report-low/out/family_report.md new file mode 100644 index 000000000..84c819643 --- /dev/null +++ b/tests/parity/golden/family/report-low/out/family_report.md @@ -0,0 +1,37 @@ +# Family analysis + +Sequenced: dad, kid, mum. In the pedigree, not sequenced: grandpa. + +## Relationships + +Relatedness is twice the kinship coefficient: about the share of the genome the two have inherited from common ancestors (1 the same person, 0.5 parent and child or full siblings, 0.25 second degree, 0 unrelated). somalier measures it at its sites; the pedigree gives the expected value. + +| Pair | Declared | Expected | Measured | Opposite homozygotes | Sites | Verdict | +|---|---|---|---|---|---|---| +| dad and kid | parent-child | 0.500 | 0.490 | 0.0% | 5000 | ok: first degree, as declared | +| dad and mum | partners | 0.000 | 0.010 | 12.0% | 5000 | ok: unrelated, as declared | +| kid and mum | parent-child | 0.500 | 0.510 | 0.0% | 5000 | ok: first degree, as declared | + +## kid + +Father: dad (sequenced). Mother: mum (sequenced). + +Sites of the joint VCF where kid has a genotype and a sequenced parent does too: 1. `kid_inheritance.tsv.gz` lists them. + +- the allele from the father is known at 0 (0.0%) +- the allele from the mother is known at 0 (0.0%) +- both parents heterozygous like the child, so no origin from the genotypes: 0 (0.0%) +- inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 1 (100.0%) + +De novo candidates (an allele neither parent has): 1, 0 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. + +## Compound heterozygotes + +No candidate variants were given (the clinical filter, step 23, did not run for a child), so no pair is reported. + +## Chromosome map + +For each child, the stretches of each chromosome that came from each parent, from the sites where that parent is heterozygous and the genotypes give the allele passed on. When a parent's own parent is sequenced too, each stretch also says which of the parent's two copies it is (the grandfather's or the grandmother's), and a change between them is a crossover. `_segments.tsv` lists them, `_dnapainter.csv` is the same list in DNA Painter's import format (GRCh38 positions), `_segments.svg` draws it. + + +A stretch starts and ends at its first and last informative site: a crossover lies somewhere between two stretches, not at a base. Short reads give no long-range phase, so without a sequenced grandparent the map says only which parent a stretch came from. diff --git a/tests/parity/golden/family/report-low/stdout b/tests/parity/golden/family/report-low/stdout new file mode 100644 index 000000000..84c819643 --- /dev/null +++ b/tests/parity/golden/family/report-low/stdout @@ -0,0 +1,37 @@ +# Family analysis + +Sequenced: dad, kid, mum. In the pedigree, not sequenced: grandpa. + +## Relationships + +Relatedness is twice the kinship coefficient: about the share of the genome the two have inherited from common ancestors (1 the same person, 0.5 parent and child or full siblings, 0.25 second degree, 0 unrelated). somalier measures it at its sites; the pedigree gives the expected value. + +| Pair | Declared | Expected | Measured | Opposite homozygotes | Sites | Verdict | +|---|---|---|---|---|---|---| +| dad and kid | parent-child | 0.500 | 0.490 | 0.0% | 5000 | ok: first degree, as declared | +| dad and mum | partners | 0.000 | 0.010 | 12.0% | 5000 | ok: unrelated, as declared | +| kid and mum | parent-child | 0.500 | 0.510 | 0.0% | 5000 | ok: first degree, as declared | + +## kid + +Father: dad (sequenced). Mother: mum (sequenced). + +Sites of the joint VCF where kid has a genotype and a sequenced parent does too: 1. `kid_inheritance.tsv.gz` lists them. + +- the allele from the father is known at 0 (0.0%) +- the allele from the mother is known at 0 (0.0%) +- both parents heterozygous like the child, so no origin from the genotypes: 0 (0.0%) +- inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 1 (100.0%) + +De novo candidates (an allele neither parent has): 1, 0 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. + +## Compound heterozygotes + +No candidate variants were given (the clinical filter, step 23, did not run for a child), so no pair is reported. + +## Chromosome map + +For each child, the stretches of each chromosome that came from each parent, from the sites where that parent is heterozygous and the genotypes give the allele passed on. When a parent's own parent is sequenced too, each stretch also says which of the parent's two copies it is (the grandfather's or the grandmother's), and a change between them is a crossover. `_segments.tsv` lists them, `_dnapainter.csv` is the same list in DNA Painter's import format (GRCh38 positions), `_segments.svg` draws it. + + +A stretch starts and ends at its first and last informative site: a crossover lies somewhere between two stretches, not at a base. Short reads give no long-range phase, so without a sequenced grandparent the map says only which parent a stretch came from. diff --git a/tests/parity/golden/family/report-phased/out/family_report.md b/tests/parity/golden/family/report-phased/out/family_report.md new file mode 100644 index 000000000..efecfce1f --- /dev/null +++ b/tests/parity/golden/family/report-phased/out/family_report.md @@ -0,0 +1,39 @@ +# Family analysis + +Sequenced: dad, kid, mum. In the pedigree, not sequenced: grandpa. + +## Relationships + +Relatedness is twice the kinship coefficient: about the share of the genome the two have inherited from common ancestors (1 the same person, 0.5 parent and child or full siblings, 0.25 second degree, 0 unrelated). somalier measures it at its sites; the pedigree gives the expected value. + +| Pair | Declared | Expected | Measured | Opposite homozygotes | Sites | Verdict | +|---|---|---|---|---|---|---| +| dad and kid | parent-child | 0.500 | 0.490 | 0.0% | 5000 | ok: first degree, as declared | +| dad and mum | partners | 0.000 | 0.010 | 12.0% | 5000 | ok: unrelated, as declared | +| kid and mum | parent-child | 0.500 | 0.510 | 0.0% | 5000 | ok: first degree, as declared | + +## kid + +Father: dad (sequenced). Mother: mum (sequenced). + +Sites of the joint VCF where kid has a genotype and a sequenced parent does too: 3. `kid_inheritance.tsv.gz` lists them. + +- the allele from the father is known at 3 (100.0%) +- the allele from the mother is known at 3 (100.0%) +- both parents heterozygous like the child, so no origin from the genotypes: 0 (0.0%) +- inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 0 (0.0%) + +De novo candidates (an allele neither parent has): 0, 0 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. + +WhatsHap phased 3 heterozygous sites of kid by inheritance. At 3 of them the genotypes also give the origin, and the phase agrees with it at 2 (66.7%). + +## Compound heterozygotes + +No candidate variants were given (the clinical filter, step 23, did not run for a child), so no pair is reported. + +## Chromosome map + +For each child, the stretches of each chromosome that came from each parent, from the sites where that parent is heterozygous and the genotypes give the allele passed on. When a parent's own parent is sequenced too, each stretch also says which of the parent's two copies it is (the grandfather's or the grandmother's), and a change between them is a crossover. `_segments.tsv` lists them, `_dnapainter.csv` is the same list in DNA Painter's import format (GRCh38 positions), `_segments.svg` draws it. + + +A stretch starts and ends at its first and last informative site: a crossover lies somewhere between two stretches, not at a base. Short reads give no long-range phase, so without a sequenced grandparent the map says only which parent a stretch came from. diff --git a/tests/parity/golden/family/report-phased/stdout b/tests/parity/golden/family/report-phased/stdout new file mode 100644 index 000000000..efecfce1f --- /dev/null +++ b/tests/parity/golden/family/report-phased/stdout @@ -0,0 +1,39 @@ +# Family analysis + +Sequenced: dad, kid, mum. In the pedigree, not sequenced: grandpa. + +## Relationships + +Relatedness is twice the kinship coefficient: about the share of the genome the two have inherited from common ancestors (1 the same person, 0.5 parent and child or full siblings, 0.25 second degree, 0 unrelated). somalier measures it at its sites; the pedigree gives the expected value. + +| Pair | Declared | Expected | Measured | Opposite homozygotes | Sites | Verdict | +|---|---|---|---|---|---|---| +| dad and kid | parent-child | 0.500 | 0.490 | 0.0% | 5000 | ok: first degree, as declared | +| dad and mum | partners | 0.000 | 0.010 | 12.0% | 5000 | ok: unrelated, as declared | +| kid and mum | parent-child | 0.500 | 0.510 | 0.0% | 5000 | ok: first degree, as declared | + +## kid + +Father: dad (sequenced). Mother: mum (sequenced). + +Sites of the joint VCF where kid has a genotype and a sequenced parent does too: 3. `kid_inheritance.tsv.gz` lists them. + +- the allele from the father is known at 3 (100.0%) +- the allele from the mother is known at 3 (100.0%) +- both parents heterozygous like the child, so no origin from the genotypes: 0 (0.0%) +- inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 0 (0.0%) + +De novo candidates (an allele neither parent has): 0, 0 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. + +WhatsHap phased 3 heterozygous sites of kid by inheritance. At 3 of them the genotypes also give the origin, and the phase agrees with it at 2 (66.7%). + +## Compound heterozygotes + +No candidate variants were given (the clinical filter, step 23, did not run for a child), so no pair is reported. + +## Chromosome map + +For each child, the stretches of each chromosome that came from each parent, from the sites where that parent is heterozygous and the genotypes give the allele passed on. When a parent's own parent is sequenced too, each stretch also says which of the parent's two copies it is (the grandfather's or the grandmother's), and a change between them is a crossover. `_segments.tsv` lists them, `_dnapainter.csv` is the same list in DNA Painter's import format (GRCh38 positions), `_segments.svg` draws it. + + +A stretch starts and ends at its first and last informative site: a crossover lies somewhere between two stretches, not at a base. Short reads give no long-range phase, so without a sequenced grandparent the map says only which parent a stretch came from. diff --git a/tests/parity/golden/family/report-trio-gz-no-par/out/de_novo.tsv b/tests/parity/golden/family/report-trio-gz-no-par/out/de_novo.tsv new file mode 100644 index 000000000..f8b4ef818 --- /dev/null +++ b/tests/parity/golden/family/report-trio-gz-no-par/out/de_novo.tsv @@ -0,0 +1,6 @@ +child chrom pos ref alt child_gt father_gt mother_gt child_dp child_gq child_ab father_dp father_gq mother_dp mother_gq quality +kid chr1 300 G A 0/1 0/0 0/0 30 50 0.53 30 50 30 50 pass +kid chr1 600 A T 1/2 0/1 0/1 30 50 0.33 30 50 30 50 pass +kid chr3 300 C T 0/1 0/0 0/0 3 7 . 30 50 30 50 low +kid chrX 74000300 G A 1 0 0/0 30 50 0.50 30 50 30 50 low +kid chrY 5000100 C T 1 0 ./. 30 50 0.50 30 50 30 50 low diff --git a/tests/parity/golden/family/report-trio-gz-no-par/out/family_report.md b/tests/parity/golden/family/report-trio-gz-no-par/out/family_report.md new file mode 100644 index 000000000..b373e29c5 --- /dev/null +++ b/tests/parity/golden/family/report-trio-gz-no-par/out/family_report.md @@ -0,0 +1,38 @@ +# Family analysis + +Sequenced: dad, kid, mum. In the pedigree, not sequenced: grandpa. + +## Relationships + +Relatedness is twice the kinship coefficient: about the share of the genome the two have inherited from common ancestors (1 the same person, 0.5 parent and child or full siblings, 0.25 second degree, 0 unrelated). somalier measures it at its sites; the pedigree gives the expected value. + +| Pair | Declared | Expected | Measured | Opposite homozygotes | Sites | Verdict | +|---|---|---|---|---|---|---| +| dad and kid | parent-child | 0.500 | 0.490 | 0.0% | 5000 | ok: first degree, as declared | +| dad and mum | partners | 0.000 | 0.010 | 12.0% | 5000 | ok: unrelated, as declared | +| kid and mum | parent-child | 0.500 | 0.510 | 0.0% | 5000 | ok: first degree, as declared | + +## kid + +Father: dad (sequenced). Mother: mum (sequenced). + +Sites of the joint VCF where kid has a genotype and a sequenced parent does too: 17. `kid_inheritance.tsv.gz` lists them. + +- the allele from the father is known at 7 (41.2%) +- the allele from the mother is known at 8 (47.1%) +- both parents heterozygous like the child, so no origin from the genotypes: 2 (11.8%) +- inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 6 (35.3%) +- left out on chrX or chrY outside the pseudoautosomal regions (sex unknown, or a heterozygous call on a single X or Y): 2 + +De novo candidates (an allele neither parent has): 5, 2 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. + +## Compound heterozygotes + +No candidate variants were given (the clinical filter, step 23, did not run for a child), so no pair is reported. + +## Chromosome map + +For each child, the stretches of each chromosome that came from each parent, from the sites where that parent is heterozygous and the genotypes give the allele passed on. When a parent's own parent is sequenced too, each stretch also says which of the parent's two copies it is (the grandfather's or the grandmother's), and a change between them is a crossover. `_segments.tsv` lists them, `_dnapainter.csv` is the same list in DNA Painter's import format (GRCh38 positions), `_segments.svg` draws it. + + +A stretch starts and ends at its first and last informative site: a crossover lies somewhere between two stretches, not at a base. Short reads give no long-range phase, so without a sequenced grandparent the map says only which parent a stretch came from. diff --git a/tests/parity/golden/family/report-trio-gz-no-par/stdout b/tests/parity/golden/family/report-trio-gz-no-par/stdout new file mode 100644 index 000000000..b373e29c5 --- /dev/null +++ b/tests/parity/golden/family/report-trio-gz-no-par/stdout @@ -0,0 +1,38 @@ +# Family analysis + +Sequenced: dad, kid, mum. In the pedigree, not sequenced: grandpa. + +## Relationships + +Relatedness is twice the kinship coefficient: about the share of the genome the two have inherited from common ancestors (1 the same person, 0.5 parent and child or full siblings, 0.25 second degree, 0 unrelated). somalier measures it at its sites; the pedigree gives the expected value. + +| Pair | Declared | Expected | Measured | Opposite homozygotes | Sites | Verdict | +|---|---|---|---|---|---|---| +| dad and kid | parent-child | 0.500 | 0.490 | 0.0% | 5000 | ok: first degree, as declared | +| dad and mum | partners | 0.000 | 0.010 | 12.0% | 5000 | ok: unrelated, as declared | +| kid and mum | parent-child | 0.500 | 0.510 | 0.0% | 5000 | ok: first degree, as declared | + +## kid + +Father: dad (sequenced). Mother: mum (sequenced). + +Sites of the joint VCF where kid has a genotype and a sequenced parent does too: 17. `kid_inheritance.tsv.gz` lists them. + +- the allele from the father is known at 7 (41.2%) +- the allele from the mother is known at 8 (47.1%) +- both parents heterozygous like the child, so no origin from the genotypes: 2 (11.8%) +- inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 6 (35.3%) +- left out on chrX or chrY outside the pseudoautosomal regions (sex unknown, or a heterozygous call on a single X or Y): 2 + +De novo candidates (an allele neither parent has): 5, 2 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. + +## Compound heterozygotes + +No candidate variants were given (the clinical filter, step 23, did not run for a child), so no pair is reported. + +## Chromosome map + +For each child, the stretches of each chromosome that came from each parent, from the sites where that parent is heterozygous and the genotypes give the allele passed on. When a parent's own parent is sequenced too, each stretch also says which of the parent's two copies it is (the grandfather's or the grandmother's), and a change between them is a crossover. `_segments.tsv` lists them, `_dnapainter.csv` is the same list in DNA Painter's import format (GRCh38 positions), `_segments.svg` draws it. + + +A stretch starts and ends at its first and last informative site: a crossover lies somewhere between two stretches, not at a base. Short reads give no long-range phase, so without a sequenced grandparent the map says only which parent a stretch came from. diff --git a/tests/parity/golden/family/report-trio/out/de_novo.tsv b/tests/parity/golden/family/report-trio/out/de_novo.tsv new file mode 100644 index 000000000..f8b4ef818 --- /dev/null +++ b/tests/parity/golden/family/report-trio/out/de_novo.tsv @@ -0,0 +1,6 @@ +child chrom pos ref alt child_gt father_gt mother_gt child_dp child_gq child_ab father_dp father_gq mother_dp mother_gq quality +kid chr1 300 G A 0/1 0/0 0/0 30 50 0.53 30 50 30 50 pass +kid chr1 600 A T 1/2 0/1 0/1 30 50 0.33 30 50 30 50 pass +kid chr3 300 C T 0/1 0/0 0/0 3 7 . 30 50 30 50 low +kid chrX 74000300 G A 1 0 0/0 30 50 0.50 30 50 30 50 low +kid chrY 5000100 C T 1 0 ./. 30 50 0.50 30 50 30 50 low diff --git a/tests/parity/golden/family/report-trio/out/family_report.md b/tests/parity/golden/family/report-trio/out/family_report.md new file mode 100644 index 000000000..ccaf9f735 --- /dev/null +++ b/tests/parity/golden/family/report-trio/out/family_report.md @@ -0,0 +1,38 @@ +# Family analysis + +Sequenced: dad, kid, mum. In the pedigree, not sequenced: grandpa. + +## Relationships + +Relatedness is twice the kinship coefficient: about the share of the genome the two have inherited from common ancestors (1 the same person, 0.5 parent and child or full siblings, 0.25 second degree, 0 unrelated). somalier measures it at its sites; the pedigree gives the expected value. + +| Pair | Declared | Expected | Measured | Opposite homozygotes | Sites | Verdict | +|---|---|---|---|---|---|---| +| dad and kid | parent-child | 0.500 | 0.490 | 0.0% | 5000 | ok: first degree, as declared | +| dad and mum | partners | 0.000 | 0.010 | 12.0% | 5000 | ok: unrelated, as declared | +| kid and mum | parent-child | 0.500 | 0.510 | 0.0% | 5000 | ok: first degree, as declared | + +## kid + +Father: dad (sequenced). Mother: mum (sequenced). + +Sites of the joint VCF where kid has a genotype and a sequenced parent does too: 18. `kid_inheritance.tsv.gz` lists them. + +- the allele from the father is known at 8 (44.4%) +- the allele from the mother is known at 9 (50.0%) +- both parents heterozygous like the child, so no origin from the genotypes: 2 (11.1%) +- inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 6 (33.3%) +- left out on chrX or chrY outside the pseudoautosomal regions (sex unknown, or a heterozygous call on a single X or Y): 1 + +De novo candidates (an allele neither parent has): 5, 2 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. + +## Compound heterozygotes + +Pairs of the clinical filter's candidate variants a child carries heterozygous in one gene: 3; in trans (one from each parent, both copies of the gene hit) 1, in cis (both from one parent) 2, origin unknown 0 (`compound_het.tsv`). + +## Chromosome map + +For each child, the stretches of each chromosome that came from each parent, from the sites where that parent is heterozygous and the genotypes give the allele passed on. When a parent's own parent is sequenced too, each stretch also says which of the parent's two copies it is (the grandfather's or the grandmother's), and a change between them is a crossover. `_segments.tsv` lists them, `_dnapainter.csv` is the same list in DNA Painter's import format (GRCh38 positions), `_segments.svg` draws it. + + +A stretch starts and ends at its first and last informative site: a crossover lies somewhere between two stretches, not at a base. Short reads give no long-range phase, so without a sequenced grandparent the map says only which parent a stretch came from. diff --git a/tests/parity/golden/family/report-trio/stdout b/tests/parity/golden/family/report-trio/stdout new file mode 100644 index 000000000..ccaf9f735 --- /dev/null +++ b/tests/parity/golden/family/report-trio/stdout @@ -0,0 +1,38 @@ +# Family analysis + +Sequenced: dad, kid, mum. In the pedigree, not sequenced: grandpa. + +## Relationships + +Relatedness is twice the kinship coefficient: about the share of the genome the two have inherited from common ancestors (1 the same person, 0.5 parent and child or full siblings, 0.25 second degree, 0 unrelated). somalier measures it at its sites; the pedigree gives the expected value. + +| Pair | Declared | Expected | Measured | Opposite homozygotes | Sites | Verdict | +|---|---|---|---|---|---|---| +| dad and kid | parent-child | 0.500 | 0.490 | 0.0% | 5000 | ok: first degree, as declared | +| dad and mum | partners | 0.000 | 0.010 | 12.0% | 5000 | ok: unrelated, as declared | +| kid and mum | parent-child | 0.500 | 0.510 | 0.0% | 5000 | ok: first degree, as declared | + +## kid + +Father: dad (sequenced). Mother: mum (sequenced). + +Sites of the joint VCF where kid has a genotype and a sequenced parent does too: 18. `kid_inheritance.tsv.gz` lists them. + +- the allele from the father is known at 8 (44.4%) +- the allele from the mother is known at 9 (50.0%) +- both parents heterozygous like the child, so no origin from the genotypes: 2 (11.1%) +- inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 6 (33.3%) +- left out on chrX or chrY outside the pseudoautosomal regions (sex unknown, or a heterozygous call on a single X or Y): 1 + +De novo candidates (an allele neither parent has): 5, 2 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. + +## Compound heterozygotes + +Pairs of the clinical filter's candidate variants a child carries heterozygous in one gene: 3; in trans (one from each parent, both copies of the gene hit) 1, in cis (both from one parent) 2, origin unknown 0 (`compound_het.tsv`). + +## Chromosome map + +For each child, the stretches of each chromosome that came from each parent, from the sites where that parent is heterozygous and the genotypes give the allele passed on. When a parent's own parent is sequenced too, each stretch also says which of the parent's two copies it is (the grandfather's or the grandmother's), and a change between them is a crossover. `_segments.tsv` lists them, `_dnapainter.csv` is the same list in DNA Painter's import format (GRCh38 positions), `_segments.svg` draws it. + + +A stretch starts and ends at its first and last informative site: a crossover lies somewhere between two stretches, not at a base. Short reads give no long-range phase, so without a sequenced grandparent the map says only which parent a stretch came from. From 93887270900b2f257355af8e440f615e36cacc0c Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 21:11:22 +0100 Subject: [PATCH 16/22] fix(clinvar): take step 06's normalised ClinVar lock with flock when installed On Ubuntu 26.04 mkdir is uutils coreutils 0.10.0, which exits 0 when another process creates the same folder at the same moment (330 of 2000 concurrent pairs both succeeded, none on Ubuntu 24.04), so the lock_acquire folder lock let two screens build the shared copy at once: the fake-docker case failed that way on the ubuntu-26.04 runner. The step now takes clinvar/.norm.lock with flock, which the kernel enforces and drops when the run ends, and keeps lock_acquire (clinvar/.norm.lock.d) only where flock is not installed. The case runs one screen without flock too, and the read-only check without flock, where the wait could hang. Bead: PL01 --- CHANGELOG.md | 2 +- docs/06-clinvar-screen.md | 2 +- scripts/06-clinvar-screen.sh | 30 +++++++++---- .../fake-docker/rv-step06-screen-integrity.sh | 44 +++++++++++++++---- 4 files changed, 58 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a55719ba7..48d4ac896 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - Step 06 (`scripts/06-clinvar-screen.sh`) no longer ships a ClinVar screen that is silently incomplete. - Its three two-command pipelines in the bcftools container (ClinVar's normalisation, the sample's PASS filter and the copy of ClinVar's fields onto the hits) ran under `sh`, which has no `pipefail`. A failed first `bcftools` went unnoticed when the second one exited 0, and the step finished with part of the records. They now run in `bash -o pipefail`, so the step stops with an error and writes no hits file. - - Two samples screened at the same time after a new install or a ClinVar refresh both rebuilt the shared `clinvar/clinvar_pathogenic_chr.norm.vcf.gz` into one `.part` file. One run could fail, or publish a `.tbi` of another write than the VCF. Now one run builds it under `clinvar/.norm.lock` while the others wait and then reuse it, and the index goes in place before the VCF. + - Two samples screened at the same time after a new install or a ClinVar refresh both rebuilt the shared `clinvar/clinvar_pathogenic_chr.norm.vcf.gz` into one `.part` file. One run could fail, or publish a `.tbi` of another write than the VCF. Now one run builds it under the lock `clinvar/.norm.lock` (`flock`, or the folder `clinvar/.norm.lock.d` where `flock` is not installed) while the others wait and then reuse it, and the index goes in place before the VCF. A stale copy in a `clinvar/` folder you cannot write stops the step with an error. No output file or option changes. The Nextflow module is not affected: it normalises in its own task folder and runs with `pipefail`. - The self-test of `scripts/ci/check-images.sh` copies the whole tree again once the list of tracked paths passes 128 KiB. It piped that list through `xargs` to `tar`, and GNU `xargs` split a list that long over two `tar` runs; the reading `tar` stopped after the first archive, so Guard's `images` job failed on a broken pipe (exit 123). One `tar` now reads the list itself (`tar --null -T -`). Nothing the check reports changes. - The phased family VCF is indexed by the Rust `og` binary. WhatsHap's step (step 38 and the Nextflow WHATSHAP_PED process) wrote `family.phased.vcf.gz.tbi` with a Python one-liner, `pysam.tabix_index`. It now runs `og vcf index -f`. The `.tbi` keeps its name and format, and its bytes, once inflated, are the ones htslib 1.24 writes (the htslib of the pysam and bcftools the pipeline pins): the same bins in the same order, the same chunks and the same linear index. Only the deflate stream can differ, as two deflate libraries differ. On a 5-million-record three-sample VCF it took 0.75 to 0.91 s and 31 MB at peak, where htslib's own `tabix` took 0.81 to 0.93 s and 19 MB. diff --git a/docs/06-clinvar-screen.md b/docs/06-clinvar-screen.md index e3143e77d..091105cf3 100644 --- a/docs/06-clinvar-screen.md +++ b/docs/06-clinvar-screen.md @@ -49,7 +49,7 @@ In the Nextflow pipeline the same floor is `ext.min_gq` and `ext.min_dp` of `CLI ### What the Script Does 1. Lists the contigs each file holds records on (`bcftools index -s`) and keeps those the reference has too: ClinVar's contigs that are in the reference, and the sample's contigs among those. `bcftools norm` stops at the first record whose contig the reference lacks, such as a provider's unplaced scaffold or one of the `NT_` contigs the full ClinVar file has, and a record on a contig the other file lacks cannot match anyway. The step prints how many of the sample's records it leaves out and on which contigs. When no contig is shared it stops with an error instead of reporting zero hits, and says which file is named the other way (`1, 2, ...` against `chr1, chr2, ...`). -2. Builds `clinvar/clinvar_pathogenic_chr.norm.vcf.gz` beside the ClinVar file the first time, and again whenever the source file is newer: ClinVar's records on the reference's contigs, multiallelic records split (`bcftools norm -m -any`) and indels left-aligned against the reference. Every sample then reuses it. Two samples screened at the same time build it once: the first takes `clinvar/.norm.lock` (a folder holding its process id, also without `flock`), the others wait, then find it built. The build goes to a `.part` file, and its index goes in place before the VCF, so the VCF and its `.tbi` always come from the same build. When the copy must be rebuilt and `clinvar/` is not writable by you, the step stops with an error. The Nextflow module reuses this file when it sits beside the `--clinvar` file and is newer; otherwise it normalises ClinVar inside the task. +2. Builds `clinvar/clinvar_pathogenic_chr.norm.vcf.gz` beside the ClinVar file the first time, and again whenever the source file is newer: ClinVar's records on the reference's contigs, multiallelic records split (`bcftools norm -m -any`) and indels left-aligned against the reference. Every sample then reuses it. Two samples screened at the same time build it once: the first takes the lock `clinvar/.norm.lock`, the others wait, then find it built. The lock is `flock`'s when `flock` is installed; without it (macOS) it is the folder `clinvar/.norm.lock.d` holding the process id. The build goes to a `.part` file, and its index goes in place before the VCF, so the VCF and its `.tbi` always come from the same build. When the copy must be rebuilt and `clinvar/` is not writable by you, the step stops with an error. The Nextflow module reuses this file when it sits beside the `--clinvar` file and is newer; otherwise it normalises ClinVar inside the task. 3. Filters the sample VCF to PASS records. If the VCF has no PASS record at all (callers that leave FILTER as `.`), it uses `-f .,PASS` instead and prints a notice saying so. If no record is left, the step stops with an error. 4. Splits and left-aligns the sample the same way. `bcftools isec` matches only identical REF/ALT, so a pathogenic allele inside a multiallelic record (genotype `1/2`) is found only after this split. The split writes the other alt allele as missing, not as the reference (`bcftools norm --multi-overlaps .`): a `1/2` record gives `1/.` and `./1`, where the default would give `1/0` and `0/1`. Both copies of such a call carry an alt allele, so neither part may read as het. A poly-T tract is where this shows most: a call of 5 T on one copy and 9 T on the other, against a reference of 7, is one `1/2` record, and its 5 T allele is a ClinVar hit. The reports word such a hit "one copy, other copy not called ref (1/.)", never het; the ACMG and carrier lists count it as one copy of two. This step does not size the tract: a short-read call of a homopolymer length can be off by one or two bases, so check the reads. A sample record whose REF is not the reference's base at that position is kept as written (`bcftools norm -c w`), with a `REF_MISMATCH` warning in the log. Such a record cannot match a ClinVar allele, so the summary of item 11 counts these records, and prints a notice to check that the VCF was called on this reference when there are any. 5. Builds the combined alleles of neighbouring records. A caller can write one complex allele, a delins, as two records a few bases apart on the same haplotype, for example a deletion and an insertion. Matched one by one, the parts can hit two single-submitter ClinVar records and miss the better reviewed record ClinVar holds for the whole allele. `bcftools norm -m +any` joins only records at the same position, so it cannot build that allele. `og clinvar join-adjacent` pairs two records when the second starts at most 10 bases after the first ends (records that overlap are never paired) and their genotypes allow both ALT alleles on one haplotype: phased with the same genotype in the same phase set (`FORMAT/PS`), or unphased with the same genotype (`0/1` and `1/0` count as the same het call; an unphased call with a missing allele such as `./1` is never paired). A phased call with a missing allele, such as `1|.`, has its ALT on a known haplotype: it pairs with a record in the same phase set whose ALT sits on that haplotype (`1|0` or `1|.`), and the combined record keeps `.` on the haplotype where either part is missing. That `.` means the copy was not called there; when the part comes from a split multiallelic record, it holds the other alt allele. Two unphased het calls can also sit on different haplotypes, and the genotype cannot tell which, so the notes of a combined allele and its parts built from them end with "unphased, so the parts may sit on different haplotypes". The combined record runs from the first part's start to the second part's end; `og clinvar join-adjacent` takes the bases between the parts from the reference (an uncompressed FASTA with its `.fai`), and `bcftools norm` trims and left-aligns it the way ClinVar was. The gene is not checked here, because the sample VCF has no gene annotation yet: a combined allele only counts when ClinVar holds a record for it. The step prints how many combined alleles it built. Only pairs are built: an allele split into three or more records is matched by its pairs only. diff --git a/scripts/06-clinvar-screen.sh b/scripts/06-clinvar-screen.sh index ecd549004..0e150b0dd 100755 --- a/scripts/06-clinvar-screen.sh +++ b/scripts/06-clinvar-screen.sh @@ -129,23 +129,31 @@ SAMPLE_DROPPED=$(comm -23 <(printf '%s\n' "$SAMPLE_CONTIGS") <(printf '%s\n' "$S # source is newer. isec matches on identical REF/ALT, so both sides must be split # and left-aligned the same way. # Every sample shares the normalised copy, so one run builds it while the others -# wait on clinvar/.norm.lock (lock_acquire, also without flock), then test it -# again and find it built. The build goes to a .part file only the lock holder +# wait on clinvar/.norm.lock, then test it again and find it built. The lock is +# flock's when flock is installed: the kernel drops it when the run ends, and a +# mkdir lock is not exclusive with the uutils mkdir of Ubuntu 26.04 (two runs +# that make the folder at once can both succeed). Without flock (macOS) it is +# lock_acquire's folder. The build goes to a .part file only the lock holder # writes; the index goes in place before the VCF, so a VCF that is newer than # the source always has the index of the same write beside it. # Each pipeline below runs in bash with pipefail: under sh a failed first # bcftools would go unnoticed when the second one exits 0. norm_stale() { [ ! -f "$CLINVAR_NORM" ] || [ ! -f "${CLINVAR_NORM}.tbi" ] || [ "$CLINVAR" -nt "$CLINVAR_NORM" ]; } if norm_stale; then - NORM_LOCK="${GENOME_DIR}/clinvar/.norm.lock" - # lock_acquire waits for as long as mkdir fails, so a folder it cannot write - # would make it wait forever. if [ ! -w "${GENOME_DIR}/clinvar" ]; then echo "ERROR: ${CLINVAR_NORM} must be rebuilt, but ${GENOME_DIR}/clinvar is not writable by you." >&2 exit 1 fi - lock_acquire "$NORM_LOCK" - trap 'lock_release "$NORM_LOCK"' EXIT + NORM_LOCK="${GENOME_DIR}/clinvar/.norm.lock" + if command -v flock >/dev/null 2>&1; then + NORM_FLOCK=true + exec 8>>"$NORM_LOCK" + flock -n 8 || { echo "Waiting for the run that builds ${CLINVAR_NORM} ..."; flock 8; } + else + NORM_FLOCK=false + lock_acquire "${NORM_LOCK}.d" + trap 'lock_release "${NORM_LOCK}.d"' EXIT + fi if norm_stale; then echo "Normalising ClinVar into ${CLINVAR_NORM} ..." rm -f "${CLINVAR_NORM}.part.vcf.gz" "${CLINVAR_NORM}.part.vcf.gz.tbi" @@ -157,8 +165,12 @@ if norm_stale; then mv "${CLINVAR_NORM}.part.vcf.gz.tbi" "${CLINVAR_NORM}.tbi" mv "${CLINVAR_NORM}.part.vcf.gz" "$CLINVAR_NORM" fi - lock_release "$NORM_LOCK" - trap - EXIT + if $NORM_FLOCK; then + exec 8>&- + else + lock_release "${NORM_LOCK}.d" + trap - EXIT + fi fi # Step 2: filter the sample. Callers that never write PASS (FILTER '.') would lose diff --git a/tests/fake-docker/rv-step06-screen-integrity.sh b/tests/fake-docker/rv-step06-screen-integrity.sh index ca5bf3eab..27cde1f32 100644 --- a/tests/fake-docker/rv-step06-screen-integrity.sh +++ b/tests/fake-docker/rv-step06-screen-integrity.sh @@ -8,7 +8,11 @@ # pipefail; # - two samples screened at the same time against a normalised ClinVar # older than its source build it once, and the published VCF and its -# .tbi come from that one write. +# .tbi come from that one write (with flock, which every Linux runner +# has; a mkdir lock is not exclusive with the uutils mkdir of Ubuntu +# 26.04); +# - without flock the step takes lock_acquire's folder and removes it; +# - a stale copy in a clinvar/ folder the user cannot write stops the step. # The docker run hook runs each container command for real, with /genome # mapped to GENOME_DIR and a fake bcftools and og first on PATH. The fake # tools write plain text and stamp every write with an id, which an index @@ -130,7 +134,7 @@ run_rc clean "${SCRIPTS}/06-clinvar-screen.sh" sample1 expect_rc clean 0 [ -f "${GENOME_DIR}/sample1/clinvar/sample1_clinvar_hits.vcf" ] || fail "a clean screen wrote no hits file" [ "$(write_id "$NORM")" = "$(write_id "${NORM}.tbi")" ] || fail "a clean screen published a .tbi of another write than the normalised ClinVar" -[ ! -e "${C}/.norm.lock" ] || fail "a clean screen left clinvar/.norm.lock behind" +[ ! -e "${C}/.norm.lock.d" ] || fail "a clean screen left the lock folder clinvar/.norm.lock.d behind" # --- A failed first stage of each nested pipeline fails the step --------------------- # name|regex of the first stage's arguments @@ -147,7 +151,7 @@ for stage in \ for f in sample1_clinvar_hits.vcf sample1_clinvar_hits.tsv; do [ ! -e "${GENOME_DIR}/sample1/clinvar/${f}" ] || fail "${name}: step 06 left ${f} after a failed bcftools" done - [ ! -e "${C}/.norm.lock" ] || fail "${name}: a failed screen left clinvar/.norm.lock behind" + [ ! -e "${C}/.norm.lock.d" ] || fail "${name}: a failed screen left the lock folder clinvar/.norm.lock.d behind" echo "${name}: a failed first bcftools fails the step and leaves no hits: PASS" done @@ -179,12 +183,29 @@ for s in sample1 sample2; do [ -f "${GENOME_DIR}/${s}/clinvar/${s}_clinvar_hits.vcf" ] || fail "${s}: no hits file after two screens at once" done [ -z "$(find "$C" -name '*.part*')" ] || fail "two screens at once left part files: $(find "$C" -name '*.part*')" -[ ! -e "${C}/.norm.lock" ] || fail "two screens at once left clinvar/.norm.lock behind" +[ ! -e "${C}/.norm.lock.d" ] || fail "two screens at once left the lock folder clinvar/.norm.lock.d behind" echo "two screens at once build the normalised ClinVar once, VCF and .tbi of one write: PASS" +# --- Without flock: lock_acquire's folder, taken and removed --------------------------- +# One run only: the folder lock is exclusive with a POSIX mkdir (macOS, GNU), +# but not with the uutils mkdir of Ubuntu 26.04, so two runs here would test +# the runner's mkdir, not the step. +touch -t 202001010000 "$NORM" "${NORM}.tbi" +rm -rf "${GENOME_DIR}/sample1/clinvar" +: > "$FAKE_TOOL_LOG" +( + hide_commands flock + run_rc no-flock "${SCRIPTS}/06-clinvar-screen.sh" sample1 + expect_rc no-flock 0 +) +grep -q '^bcftools norm .*clinvar_pathogenic_chr\.norm' "$FAKE_TOOL_LOG" || fail "without flock the stale normalised ClinVar was not rebuilt" +[ "$(write_id "$NORM")" = "$(write_id "${NORM}.tbi")" ] || fail "without flock the published .tbi is from another write than the VCF" +[ ! -e "${C}/.norm.lock.d" ] || fail "without flock the lock folder clinvar/.norm.lock.d was left behind" +echo "without flock the step builds under lock_acquire's folder and removes it: PASS" + # --- A stale normalised ClinVar in a folder the user cannot write ---------------------- -# The lock is a folder made in clinvar/, so the step must stop with an error -# there, not wait for a lock it can never take. root writes anywhere, so +# Without flock the lock is a folder made in clinvar/, so the step must stop +# with an error there, not wait for a lock it can never take. root writes anywhere, so # this part needs an ordinary user, as on the CI runner. if [ "$(id -u)" -eq 0 ]; then echo "read-only clinvar/: not checked as root, which can write any folder" @@ -192,10 +213,15 @@ else touch -t 202001010000 "$NORM" "${NORM}.tbi" rm -rf "${GENOME_DIR}/sample1/clinvar" chmod a-w "$C" - run_rc readonly-clinvar timeout 60 "${SCRIPTS}/06-clinvar-screen.sh" sample1 + rc=0 + ( + hide_commands flock + run_rc readonly-clinvar timeout 60 "${SCRIPTS}/06-clinvar-screen.sh" sample1 + exit "$RC" + ) || rc=$? chmod u+w "$C" - [ "$RC" -ne 124 ] || fail "step 06 was still waiting after 60 s for a lock in a clinvar/ folder it cannot write" - [ "$RC" -ne 0 ] || fail "step 06 exited 0 with a stale normalised ClinVar it cannot rebuild" + [ "$rc" -ne 124 ] || fail "step 06 was still waiting after 60 s for a lock in a clinvar/ folder it cannot write" + [ "$rc" -ne 0 ] || fail "step 06 exited 0 with a stale normalised ClinVar it cannot rebuild" output_has readonly-clinvar 'clinvar is not writable' echo "a stale normalised ClinVar in a read-only folder stops the step with an error: PASS" fi From 817555a055ef10e4d26dfa920f2860e91d9a4938 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 22:38:30 +0100 Subject: [PATCH 17/22] test(parity): take the 3gen and bench family goldens from the runner's gawk input --- .../report-3gen-duo/out/family_report.md | 18 ++++++++--------- .../golden/family/report-3gen-duo/stdout | 18 ++++++++--------- .../family/report-3gen/out/family_report.md | 14 ++++++------- tests/parity/golden/family/report-3gen/stdout | 14 ++++++------- .../family/report-bench/out/family_report.md | 20 +++++++++---------- .../parity/golden/family/report-bench/stdout | 20 +++++++++---------- 6 files changed, 52 insertions(+), 52 deletions(-) diff --git a/tests/parity/golden/family/report-3gen-duo/out/family_report.md b/tests/parity/golden/family/report-3gen-duo/out/family_report.md index a21df2733..e0acfa3b0 100644 --- a/tests/parity/golden/family/report-3gen-duo/out/family_report.md +++ b/tests/parity/golden/family/report-3gen-duo/out/family_report.md @@ -15,9 +15,9 @@ Father: gpa (sequenced). Mother: gma (sequenced). Sites of the joint VCF where dad has a genotype and a sequenced parent does too: 35000. `dad_inheritance.tsv.gz` lists them. -- the allele from the father is known at 26293 (75.1%) -- the allele from the mother is known at 31293 (89.4%) -- both parents heterozygous like the child, so no origin from the genotypes: 3707 (10.6%) +- the allele from the father is known at 26209 (74.9%) +- the allele from the mother is known at 31209 (89.2%) +- both parents heterozygous like the child, so no origin from the genotypes: 3791 (10.8%) - inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 0 (0.0%) De novo candidates (an allele neither parent has): 0, 0 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. @@ -28,12 +28,12 @@ Father: dad (sequenced). Mother: mum (not sequenced). Sites of the joint VCF where kid has a genotype and a sequenced parent does too: 30000. `kid_inheritance.tsv.gz` lists them. -- the allele from the father is known at 22455 (74.8%) -- the allele from the mother is known at 22455 (74.8%) -- both parents heterozygous like the child, so no origin from the genotypes: 7545 (25.1%) +- the allele from the father is known at 22524 (75.1%) +- the allele from the mother is known at 22524 (75.1%) +- both parents heterozygous like the child, so no origin from the genotypes: 7476 (24.9%) - inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 0 (0.0%) -The mother is not sequenced. At 22455 of these sites (74.8%) the genotypes give the allele the mother passed on to kid: that is the half of the mother's genome kid carries. The other half was not passed on and is not seen here. +The mother is not sequenced. At 22524 of these sites (75.1%) the genotypes give the allele the mother passed on to kid: that is the half of the mother's genome kid carries. The other half was not passed on and is not seen here. No de novo candidates are called: that needs both parents sequenced. @@ -46,8 +46,8 @@ No candidate variants were given (the clinical filter, step 23, did not run for For each child, the stretches of each chromosome that came from each parent, from the sites where that parent is heterozygous and the genotypes give the allele passed on. When a parent's own parent is sequenced too, each stretch also says which of the parent's two copies it is (the grandfather's or the grandmother's), and a change between them is a crossover. `_segments.tsv` lists them, `_dnapainter.csv` is the same list in DNA Painter's import format (GRCh38 positions), `_segments.svg` draws it. - dad: from the father (gpa): 3 stretches over 60.0 Mb; no grandparent on this side is sequenced, so the two copies of the father are not told apart -- dad: from the mother (gma): 4 stretches over 69.9 Mb; no grandparent on this side is sequenced, so the two copies of the mother are not told apart +- dad: from the mother (gma): 4 stretches over 70.0 Mb; no grandparent on this side is sequenced, so the two copies of the mother are not told apart - kid: from the father (dad): 4 stretches over 59.9 Mb, told apart as the copy from gma or gpa; 1 crossover, 0 discordant sites absorbed -- kid: from the mother (mum, not sequenced): 3 stretches over 59.9 Mb, the copy that did not come from the sequenced parent +- kid: from the mother (mum, not sequenced): 3 stretches over 60.0 Mb, the copy that did not come from the sequenced parent A stretch starts and ends at its first and last informative site: a crossover lies somewhere between two stretches, not at a base. Short reads give no long-range phase, so without a sequenced grandparent the map says only which parent a stretch came from. diff --git a/tests/parity/golden/family/report-3gen-duo/stdout b/tests/parity/golden/family/report-3gen-duo/stdout index a21df2733..e0acfa3b0 100644 --- a/tests/parity/golden/family/report-3gen-duo/stdout +++ b/tests/parity/golden/family/report-3gen-duo/stdout @@ -15,9 +15,9 @@ Father: gpa (sequenced). Mother: gma (sequenced). Sites of the joint VCF where dad has a genotype and a sequenced parent does too: 35000. `dad_inheritance.tsv.gz` lists them. -- the allele from the father is known at 26293 (75.1%) -- the allele from the mother is known at 31293 (89.4%) -- both parents heterozygous like the child, so no origin from the genotypes: 3707 (10.6%) +- the allele from the father is known at 26209 (74.9%) +- the allele from the mother is known at 31209 (89.2%) +- both parents heterozygous like the child, so no origin from the genotypes: 3791 (10.8%) - inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 0 (0.0%) De novo candidates (an allele neither parent has): 0, 0 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. @@ -28,12 +28,12 @@ Father: dad (sequenced). Mother: mum (not sequenced). Sites of the joint VCF where kid has a genotype and a sequenced parent does too: 30000. `kid_inheritance.tsv.gz` lists them. -- the allele from the father is known at 22455 (74.8%) -- the allele from the mother is known at 22455 (74.8%) -- both parents heterozygous like the child, so no origin from the genotypes: 7545 (25.1%) +- the allele from the father is known at 22524 (75.1%) +- the allele from the mother is known at 22524 (75.1%) +- both parents heterozygous like the child, so no origin from the genotypes: 7476 (24.9%) - inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 0 (0.0%) -The mother is not sequenced. At 22455 of these sites (74.8%) the genotypes give the allele the mother passed on to kid: that is the half of the mother's genome kid carries. The other half was not passed on and is not seen here. +The mother is not sequenced. At 22524 of these sites (75.1%) the genotypes give the allele the mother passed on to kid: that is the half of the mother's genome kid carries. The other half was not passed on and is not seen here. No de novo candidates are called: that needs both parents sequenced. @@ -46,8 +46,8 @@ No candidate variants were given (the clinical filter, step 23, did not run for For each child, the stretches of each chromosome that came from each parent, from the sites where that parent is heterozygous and the genotypes give the allele passed on. When a parent's own parent is sequenced too, each stretch also says which of the parent's two copies it is (the grandfather's or the grandmother's), and a change between them is a crossover. `_segments.tsv` lists them, `_dnapainter.csv` is the same list in DNA Painter's import format (GRCh38 positions), `_segments.svg` draws it. - dad: from the father (gpa): 3 stretches over 60.0 Mb; no grandparent on this side is sequenced, so the two copies of the father are not told apart -- dad: from the mother (gma): 4 stretches over 69.9 Mb; no grandparent on this side is sequenced, so the two copies of the mother are not told apart +- dad: from the mother (gma): 4 stretches over 70.0 Mb; no grandparent on this side is sequenced, so the two copies of the mother are not told apart - kid: from the father (dad): 4 stretches over 59.9 Mb, told apart as the copy from gma or gpa; 1 crossover, 0 discordant sites absorbed -- kid: from the mother (mum, not sequenced): 3 stretches over 59.9 Mb, the copy that did not come from the sequenced parent +- kid: from the mother (mum, not sequenced): 3 stretches over 60.0 Mb, the copy that did not come from the sequenced parent A stretch starts and ends at its first and last informative site: a crossover lies somewhere between two stretches, not at a base. Short reads give no long-range phase, so without a sequenced grandparent the map says only which parent a stretch came from. diff --git a/tests/parity/golden/family/report-3gen/out/family_report.md b/tests/parity/golden/family/report-3gen/out/family_report.md index 642cc4774..3a81ff6f2 100644 --- a/tests/parity/golden/family/report-3gen/out/family_report.md +++ b/tests/parity/golden/family/report-3gen/out/family_report.md @@ -15,9 +15,9 @@ Father: gpa (sequenced). Mother: gma (sequenced). Sites of the joint VCF where dad has a genotype and a sequenced parent does too: 35000. `dad_inheritance.tsv.gz` lists them. -- the allele from the father is known at 26293 (75.1%) -- the allele from the mother is known at 31293 (89.4%) -- both parents heterozygous like the child, so no origin from the genotypes: 3707 (10.6%) +- the allele from the father is known at 26209 (74.9%) +- the allele from the mother is known at 31209 (89.2%) +- both parents heterozygous like the child, so no origin from the genotypes: 3791 (10.8%) - inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 0 (0.0%) De novo candidates (an allele neither parent has): 0, 0 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. @@ -28,9 +28,9 @@ Father: dad (sequenced). Mother: mum (sequenced). Sites of the joint VCF where kid has a genotype and a sequenced parent does too: 35000. `kid_inheritance.tsv.gz` lists them. -- the allele from the father is known at 26192 (74.8%) -- the allele from the mother is known at 31192 (89.1%) -- both parents heterozygous like the child, so no origin from the genotypes: 3808 (10.9%) +- the allele from the father is known at 26289 (75.1%) +- the allele from the mother is known at 31289 (89.4%) +- both parents heterozygous like the child, so no origin from the genotypes: 3711 (10.6%) - inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 0 (0.0%) De novo candidates (an allele neither parent has): 0, 0 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. @@ -44,7 +44,7 @@ No candidate variants were given (the clinical filter, step 23, did not run for For each child, the stretches of each chromosome that came from each parent, from the sites where that parent is heterozygous and the genotypes give the allele passed on. When a parent's own parent is sequenced too, each stretch also says which of the parent's two copies it is (the grandfather's or the grandmother's), and a change between them is a crossover. `_segments.tsv` lists them, `_dnapainter.csv` is the same list in DNA Painter's import format (GRCh38 positions), `_segments.svg` draws it. - dad: from the father (gpa): 3 stretches over 60.0 Mb; no grandparent on this side is sequenced, so the two copies of the father are not told apart -- dad: from the mother (gma): 4 stretches over 69.9 Mb; no grandparent on this side is sequenced, so the two copies of the mother are not told apart +- dad: from the mother (gma): 4 stretches over 70.0 Mb; no grandparent on this side is sequenced, so the two copies of the mother are not told apart - kid: from the father (dad): 4 stretches over 59.9 Mb, told apart as the copy from gma or gpa; 1 crossover, 3 discordant sites absorbed - kid: from the mother (mum): 4 stretches over 70.0 Mb; no grandparent on this side is sequenced, so the two copies of the mother are not told apart diff --git a/tests/parity/golden/family/report-3gen/stdout b/tests/parity/golden/family/report-3gen/stdout index 642cc4774..3a81ff6f2 100644 --- a/tests/parity/golden/family/report-3gen/stdout +++ b/tests/parity/golden/family/report-3gen/stdout @@ -15,9 +15,9 @@ Father: gpa (sequenced). Mother: gma (sequenced). Sites of the joint VCF where dad has a genotype and a sequenced parent does too: 35000. `dad_inheritance.tsv.gz` lists them. -- the allele from the father is known at 26293 (75.1%) -- the allele from the mother is known at 31293 (89.4%) -- both parents heterozygous like the child, so no origin from the genotypes: 3707 (10.6%) +- the allele from the father is known at 26209 (74.9%) +- the allele from the mother is known at 31209 (89.2%) +- both parents heterozygous like the child, so no origin from the genotypes: 3791 (10.8%) - inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 0 (0.0%) De novo candidates (an allele neither parent has): 0, 0 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. @@ -28,9 +28,9 @@ Father: dad (sequenced). Mother: mum (sequenced). Sites of the joint VCF where kid has a genotype and a sequenced parent does too: 35000. `kid_inheritance.tsv.gz` lists them. -- the allele from the father is known at 26192 (74.8%) -- the allele from the mother is known at 31192 (89.1%) -- both parents heterozygous like the child, so no origin from the genotypes: 3808 (10.9%) +- the allele from the father is known at 26289 (75.1%) +- the allele from the mother is known at 31289 (89.4%) +- both parents heterozygous like the child, so no origin from the genotypes: 3711 (10.6%) - inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 0 (0.0%) De novo candidates (an allele neither parent has): 0, 0 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. @@ -44,7 +44,7 @@ No candidate variants were given (the clinical filter, step 23, did not run for For each child, the stretches of each chromosome that came from each parent, from the sites where that parent is heterozygous and the genotypes give the allele passed on. When a parent's own parent is sequenced too, each stretch also says which of the parent's two copies it is (the grandfather's or the grandmother's), and a change between them is a crossover. `_segments.tsv` lists them, `_dnapainter.csv` is the same list in DNA Painter's import format (GRCh38 positions), `_segments.svg` draws it. - dad: from the father (gpa): 3 stretches over 60.0 Mb; no grandparent on this side is sequenced, so the two copies of the father are not told apart -- dad: from the mother (gma): 4 stretches over 69.9 Mb; no grandparent on this side is sequenced, so the two copies of the mother are not told apart +- dad: from the mother (gma): 4 stretches over 70.0 Mb; no grandparent on this side is sequenced, so the two copies of the mother are not told apart - kid: from the father (dad): 4 stretches over 59.9 Mb, told apart as the copy from gma or gpa; 1 crossover, 3 discordant sites absorbed - kid: from the mother (mum): 4 stretches over 70.0 Mb; no grandparent on this side is sequenced, so the two copies of the mother are not told apart diff --git a/tests/parity/golden/family/report-bench/out/family_report.md b/tests/parity/golden/family/report-bench/out/family_report.md index 0d102a174..33cb140c4 100644 --- a/tests/parity/golden/family/report-bench/out/family_report.md +++ b/tests/parity/golden/family/report-bench/out/family_report.md @@ -13,13 +13,13 @@ Relatedness is twice the kinship coefficient: about the share of the genome the Father: GF (sequenced). Mother: GM (sequenced). -Sites of the joint VCF where DAD has a genotype and a sequenced parent does too: 199936. `DAD_inheritance.tsv.gz` lists them. +Sites of the joint VCF where DAD has a genotype and a sequenced parent does too: 199931. `DAD_inheritance.tsv.gz` lists them. -- the allele from the father is known at 176839 (88.4%) -- the allele from the mother is known at 176890 (88.5%) -- both parents heterozygous like the child, so no origin from the genotypes: 23046 (11.5%) +- the allele from the father is known at 176956 (88.5%) +- the allele from the mother is known at 177002 (88.5%) +- both parents heterozygous like the child, so no origin from the genotypes: 22929 (11.5%) - inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 0 (0.0%) -- left out on chrX or chrY outside the pseudoautosomal regions (sex unknown, or a heterozygous call on a single X or Y): 49 +- left out on chrX or chrY outside the pseudoautosomal regions (sex unknown, or a heterozygous call on a single X or Y): 54 De novo candidates (an allele neither parent has): 0, 0 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. @@ -29,9 +29,9 @@ Father: DAD (sequenced). Mother: MOM (sequenced). Sites of the joint VCF where KID has a genotype and a sequenced parent does too: 199985. `KID_inheritance.tsv.gz` lists them. -- the allele from the father is known at 176935 (88.5%) -- the allele from the mother is known at 176935 (88.5%) -- both parents heterozygous like the child, so no origin from the genotypes: 23050 (11.5%) +- the allele from the father is known at 177017 (88.5%) +- the allele from the mother is known at 177017 (88.5%) +- both parents heterozygous like the child, so no origin from the genotypes: 22968 (11.5%) - inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 0 (0.0%) De novo candidates (an allele neither parent has): 0, 0 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. @@ -46,7 +46,7 @@ For each child, the stretches of each chromosome that came from each parent, fro - DAD: from the father (GF): 22 stretches over 19.1 Mb; no grandparent on this side is sequenced, so the two copies of the father are not told apart - DAD: from the mother (GM): 23 stretches over 19.1 Mb; no grandparent on this side is sequenced, so the two copies of the mother are not told apart -- KID: from the father (DAD): 22 stretches over 19.1 Mb, told apart as the copy from GF or GM; 0 crossovers, 25582 discordant sites absorbed -- KID: from the mother (MOM): 22 stretches over 19.1 Mb; no grandparent on this side is sequenced, so the two copies of the mother are not told apart +- KID: from the father (DAD): 22 stretches over 19.1 Mb, told apart as the copy from GF or GM; 0 crossovers, 25195 discordant sites absorbed +- KID: from the mother (MOM): 23 stretches over 19.1 Mb; no grandparent on this side is sequenced, so the two copies of the mother are not told apart A stretch starts and ends at its first and last informative site: a crossover lies somewhere between two stretches, not at a base. Short reads give no long-range phase, so without a sequenced grandparent the map says only which parent a stretch came from. diff --git a/tests/parity/golden/family/report-bench/stdout b/tests/parity/golden/family/report-bench/stdout index 0d102a174..33cb140c4 100644 --- a/tests/parity/golden/family/report-bench/stdout +++ b/tests/parity/golden/family/report-bench/stdout @@ -13,13 +13,13 @@ Relatedness is twice the kinship coefficient: about the share of the genome the Father: GF (sequenced). Mother: GM (sequenced). -Sites of the joint VCF where DAD has a genotype and a sequenced parent does too: 199936. `DAD_inheritance.tsv.gz` lists them. +Sites of the joint VCF where DAD has a genotype and a sequenced parent does too: 199931. `DAD_inheritance.tsv.gz` lists them. -- the allele from the father is known at 176839 (88.4%) -- the allele from the mother is known at 176890 (88.5%) -- both parents heterozygous like the child, so no origin from the genotypes: 23046 (11.5%) +- the allele from the father is known at 176956 (88.5%) +- the allele from the mother is known at 177002 (88.5%) +- both parents heterozygous like the child, so no origin from the genotypes: 22929 (11.5%) - inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 0 (0.0%) -- left out on chrX or chrY outside the pseudoautosomal regions (sex unknown, or a heterozygous call on a single X or Y): 49 +- left out on chrX or chrY outside the pseudoautosomal regions (sex unknown, or a heterozygous call on a single X or Y): 54 De novo candidates (an allele neither parent has): 0, 0 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. @@ -29,9 +29,9 @@ Father: DAD (sequenced). Mother: MOM (sequenced). Sites of the joint VCF where KID has a genotype and a sequenced parent does too: 199985. `KID_inheritance.tsv.gz` lists them. -- the allele from the father is known at 176935 (88.5%) -- the allele from the mother is known at 176935 (88.5%) -- both parents heterozygous like the child, so no origin from the genotypes: 23050 (11.5%) +- the allele from the father is known at 177017 (88.5%) +- the allele from the mother is known at 177017 (88.5%) +- both parents heterozygous like the child, so no origin from the genotypes: 22968 (11.5%) - inconsistent with the declared parents (a genotype error, a deletion, or a new mutation): 0 (0.0%) De novo candidates (an allele neither parent has): 0, 0 of them with GQ >= 20 and depth >= 10 in the child and in each parent who passes the site on (on a son's X outside the pseudoautosomal regions the mother, on his Y the father), fewer than 2 reads and under 5% of the reads on the new allele in each such parent and, for a heterozygous call, an allele balance of 0.25 to 0.75 (`de_novo.tsv`). Most candidates of a genome are genotype errors; a candidate that matters needs confirming by another method. @@ -46,7 +46,7 @@ For each child, the stretches of each chromosome that came from each parent, fro - DAD: from the father (GF): 22 stretches over 19.1 Mb; no grandparent on this side is sequenced, so the two copies of the father are not told apart - DAD: from the mother (GM): 23 stretches over 19.1 Mb; no grandparent on this side is sequenced, so the two copies of the mother are not told apart -- KID: from the father (DAD): 22 stretches over 19.1 Mb, told apart as the copy from GF or GM; 0 crossovers, 25582 discordant sites absorbed -- KID: from the mother (MOM): 22 stretches over 19.1 Mb; no grandparent on this side is sequenced, so the two copies of the mother are not told apart +- KID: from the father (DAD): 22 stretches over 19.1 Mb, told apart as the copy from GF or GM; 0 crossovers, 25195 discordant sites absorbed +- KID: from the mother (MOM): 23 stretches over 19.1 Mb; no grandparent on this side is sequenced, so the two copies of the mother are not told apart A stretch starts and ends at its first and last informative site: a crossover lies somewhere between two stretches, not at a base. Short reads give no long-range phase, so without a sequenced grandparent the map says only which parent a stretch came from. From cc99fa2054a2cb24e09ece24e67a09cec06e9fb7 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 18 Nov 2026 10:17:09 +0100 Subject: [PATCH 18/22] fix(family): more identical genotypes than sites is no count either somalier's ibs2 counts the sites where the two share both alleles, so it can never pass n. A pair with ibs2 over n at 1,000 sites or more now gets not_checked with the counts reason, as ibs0 over n already did. The sites floor keeps its precedence. Bead: C18 --- CHANGELOG.md | 2 +- docs/multi-sample.md | 2 +- rust/crates/og-steps/src/family/check.rs | 5 +++-- rust/crates/og-steps/tests/rv_family_check_nan.rs | 13 ++++++++++--- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40f52bbf3..94eee8917 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- `og family check` (step 38 and the Nextflow `FAMILY_CHECK` process) no longer reports `ok` for a pair somalier could not measure. somalier writes `nan` as the relatedness when it had nothing to divide by. Every comparison with NaN is false, so the degree bins read it as unrelated, and a pair declared unrelated or distant passed as "unrelated, as declared". Such a pair, one with `inf`, and one with counts that cannot be counts (negative, or more opposite homozygotes than sites) now get the verdict `not_checked`, a reason that names the value ("somalier gave relatedness 'nan' for this pair: not measured") and a `WARNING:` line. The step still exits 0 for them. Under 1,000 sites the reason stays "too few to tell", unless a count is negative. Before, `inf` stopped the step as a mismatch ("same person"). The columns of `relationships.tsv` do not change; `not_checked` was already one of its verdicts. +- `og family check` (step 38 and the Nextflow `FAMILY_CHECK` process) no longer reports `ok` for a pair somalier could not measure. somalier writes `nan` as the relatedness when it had nothing to divide by. Every comparison with NaN is false, so the degree bins read it as unrelated, and a pair declared unrelated or distant passed as "unrelated, as declared". Such a pair, one with `inf`, and one with counts that cannot be counts (negative, or more opposite homozygotes or more identical genotypes than sites) now get the verdict `not_checked`, a reason that names the value ("somalier gave relatedness 'nan' for this pair: not measured") and a `WARNING:` line. The step still exits 0 for them. Under 1,000 sites the reason stays "too few to tell", unless a count is negative. Before, `inf` stopped the step as a mismatch ("same person"). The columns of `relationships.tsv` do not change; `not_checked` was already one of its verdicts. - A new page, [User journeys](docs/user-journeys.md), records where a new user gets stuck on each surface: the docs, the demo, the shell and the local site. It gives the outcome of each journey in the first test-as-user run (2026-11-17, commit `65095e97`), the fix for each finding, and what the run could not reach. A fix that lands has a journey to walk again. - VCF intake now refuses a record with GT after another FORMAT key, such as `DP:GT:GQ`, before a reader can mistake depth for the genotype. The check exits 2 and names the sample, file, first offending position and FORMAT field. It tells users to reorder the keys and sample values together or get a corrected VCF. Records without GT remain allowed by this ordering check. Valid input and output formats are unchanged. Part of #411 (clin-37). - The `archaic-same-person` job of `public-evidence.yml` puts step 43's Neanderthal totals beside Chen et al.'s released calls with `og archaic compare` and `og archaic genomes`, in Rust. They replace `bin/archaic_compare.py`, which is gone with its test, `tests/test_archaic_compare.py`; those checks are now cargo tests in og-ancestry. The tables are the same, byte for byte, on 20 parity cases and on the inputs of the last green run of the job (NA12878, chromosomes 19 to 22). No step of the pipeline runs them, and no option or output changes. What changes on purpose: a missing file, an unreadable number or an empty spans file stops with an `ERROR:` line and exit 1 instead of a Python traceback; an interval with a negative start or an end before its start stops the command, where the Python counted a negative length; and a `--mine` value without `=` is a usage error (exit 2). diff --git a/docs/multi-sample.md b/docs/multi-sample.md index 4aecf0690..e0091c391 100644 --- a/docs/multi-sample.md +++ b/docs/multi-sample.md @@ -70,7 +70,7 @@ It reads the `family.vcf.gz` and `pedigree.tsv` the earlier run left in `$GENOME ### What it does 1. **Joint call.** GLnexus (`GLNEXUS_IMAGE`, its DeepVariantWGS preset) merges the members' DeepVariant gVCFs into one VCF, `family.vcf.gz`. A site where one person has a variant gets the others' genotypes from their reference blocks, so "not called" and "matches the reference" are told apart. The container gets 16 GB (`GLNEXUS_MEM` in step 38, the `GLNEXUS` process in `conf/base.config`), and GLnexus's own `--mem-gbytes` budget is half of it. That budget sizes only GLnexus's database cache and write buffers; the alleles it discovers and the sites it merges are held outside it. So a budget near the limit leaves no room for them. Whether 16 GB is enough for three whole genomes under the half rule has not been measured yet; when the task is killed for memory, Nextflow retries it once with 32 GB (capped at `--max_memory`), and step 38 takes a larger `GLNEXUS_MEM`. -2. **Relationship check.** somalier extracts each member's genotypes at its sites from the gVCF and `somalier relate` measures the relatedness of every pair: about the share of the genome two people inherited from common ancestors (1 the same person, 0.5 parent and child or full siblings, 0.25 second degree, 0 unrelated). The pedigree gives the expected value. A pair must fall in the same degree as declared (bins at 0.707, 0.354, 0.177 and 0.088); a declared parent and child must also be opposite homozygotes at under 1% of the sites, which siblings are not; a pair declared unrelated or distant must not measure first degree. A pair that disagrees stops the step with exit 1, before any phasing, and `relationships.tsv` says which pair and why: two samples were swapped, or a pedigree line is wrong. A pair compared at fewer than 1,000 sites is reported as not checked. So is a pair somalier gave no relatedness for (`nan` or `inf`), or counts that cannot be counts (negative, or more opposite homozygotes than sites): the reason names the value, a `WARNING:` line says it, and the step goes on. Under 1,000 sites the reason stays the number of sites, unless a count is negative. +2. **Relationship check.** somalier extracts each member's genotypes at its sites from the gVCF and `somalier relate` measures the relatedness of every pair: about the share of the genome two people inherited from common ancestors (1 the same person, 0.5 parent and child or full siblings, 0.25 second degree, 0 unrelated). The pedigree gives the expected value. A pair must fall in the same degree as declared (bins at 0.707, 0.354, 0.177 and 0.088); a declared parent and child must also be opposite homozygotes at under 1% of the sites, which siblings are not; a pair declared unrelated or distant must not measure first degree. A pair that disagrees stops the step with exit 1, before any phasing, and `relationships.tsv` says which pair and why: two samples were swapped, or a pedigree line is wrong. A pair compared at fewer than 1,000 sites is reported as not checked. So is a pair somalier gave no relatedness for (`nan` or `inf`), or counts that cannot be counts (negative, or more opposite homozygotes or more identical genotypes than sites): the reason names the value, a `WARNING:` line says it, and the step goes on. Under 1,000 sites the reason stays the number of sites, unless a count is negative. 3. **Phasing by inheritance.** For each child whose two parents are sequenced, WhatsHap (`WHATSHAP_IMAGE`, `whatshap phase --ped`) phases the family by inheritance alone; no reads are given. A site where the child and both parents are heterozygous stays unphased, because no genotype tells it. 4. **The report.** For each child with a sequenced parent, `_inheritance.tsv.gz` gives, at every site of the joint VCF where the child and a sequenced parent have a genotype, the allele from the father and the allele from the mother when the genotypes tell (`how` is `genotype`), or `ambiguous` (both parents heterozygous like the child) or `inconsistent` (a genotype error, a deletion or a new mutation). On chrX and chrY outside the pseudoautosomal regions a son's X comes from his mother and his Y from his father; the child's sex must be known there. chrM is left out. With one parent sequenced, the allele the other parent passed on is known wherever the sequenced parent is homozygous or differs from the child: that is the half of the unsequenced parent's genome the child carries, and `family_report.md` gives its share of the sites. The half the child did not inherit is not seen. diff --git a/rust/crates/og-steps/src/family/check.rs b/rust/crates/og-steps/src/family/check.rs index c88ddd441..6f660dc8b 100644 --- a/rust/crates/og-steps/src/family/check.rs +++ b/rust/crates/og-steps/src/family/check.rs @@ -85,7 +85,8 @@ fn verdict( /// somalier writes `nan` (or `inf`) when it had nothing to divide by; every /// comparison with NaN is false, so [`verdict`] would bin it as unrelated /// and call a pair declared unrelated or distant ok. A negative count, or -/// more opposite homozygotes than sites, is no count somalier can make. +/// more opposite homozygotes (ibs0) or identical genotypes (ibs2) than +/// sites, is no count somalier can make. /// Under [`MIN_SITES`] sites [`verdict`] already says too few to tell, the /// more useful reason, so only a negative count comes before it. fn unmeasured(rel_text: &str, rel: f64, ibs0: i128, ibs2: i128, n: i128) -> Option { @@ -101,7 +102,7 @@ fn unmeasured(rel_text: &str, rel: f64, ibs0: i128, ibs2: i128, n: i128) -> Opti if n < MIN_SITES { return None; } - if ibs0 > n { + if ibs0 > n || ibs2 > n { return Some(counts()); } if !rel.is_finite() { diff --git a/rust/crates/og-steps/tests/rv_family_check_nan.rs b/rust/crates/og-steps/tests/rv_family_check_nan.rs index c99634384..06bf3b2da 100644 --- a/rust/crates/og-steps/tests/rv_family_check_nan.rs +++ b/rust/crates/og-steps/tests/rv_family_check_nan.rs @@ -4,9 +4,10 @@ //! had nothing to divide by. Every comparison with NaN is false, so the //! degree bins read it as unrelated, and a pair the pedigree declares //! unrelated or distant came out `ok`. Such a pair, and one with counts -//! that cannot be counts (negative, or more opposite homozygotes than -//! sites), is `not_checked` with the reason and a `WARNING:` line, and the -//! step still exits 0. A finite relatedness keeps its verdict. +//! that cannot be counts (negative, or more opposite homozygotes or more +//! identical genotypes than sites), is `not_checked` with the reason and a +//! `WARNING:` line, and the step still exits 0. A finite relatedness keeps +//! its verdict. //! //! The case that reads stderr runs `og family` in a child process: this //! test binary itself, with [`CHILD`] set (see [`child_entry`]). @@ -182,6 +183,12 @@ fn counts_that_cannot_be_counts_are_not_checked() { "a\tb\t0.01\t2000\t500\t1000", &reason(2000, 500, 1000), ); + assert_not_checked( + "ibs2-over-n", + PARTNERS, + "a\tb\t0.01\t10\t2000\t1000", + &reason(10, 2000, 1000), + ); assert_not_checked( "negative-ibs2", PARTNERS, From 7f6e39b78b8d6a7e305ab5209f009431d18b6998 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 18 Nov 2026 12:26:05 +0100 Subject: [PATCH 19/22] test(family): a declared parent and child with nan relatedness is not checked, not a mismatch The changelog now says that nan for a pair declared first or second degree used to stop the step, and no longer does. --- CHANGELOG.md | 2 +- .../og-steps/tests/rv_family_check_nan.rs | 23 ++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94eee8917..15767efc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- `og family check` (step 38 and the Nextflow `FAMILY_CHECK` process) no longer reports `ok` for a pair somalier could not measure. somalier writes `nan` as the relatedness when it had nothing to divide by. Every comparison with NaN is false, so the degree bins read it as unrelated, and a pair declared unrelated or distant passed as "unrelated, as declared". Such a pair, one with `inf`, and one with counts that cannot be counts (negative, or more opposite homozygotes or more identical genotypes than sites) now get the verdict `not_checked`, a reason that names the value ("somalier gave relatedness 'nan' for this pair: not measured") and a `WARNING:` line. The step still exits 0 for them. Under 1,000 sites the reason stays "too few to tell", unless a count is negative. Before, `inf` stopped the step as a mismatch ("same person"). The columns of `relationships.tsv` do not change; `not_checked` was already one of its verdicts. +- `og family check` (step 38 and the Nextflow `FAMILY_CHECK` process) no longer reports `ok` for a pair somalier could not measure. somalier writes `nan` as the relatedness when it had nothing to divide by. Every comparison with NaN is false, so the degree bins read it as unrelated, and a pair declared unrelated or distant passed as "unrelated, as declared". Such a pair, one with `inf`, and one with counts that cannot be counts (negative, or more opposite homozygotes or more identical genotypes than sites) now get the verdict `not_checked`, a reason that names the value ("somalier gave relatedness 'nan' for this pair: not measured") and a `WARNING:` line. The step exits 0 for them. Under 1,000 sites the reason stays "too few to tell", unless a count is negative. Before, `inf`, and `nan` for a pair declared first or second degree, stopped the step as a mismatch. The columns of `relationships.tsv` do not change; `not_checked` was already one of its verdicts. - A new page, [User journeys](docs/user-journeys.md), records where a new user gets stuck on each surface: the docs, the demo, the shell and the local site. It gives the outcome of each journey in the first test-as-user run (2026-11-17, commit `65095e97`), the fix for each finding, and what the run could not reach. A fix that lands has a journey to walk again. - VCF intake now refuses a record with GT after another FORMAT key, such as `DP:GT:GQ`, before a reader can mistake depth for the genotype. The check exits 2 and names the sample, file, first offending position and FORMAT field. It tells users to reorder the keys and sample values together or get a corrected VCF. Records without GT remain allowed by this ordering check. Valid input and output formats are unchanged. Part of #411 (clin-37). - The `archaic-same-person` job of `public-evidence.yml` puts step 43's Neanderthal totals beside Chen et al.'s released calls with `og archaic compare` and `og archaic genomes`, in Rust. They replace `bin/archaic_compare.py`, which is gone with its test, `tests/test_archaic_compare.py`; those checks are now cargo tests in og-ancestry. The tables are the same, byte for byte, on 20 parity cases and on the inputs of the last green run of the job (NA12878, chromosomes 19 to 22). No step of the pipeline runs them, and no option or output changes. What changes on purpose: a missing file, an unreadable number or an empty spans file stops with an `ERROR:` line and exit 1 instead of a Python traceback; an interval with a negative start or an end before its start stops the command, where the Python counted a negative length; and a `--mine` value without `=` is a usage error (exit 2). diff --git a/rust/crates/og-steps/tests/rv_family_check_nan.rs b/rust/crates/og-steps/tests/rv_family_check_nan.rs index 06bf3b2da..57dc59c7a 100644 --- a/rust/crates/og-steps/tests/rv_family_check_nan.rs +++ b/rust/crates/og-steps/tests/rv_family_check_nan.rs @@ -6,7 +6,8 @@ //! unrelated or distant came out `ok`. Such a pair, and one with counts //! that cannot be counts (negative, or more opposite homozygotes or more //! identical genotypes than sites), is `not_checked` with the reason and a -//! `WARNING:` line, and the step still exits 0. A finite relatedness keeps +//! `WARNING:` line, and the step exits 0, also for a pair declared first or +//! second degree, where a `nan` used to be a mismatch. A finite relatedness keeps //! its verdict. //! //! The case that reads stderr runs `og family` in a child process: this @@ -24,6 +25,9 @@ const PAIRS_HEAD: &str = "#sample_a\tsample_b\trelatedness\tibs0\tibs2\tn\n"; /// Two sequenced partners: declared unrelated, expected relatedness 0. const PARTNERS: &str = "a\tmale\t\t\tb\tyes\tyes\nb\tfemale\t\t\ta\tyes\tyes\n"; +/// A sequenced father and son: declared parent-child, expected 0.5. +const PARENT_CHILD: &str = "f\tmale\t\t\t\tyes\tyes\nk\tmale\tf\t\t\tyes\tyes\n"; + /// Two sequenced first cousins (expected 0.125, third degree): the /// grandparents, their two children and the children's partners are in /// the pedigree, not sequenced. @@ -169,6 +173,23 @@ fn nan_relatedness_of_declared_third_degree_pair_is_not_checked() { ); } +/// A declared first-degree pair with `nan` used to bin as unrelated, a +/// mismatch that stopped the step with exit 1; it is not a measurement +/// either, so it is not_checked and the step goes on. +#[test] +fn nan_relatedness_of_declared_parent_child_is_not_checked() { + let (code, _, row) = check("parent-child-ok", PARENT_CHILD, "f\tk\t0.49\t2\t3000\t5000"); + assert_eq!(code, 0); + assert_eq!(get(&row, "expected_relatedness"), "0.500", "{row:?}"); + assert_eq!(get(&row, "verdict"), "ok", "{row:?}"); + assert_not_checked( + "nan-parent-child", + PARENT_CHILD, + "f\tk\tnan\t2\t3000\t5000", + "somalier gave relatedness 'nan' for this pair: not measured", + ); +} + #[test] fn counts_that_cannot_be_counts_are_not_checked() { let reason = |ibs0: i32, ibs2: i32, n: i32| { From d3a226ec74fe1a049654b90f2fe772461c233f99 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 18 Nov 2026 12:29:18 +0100 Subject: [PATCH 20/22] fix(clinvar): open step 06's normalised ClinVar lock read-only so another user's lock file does not stop a rebuild --- scripts/06-clinvar-screen.sh | 7 ++++-- .../fake-docker/rv-step06-screen-integrity.sh | 22 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/scripts/06-clinvar-screen.sh b/scripts/06-clinvar-screen.sh index 0e150b0dd..0065dbe5f 100755 --- a/scripts/06-clinvar-screen.sh +++ b/scripts/06-clinvar-screen.sh @@ -147,7 +147,10 @@ if norm_stale; then NORM_LOCK="${GENOME_DIR}/clinvar/.norm.lock" if command -v flock >/dev/null 2>&1; then NORM_FLOCK=true - exec 8>>"$NORM_LOCK" + # Opened read-only, as the refresh lock above: flock locks a read-only fd, and a + # .norm.lock another user made (mode 0644) must not stop a rebuild. + [ -e "$NORM_LOCK" ] || : >> "$NORM_LOCK" + exec 8<"$NORM_LOCK" flock -n 8 || { echo "Waiting for the run that builds ${CLINVAR_NORM} ..."; flock 8; } else NORM_FLOCK=false @@ -166,7 +169,7 @@ if norm_stale; then mv "${CLINVAR_NORM}.part.vcf.gz" "$CLINVAR_NORM" fi if $NORM_FLOCK; then - exec 8>&- + exec 8<&- else lock_release "${NORM_LOCK}.d" trap - EXIT diff --git a/tests/fake-docker/rv-step06-screen-integrity.sh b/tests/fake-docker/rv-step06-screen-integrity.sh index 27cde1f32..b95f347f7 100644 --- a/tests/fake-docker/rv-step06-screen-integrity.sh +++ b/tests/fake-docker/rv-step06-screen-integrity.sh @@ -203,6 +203,28 @@ grep -q '^bcftools norm .*clinvar_pathogenic_chr\.norm' "$FAKE_TOOL_LOG" || fail [ ! -e "${C}/.norm.lock.d" ] || fail "without flock the lock folder clinvar/.norm.lock.d was left behind" echo "without flock the step builds under lock_acquire's folder and removes it: PASS" +# --- A lock file the user cannot write, in a clinvar/ folder they can ----------------- +# Another user of a shared genome folder made clinvar/.norm.lock with mode +# 0644. flock locks a read-only fd, so the step must still rebuild a stale +# copy, as it could before the lock existed. A file of the user's own with +# no write bit stands in for it; root opens any file, so this needs an +# ordinary user and flock, as on the CI runner. +if [ "$(id -u)" -eq 0 ] || ! command -v flock >/dev/null 2>&1; then + echo "read-only .norm.lock: not checked as root or without flock" +else + touch -t 202001010000 "$NORM" "${NORM}.tbi" + rm -rf "${GENOME_DIR}/sample1/clinvar" + : > "$FAKE_TOOL_LOG" + : >> "${C}/.norm.lock" + chmod a-w "${C}/.norm.lock" + run_rc readonly-lock timeout 60 "${SCRIPTS}/06-clinvar-screen.sh" sample1 + chmod u+w "${C}/.norm.lock" + expect_rc readonly-lock 0 + grep -q '^bcftools norm .*clinvar_pathogenic_chr\.norm' "$FAKE_TOOL_LOG" || fail "with a read-only .norm.lock the stale normalised ClinVar was not rebuilt" + [ "$(write_id "$NORM")" = "$(write_id "${NORM}.tbi")" ] || fail "with a read-only .norm.lock the published .tbi is from another write than the VCF" + echo "a read-only .norm.lock in a writable clinvar/ still lets the step rebuild: PASS" +fi + # --- A stale normalised ClinVar in a folder the user cannot write ---------------------- # Without flock the lock is a folder made in clinvar/, so the step must stop # with an error there, not wait for a lock it can never take. root writes anywhere, so From e620a1328e0d2df7fa7acfaaa98261279335e04c Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 18 Nov 2026 12:47:56 +0100 Subject: [PATCH 21/22] fix(clinvar): make every step 06 build of the normalised ClinVar take one lock and publish it safely A run with flock and a run without it took different locks and could both rebuild the shared copy. Every build now holds the folder lock, with flock in front where installed. The old VCF is removed before the new index goes in, so a run that stops halfway never leaves an old VCF beside a new .tbi. The flock file is made readable by every user, and an unreadable one stops the step with an error that names it. --- CHANGELOG.md | 4 +- docs/06-clinvar-screen.md | 2 +- scripts/06-clinvar-screen.sh | 45 +++---- .../fake-docker/rv-step06-screen-integrity.sh | 118 +++++++++++++++++- 4 files changed, 144 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ad00814f..d84184265 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - Step 06 (`scripts/06-clinvar-screen.sh`) no longer ships a ClinVar screen that is silently incomplete. - Its three two-command pipelines in the bcftools container (ClinVar's normalisation, the sample's PASS filter and the copy of ClinVar's fields onto the hits) ran under `sh`, which has no `pipefail`. A failed first `bcftools` went unnoticed when the second one exited 0, and the step finished with part of the records. They now run in `bash -o pipefail`, so the step stops with an error and writes no hits file. - - Two samples screened at the same time after a new install or a ClinVar refresh both rebuilt the shared `clinvar/clinvar_pathogenic_chr.norm.vcf.gz` into one `.part` file. One run could fail, or publish a `.tbi` of another write than the VCF. Now one run builds it under the lock `clinvar/.norm.lock` (`flock`, or the folder `clinvar/.norm.lock.d` where `flock` is not installed) while the others wait and then reuse it, and the index goes in place before the VCF. A stale copy in a `clinvar/` folder you cannot write stops the step with an error. - No output file or option changes. The Nextflow module is not affected: it normalises in its own task folder and runs with `pipefail`. + - Two samples screened at the same time after a new install or a ClinVar refresh both rebuilt the shared `clinvar/clinvar_pathogenic_chr.norm.vcf.gz` into one `.part` file. One run could fail, or publish a `.tbi` of another write than the VCF. Now one run builds it while the others wait and then reuse it. Every run that builds holds the folder `clinvar/.norm.lock.d`, with or without `flock`; where `flock` is installed it first takes `flock` on `clinvar/.norm.lock`, so a run with `flock` and a run without it still exclude each other. The old VCF is removed first and the new index goes in place before the new VCF, so a run that stops halfway leaves a copy the next run rebuilds. A stale copy in a `clinvar/` folder you cannot write stops the step with an error, and so does a `clinvar/.norm.lock` you cannot read; the step makes that file readable by every user. + No output file or option changes. The Nextflow module's code does not change. It reuses the copy step 06 publishes when that VCF and its `.tbi` sit beside the source and the VCF is newer; otherwise it normalises in its own task folder, with `pipefail`. - A new page, [User journeys](docs/user-journeys.md), records where a new user gets stuck on each surface: the docs, the demo, the shell and the local site. It gives the outcome of each journey in the first test-as-user run (2026-11-17, commit `65095e97`), the fix for each finding, and what the run could not reach. A fix that lands has a journey to walk again. - VCF intake now refuses a record with GT after another FORMAT key, such as `DP:GT:GQ`, before a reader can mistake depth for the genotype. The check exits 2 and names the sample, file, first offending position and FORMAT field. It tells users to reorder the keys and sample values together or get a corrected VCF. Records without GT remain allowed by this ordering check. Valid input and output formats are unchanged. Part of #411 (clin-37). - The `archaic-same-person` job of `public-evidence.yml` puts step 43's Neanderthal totals beside Chen et al.'s released calls with `og archaic compare` and `og archaic genomes`, in Rust. They replace `bin/archaic_compare.py`, which is gone with its test, `tests/test_archaic_compare.py`; those checks are now cargo tests in og-ancestry. The tables are the same, byte for byte, on 20 parity cases and on the inputs of the last green run of the job (NA12878, chromosomes 19 to 22). No step of the pipeline runs them, and no option or output changes. What changes on purpose: a missing file, an unreadable number or an empty spans file stops with an `ERROR:` line and exit 1 instead of a Python traceback; an interval with a negative start or an end before its start stops the command, where the Python counted a negative length; and a `--mine` value without `=` is a usage error (exit 2). diff --git a/docs/06-clinvar-screen.md b/docs/06-clinvar-screen.md index 091105cf3..408a289e6 100644 --- a/docs/06-clinvar-screen.md +++ b/docs/06-clinvar-screen.md @@ -49,7 +49,7 @@ In the Nextflow pipeline the same floor is `ext.min_gq` and `ext.min_dp` of `CLI ### What the Script Does 1. Lists the contigs each file holds records on (`bcftools index -s`) and keeps those the reference has too: ClinVar's contigs that are in the reference, and the sample's contigs among those. `bcftools norm` stops at the first record whose contig the reference lacks, such as a provider's unplaced scaffold or one of the `NT_` contigs the full ClinVar file has, and a record on a contig the other file lacks cannot match anyway. The step prints how many of the sample's records it leaves out and on which contigs. When no contig is shared it stops with an error instead of reporting zero hits, and says which file is named the other way (`1, 2, ...` against `chr1, chr2, ...`). -2. Builds `clinvar/clinvar_pathogenic_chr.norm.vcf.gz` beside the ClinVar file the first time, and again whenever the source file is newer: ClinVar's records on the reference's contigs, multiallelic records split (`bcftools norm -m -any`) and indels left-aligned against the reference. Every sample then reuses it. Two samples screened at the same time build it once: the first takes the lock `clinvar/.norm.lock`, the others wait, then find it built. The lock is `flock`'s when `flock` is installed; without it (macOS) it is the folder `clinvar/.norm.lock.d` holding the process id. The build goes to a `.part` file, and its index goes in place before the VCF, so the VCF and its `.tbi` always come from the same build. When the copy must be rebuilt and `clinvar/` is not writable by you, the step stops with an error. The Nextflow module reuses this file when it sits beside the `--clinvar` file and is newer; otherwise it normalises ClinVar inside the task. +2. Builds `clinvar/clinvar_pathogenic_chr.norm.vcf.gz` beside the ClinVar file the first time, and again whenever the source file is newer: ClinVar's records on the reference's contigs, multiallelic records split (`bcftools norm -m -any`) and indels left-aligned against the reference. Every sample then reuses it. It is rebuilt as well when its `.tbi` is missing. Two samples screened at the same time build it once: the first takes the lock, the others wait, then find it built. The lock is the folder `clinvar/.norm.lock.d` holding the process id, which every run takes, with or without `flock`. Where `flock` is installed, a run first takes `flock` on `clinvar/.norm.lock`, so only one run at a time makes the folder. The build goes to a `.part` file. The old VCF is removed first, and the new index goes in place before the new VCF, so a VCF newer than its source has the `.tbi` of the same build beside it, even after a run that stopped halfway. When the copy must be rebuilt and `clinvar/` is not writable by you, the step stops with an error. It also stops when you cannot read `clinvar/.norm.lock` (another user's file); the step makes that file readable by everyone. The Nextflow module reuses this file when it sits beside the `--clinvar` file with its `.tbi` and is newer than it; otherwise, a `.tbi` missing included, it normalises ClinVar inside the task. 3. Filters the sample VCF to PASS records. If the VCF has no PASS record at all (callers that leave FILTER as `.`), it uses `-f .,PASS` instead and prints a notice saying so. If no record is left, the step stops with an error. 4. Splits and left-aligns the sample the same way. `bcftools isec` matches only identical REF/ALT, so a pathogenic allele inside a multiallelic record (genotype `1/2`) is found only after this split. The split writes the other alt allele as missing, not as the reference (`bcftools norm --multi-overlaps .`): a `1/2` record gives `1/.` and `./1`, where the default would give `1/0` and `0/1`. Both copies of such a call carry an alt allele, so neither part may read as het. A poly-T tract is where this shows most: a call of 5 T on one copy and 9 T on the other, against a reference of 7, is one `1/2` record, and its 5 T allele is a ClinVar hit. The reports word such a hit "one copy, other copy not called ref (1/.)", never het; the ACMG and carrier lists count it as one copy of two. This step does not size the tract: a short-read call of a homopolymer length can be off by one or two bases, so check the reads. A sample record whose REF is not the reference's base at that position is kept as written (`bcftools norm -c w`), with a `REF_MISMATCH` warning in the log. Such a record cannot match a ClinVar allele, so the summary of item 11 counts these records, and prints a notice to check that the VCF was called on this reference when there are any. 5. Builds the combined alleles of neighbouring records. A caller can write one complex allele, a delins, as two records a few bases apart on the same haplotype, for example a deletion and an insertion. Matched one by one, the parts can hit two single-submitter ClinVar records and miss the better reviewed record ClinVar holds for the whole allele. `bcftools norm -m +any` joins only records at the same position, so it cannot build that allele. `og clinvar join-adjacent` pairs two records when the second starts at most 10 bases after the first ends (records that overlap are never paired) and their genotypes allow both ALT alleles on one haplotype: phased with the same genotype in the same phase set (`FORMAT/PS`), or unphased with the same genotype (`0/1` and `1/0` count as the same het call; an unphased call with a missing allele such as `./1` is never paired). A phased call with a missing allele, such as `1|.`, has its ALT on a known haplotype: it pairs with a record in the same phase set whose ALT sits on that haplotype (`1|0` or `1|.`), and the combined record keeps `.` on the haplotype where either part is missing. That `.` means the copy was not called there; when the part comes from a split multiallelic record, it holds the other alt allele. Two unphased het calls can also sit on different haplotypes, and the genotype cannot tell which, so the notes of a combined allele and its parts built from them end with "unphased, so the parts may sit on different haplotypes". The combined record runs from the first part's start to the second part's end; `og clinvar join-adjacent` takes the bases between the parts from the reference (an uncompressed FASTA with its `.fai`), and `bcftools norm` trims and left-aligns it the way ClinVar was. The gene is not checked here, because the sample VCF has no gene annotation yet: a combined allele only counts when ClinVar holds a record for it. The step prints how many combined alleles it built. Only pairs are built: an allele split into three or more records is matched by its pairs only. diff --git a/scripts/06-clinvar-screen.sh b/scripts/06-clinvar-screen.sh index 0065dbe5f..b22d36d10 100755 --- a/scripts/06-clinvar-screen.sh +++ b/scripts/06-clinvar-screen.sh @@ -129,13 +129,15 @@ SAMPLE_DROPPED=$(comm -23 <(printf '%s\n' "$SAMPLE_CONTIGS") <(printf '%s\n' "$S # source is newer. isec matches on identical REF/ALT, so both sides must be split # and left-aligned the same way. # Every sample shares the normalised copy, so one run builds it while the others -# wait on clinvar/.norm.lock, then test it again and find it built. The lock is -# flock's when flock is installed: the kernel drops it when the run ends, and a -# mkdir lock is not exclusive with the uutils mkdir of Ubuntu 26.04 (two runs -# that make the folder at once can both succeed). Without flock (macOS) it is -# lock_acquire's folder. The build goes to a .part file only the lock holder -# writes; the index goes in place before the VCF, so a VCF that is newer than -# the source always has the index of the same write beside it. +# wait, then test it again and find it built. Every run that builds takes the same +# lock, lock_acquire's folder clinvar/.norm.lock.d, whether flock is on its PATH or +# not. Where flock is installed, a run first takes flock on clinvar/.norm.lock, so +# only one run at a time tries to make the folder: a mkdir lock is not exclusive +# with the uutils mkdir of Ubuntu 26.04 (two runs that make the folder at once can +# both succeed). The build goes to a .part file only the lock holder writes. The +# old VCF is removed first and the new index goes in place before the new VCF, so +# a VCF that is newer than the source always has the index of the same write +# beside it, even when a run stops halfway. # Each pipeline below runs in bash with pipefail: under sh a failed first # bcftools would go unnoticed when the second one exits 0. norm_stale() { [ ! -f "$CLINVAR_NORM" ] || [ ! -f "${CLINVAR_NORM}.tbi" ] || [ "$CLINVAR" -nt "$CLINVAR_NORM" ]; } @@ -146,17 +148,20 @@ if norm_stale; then fi NORM_LOCK="${GENOME_DIR}/clinvar/.norm.lock" if command -v flock >/dev/null 2>&1; then - NORM_FLOCK=true - # Opened read-only, as the refresh lock above: flock locks a read-only fd, and a - # .norm.lock another user made (mode 0644) must not stop a rebuild. - [ -e "$NORM_LOCK" ] || : >> "$NORM_LOCK" + # Made readable by everyone (the library's umask is 077) and opened read-only, as + # the refresh lock above: flock locks a read-only fd, so a .norm.lock another + # user made does not stop a rebuild. + [ -e "$NORM_LOCK" ] || (umask 022; : >> "$NORM_LOCK") + if [ ! -r "$NORM_LOCK" ]; then + echo "ERROR: ${CLINVAR_NORM} must be rebuilt, but you cannot read its lock ${NORM_LOCK}." >&2 + echo " Its owner can let every user take it: chmod a+r ${NORM_LOCK}" >&2 + exit 1 + fi exec 8<"$NORM_LOCK" flock -n 8 || { echo "Waiting for the run that builds ${CLINVAR_NORM} ..."; flock 8; } - else - NORM_FLOCK=false - lock_acquire "${NORM_LOCK}.d" - trap 'lock_release "${NORM_LOCK}.d"' EXIT fi + lock_acquire "${NORM_LOCK}.d" + trap 'lock_release "${NORM_LOCK}.d"' EXIT if norm_stale; then echo "Normalising ClinVar into ${CLINVAR_NORM} ..." rm -f "${CLINVAR_NORM}.part.vcf.gz" "${CLINVAR_NORM}.part.vcf.gz.tbi" @@ -165,15 +170,13 @@ if norm_stale; then bash -o pipefail -c "set -e; bcftools view -T '$(cpath "$CLINVAR_TARGETS")' -Ou '$(cpath "$CLINVAR")' \ | bcftools norm -m -any -c w -f '$(cpath "$REF")' -Oz -o '$(cpath "${CLINVAR_NORM}.part.vcf.gz")' -" bcftools_run --rw "$(dirname "$CLINVAR_NORM")" bcftools index -t "$(cpath "${CLINVAR_NORM}.part.vcf.gz")" + rm -f "$CLINVAR_NORM" mv "${CLINVAR_NORM}.part.vcf.gz.tbi" "${CLINVAR_NORM}.tbi" mv "${CLINVAR_NORM}.part.vcf.gz" "$CLINVAR_NORM" fi - if $NORM_FLOCK; then - exec 8<&- - else - lock_release "${NORM_LOCK}.d" - trap - EXIT - fi + lock_release "${NORM_LOCK}.d" + trap - EXIT + exec 8<&- fi # Step 2: filter the sample. Callers that never write PASS (FILTER '.') would lose diff --git a/tests/fake-docker/rv-step06-screen-integrity.sh b/tests/fake-docker/rv-step06-screen-integrity.sh index b95f347f7..723ce7fc9 100644 --- a/tests/fake-docker/rv-step06-screen-integrity.sh +++ b/tests/fake-docker/rv-step06-screen-integrity.sh @@ -11,7 +11,13 @@ # .tbi come from that one write (with flock, which every Linux runner # has; a mkdir lock is not exclusive with the uutils mkdir of Ubuntu # 26.04); -# - without flock the step takes lock_acquire's folder and removes it; +# - every build holds lock_acquire's folder clinvar/.norm.lock.d, and flock +# on clinvar/.norm.lock too where flock is installed, so a run with flock +# and a run without it exclude each other; the folder is removed after; +# - a build that stops between its two moves leaves a copy the next run +# rebuilds, never an old VCF beside a new .tbi; +# - a .norm.lock the user can read but not write does not stop a rebuild, +# and one they cannot read stops it with an error that names the file; # - a stale copy in a clinvar/ folder the user cannot write stops the step. # The docker run hook runs each container command for real, with /genome # mapped to GENOME_DIR and a fake bcftools and og first on PATH. The fake @@ -37,6 +43,9 @@ export FAKE_TOOL_LOG="${CASE_WORK}/tools.log" # missing input. FAKE_FAIL (a regex over the arguments) makes the matching # call write a header and exit 1. FAKE_NORM_SLEEP slows the norm that # writes the normalised ClinVar, to hold two runs in that window together. +# That norm also fails unless the run holds the build lock: the folder +# clinvar/.norm.lock.d with its pid file, and, when flock is on PATH, the +# flock on clinvar/.norm.lock (a second open of the file cannot take it). cat > "${TOOLS}/bcftools" <<'TOOL' #!/usr/bin/env bash set -uo pipefail @@ -76,6 +85,11 @@ case "$sub" in query) [ -z "$out" ] || : > "$out"; exit 0 ;; esac if [ "$sub" = norm ] && [[ "$out" == *clinvar_pathogenic_chr.norm* ]]; then + lock="${GENOME_DIR}/clinvar/.norm.lock" + [ -f "${lock}.d/pid" ] || { echo "fake bcftools: building without the folder lock ${lock}.d" >&2; exit 1; } + if command -v flock >/dev/null 2>&1 && flock -n "$lock" true; then + echo "fake bcftools: building without the flock on ${lock}" >&2; exit 1 + fi /bin/sleep "${FAKE_NORM_SLEEP:-0}" fi body=$(printf '##fileformat=VCFv4.2\n##write=%s.%s.%s\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n' "$sub" "$$" "$RANDOM") @@ -127,6 +141,20 @@ chmod +x "${CASE_WORK}/hook-run" export FAKE_DOCKER_RUN_HOOK="${CASE_WORK}/hook-run" FAKE_TOOLS="$TOOLS" write_id() { grep -m1 '^##write=' "$1" 2>/dev/null || echo none; } +# wait_for FILE: wait up to 30 s for FILE to exist. +wait_for() { + local _ + for _ in $(seq 60); do [ -e "$1" ] && return 0; /bin/sleep 0.5; done + return 1 +} +# stale_norm: a normalised ClinVar older than its source, and no screen yet. +stale_norm() { + printf '##write=old\n' > "$NORM" + printf '##write=old\n' > "${NORM}.tbi" + touch -t 202001010000 "$NORM" "${NORM}.tbi" + rm -rf "${GENOME_DIR}"/sample[12]/clinvar + : > "$FAKE_TOOL_LOG" +} # --- A full screen with the fake tools succeeds ------------------------------------- # The positive control: the cases below fail only because of the failure they plant. @@ -203,6 +231,74 @@ grep -q '^bcftools norm .*clinvar_pathogenic_chr\.norm' "$FAKE_TOOL_LOG" || fail [ ! -e "${C}/.norm.lock.d" ] || fail "without flock the lock folder clinvar/.norm.lock.d was left behind" echo "without flock the step builds under lock_acquire's folder and removes it: PASS" +# --- A run with flock and a run without it, one after the other starts ---------------- +# The second run starts once the first holds the folder lock, so the two never +# make the folder at the same moment (that would test the runner's mkdir). +# Either order must build once. Needs flock, as on the CI runner. +if ! command -v flock >/dev/null 2>&1; then + echo "flock and no-flock runs together: not checked without flock" +else + export FAKE_NORM_SLEEP=3 + for order in no-flock-first flock-first; do + stale_norm + rc1=0 rc2=0 + if [ "$order" = no-flock-first ]; then + ( hide_commands flock; exec "${SCRIPTS}/06-clinvar-screen.sh" sample1 ) > "${CASE_WORK}/${order}-1.out" 2>&1 & + else + "${SCRIPTS}/06-clinvar-screen.sh" sample1 > "${CASE_WORK}/${order}-1.out" 2>&1 & + fi + p1=$! + held=true + wait_for "${C}/.norm.lock.d/pid" || held=false + if [ "$order" = no-flock-first ]; then + "${SCRIPTS}/06-clinvar-screen.sh" sample2 > "${CASE_WORK}/${order}-2.out" 2>&1 & + else + ( hide_commands flock; exec "${SCRIPTS}/06-clinvar-screen.sh" sample2 ) > "${CASE_WORK}/${order}-2.out" 2>&1 & + fi + p2=$! + wait "$p1" || rc1=$? + wait "$p2" || rc2=$? + cat "${CASE_WORK}/${order}-1.out" "${CASE_WORK}/${order}-2.out" + $held || fail "${order}: the first run never took the folder lock clinvar/.norm.lock.d" + [ "$rc1" -eq 0 ] && [ "$rc2" -eq 0 ] || fail "${order}: the two screens exited ${rc1} and ${rc2}, want 0 and 0" + builds=$(grep -c '^bcftools norm .*clinvar_pathogenic_chr\.norm' "$FAKE_TOOL_LOG" || true) + [ "$builds" -eq 1 ] || fail "${order}: a run with flock and one without normalised ClinVar ${builds} times, want once" + [ "$(write_id "$NORM")" = "$(write_id "${NORM}.tbi")" ] || fail "${order}: the published .tbi is from another write than the VCF" + [ ! -e "${C}/.norm.lock.d" ] || fail "${order}: the lock folder clinvar/.norm.lock.d was left behind" + echo "${order}: a run with flock and one without build the normalised ClinVar once: PASS" + done + unset FAKE_NORM_SLEEP +fi + +# --- A build that stops between its two moves ---------------------------------------- +# A VCF newer than its source but without its .tbi must be rebuilt. If the +# run stops after the new .tbi is in place and before the new VCF, the old +# VCF must not stay beside the new index looking current: the next run +# must rebuild and publish a VCF and .tbi of one write. +printf '##write=old\n' > "$NORM" +rm -f "${NORM}.tbi" +touch -t 203001010000 "$NORM" +rm -rf "${GENOME_DIR}/sample1/clinvar" +: > "$FAKE_TOOL_LOG" +mkdir -p "${CASE_WORK}/stop-mv" +cat > "${CASE_WORK}/stop-mv/mv" <<'TOOL' +#!/usr/bin/env bash +# Stops like a killed run on the move that publishes the normalised VCF. +case "${*: -1}" in *clinvar_pathogenic_chr.norm.vcf.gz) echo "fake mv: stopped before ${*: -1}" >&2; exit 1 ;; esac +PATH=${PATH#*:} exec mv "$@" +TOOL +chmod +x "${CASE_WORK}/stop-mv/mv" +PATH="${CASE_WORK}/stop-mv:${PATH}" run_rc stop-mv "${SCRIPTS}/06-clinvar-screen.sh" sample1 +grep -q 'fake mv: stopped' "${CASE_WORK}/stop-mv.out" || fail "stop-mv: the planted stop never ran" +[ "$RC" -ne 0 ] || fail "stop-mv: step 06 exited 0 after its move failed" +rm -rf "${GENOME_DIR}/sample1/clinvar" "${C}/.norm.lock.d" +: > "$FAKE_TOOL_LOG" +run_rc after-stop "${SCRIPTS}/06-clinvar-screen.sh" sample1 +expect_rc after-stop 0 +grep -q '^bcftools norm .*clinvar_pathogenic_chr\.norm' "$FAKE_TOOL_LOG" || fail "after a build stopped halfway the next run did not rebuild the normalised ClinVar" +[ "$(write_id "$NORM")" = "$(write_id "${NORM}.tbi")" ] || fail "after a build stopped halfway the next run used a .tbi ($(write_id "${NORM}.tbi")) of another write than the VCF ($(write_id "$NORM"))" +echo "a build that stops between its moves leaves a copy the next run rebuilds: PASS" + # --- A lock file the user cannot write, in a clinvar/ folder they can ----------------- # Another user of a shared genome folder made clinvar/.norm.lock with mode # 0644. flock locks a read-only fd, so the step must still rebuild a stale @@ -223,6 +319,26 @@ else grep -q '^bcftools norm .*clinvar_pathogenic_chr\.norm' "$FAKE_TOOL_LOG" || fail "with a read-only .norm.lock the stale normalised ClinVar was not rebuilt" [ "$(write_id "$NORM")" = "$(write_id "${NORM}.tbi")" ] || fail "with a read-only .norm.lock the published .tbi is from another write than the VCF" echo "a read-only .norm.lock in a writable clinvar/ still lets the step rebuild: PASS" + + # The step makes the lock readable by everyone, though the library's umask is 077. + rm -f "${C}/.norm.lock" + touch -t 202001010000 "$NORM" "${NORM}.tbi" + rm -rf "${GENOME_DIR}/sample1/clinvar" + run_rc new-lock "${SCRIPTS}/06-clinvar-screen.sh" sample1 + expect_rc new-lock 0 + [ -n "$(find "${C}/.norm.lock" -perm -044)" ] || fail "step 06 made clinvar/.norm.lock without read access for others: $(ls -l "${C}/.norm.lock")" + echo "step 06 makes clinvar/.norm.lock readable by every user: PASS" + + # A lock the user cannot read: an error that names it, not a bare shell error. + touch -t 202001010000 "$NORM" "${NORM}.tbi" + rm -rf "${GENOME_DIR}/sample1/clinvar" + chmod a-r "${C}/.norm.lock" + run_rc unreadable-lock timeout 60 "${SCRIPTS}/06-clinvar-screen.sh" sample1 + chmod u+r "${C}/.norm.lock" + [ "$RC" -ne 0 ] || fail "step 06 exited 0 with a stale copy and a lock it cannot read" + output_has unreadable-lock 'cannot read its lock' + output_has unreadable-lock 'chmod a\+r' + echo "a .norm.lock the user cannot read stops the step with an error that names it: PASS" fi # --- A stale normalised ClinVar in a folder the user cannot write ---------------------- From 7872c2715df32a9a5b569f5f03774469b458e94c Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:04:52 +0100 Subject: [PATCH 22/22] fix(clinvar): do not stop step 06 when another user creates its lock file first --- scripts/06-clinvar-screen.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/06-clinvar-screen.sh b/scripts/06-clinvar-screen.sh index b22d36d10..df812e75a 100755 --- a/scripts/06-clinvar-screen.sh +++ b/scripts/06-clinvar-screen.sh @@ -150,8 +150,8 @@ if norm_stale; then if command -v flock >/dev/null 2>&1; then # Made readable by everyone (the library's umask is 077) and opened read-only, as # the refresh lock above: flock locks a read-only fd, so a .norm.lock another - # user made does not stop a rebuild. - [ -e "$NORM_LOCK" ] || (umask 022; : >> "$NORM_LOCK") + # user made does not stop a rebuild, even one made between the test and the append. + [ -e "$NORM_LOCK" ] || (umask 022; : >> "$NORM_LOCK") 2>/dev/null || true if [ ! -r "$NORM_LOCK" ]; then echo "ERROR: ${CLINVAR_NORM} must be rebuilt, but you cannot read its lock ${NORM_LOCK}." >&2 echo " Its owner can let every user take it: chmod a+r ${NORM_LOCK}" >&2