diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index d203d4668..d26c80bfa 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -383,6 +383,24 @@ jobs: grep -E "^test ${test} .*FAILED" "$log" || { cat "$log"; echo "ERROR: ${fault} did not fail ${test}"; exit 1; } echo "Control: ${fault} failed ${test}, as it must." done + # The reference-block labels of the multi-SNP rules are written in the + # test, not read from the rules table: with WRONG_REFERENCE planted as + # the label of APOL1 G0/G0 and CAD9p21 nonrisk/nonrisk, exactly those + # two tests must fail. + - name: Control (a changed reference-block label fails its independent expectation) + run: | + log="${RUNNER_TEMP}/wrong-reference.log" + rc=0 + cargo test --locked --no-fail-fast -p og-steps --features fault-wrong-reference --test genome_traits \ + > "$log" 2>&1 || rc=$? + cat "$log" + [ "$rc" -ne 0 ] || { echo "ERROR: the planted WRONG_REFERENCE labels passed"; exit 1; } + for test in reference_blocks_apol1 reference_blocks_cad9p21; do + grep -E "^test ${test} \.\.\. FAILED$" "$log" || { echo "ERROR: ${test} did not catch WRONG_REFERENCE"; exit 1; } + done + grep -E 'test result: FAILED\. [0-9]+ passed; 2 failed;' "$log" \ + || { echo "ERROR: a test other than the two planted ones failed"; exit 1; } + echo "Control: WRONG_REFERENCE failed reference_blocks_apol1 and reference_blocks_cad9p21." - name: Traits controls (diplotype order, REF check and missing phase rows) env: CARGO_BUILD_JOBS: 3 @@ -755,43 +773,6 @@ jobs: echo "${#tests[@]} tests carry the og-leg marker" og_leg "${tests[@]}" - - name: Traits independent expectations reject changed reference labels and marker metadata - working-directory: . - env: - OG: ${{ github.workspace }}/bin/og - OG_REQUIRED: 1 - run: | - python3 - <<'PY' - import contextlib, io, pathlib, runpy, tempfile - test = runpy.run_path("tests/test_traits_report.py")["main"].__globals__ - original = pathlib.Path(test["RULES"]).read_text() - with tempfile.TemporaryDirectory() as work: - for rid, key, label in [("APOL1", "G0/G0", "G0/G0"), ("CAD9p21", "nonrisk/nonrisk", "no risk copy")]: - path = pathlib.Path(work) / "rules.tsv" - old = f"combination\t{rid}\t{key}\t{label}\t" - assert old in original - path.write_text(original.replace(old, f"combination\t{rid}\t{key}\tWRONG_REFERENCE\t")) - test["RULES"], test["FAILS"] = str(path), 0 - log = io.StringIO() - with contextlib.redirect_stdout(log): - rc = test["main"]() - print(log.getvalue()) - assert rc == 1 and test["FAILS"] == 1 - assert f"[FAIL] {rid}, reference blocks only: {label}" in log.getvalue() - print(f"Control: {rid} WRONG_REFERENCE failed its independent expectation") - marker = test["MARKERS"]["SMN1_27134"] - rs, chrom, pos, ref, alt = marker["dbsnp"] - marker["dbsnp"] = rs, chrom, pos + 1, ref, alt - test["FAILS"] = 0 - log = io.StringIO() - with contextlib.redirect_stdout(log): - test["marker_checks"]() - print(log.getvalue()) - assert test["FAILS"] == 1 - assert "[FAIL] marker SMN1_27134: the pinned dbSNP record agrees" in log.getvalue() - print("Control: retained SMN1 off-copy metadata failed its independent expectation") - PY - # The fake-docker job builds no og, so its cn-engine-calls case says it # is skipped there; here bin/og is built and the case must run. - name: The fake-docker case cn-engine-calls (steps 35, 41 and 46 through og cn) with og diff --git a/CHANGELOG.md b/CHANGELOG.md index c9dcf0c9a..72d9c85b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- `og report render` now writes the pharmacogenomics, HLA and mitochondrial sections of both reports: PharmCAT, the CPIC drug-gene recommendations, pypgx with the PGx copy number, CYP2D6 across the callers with step 36's verdict, HLA typing with its derived lines (coeliac DQ types, the DQB1*02 dose, the drug and disease alleles) and the KIR genes, and the mitochondrial variants with the confirmed list, the large deletions and the mtDNA copy number. No step calls the verb yet, so nothing a run writes changes. On 110 summaries (the 99 report-render parity cases and 11 new changes of one summary that reach each state of these sections) its output is byte for byte the Python's for those parts. The mitochondrial card's chart is left out until the charts are ported. The checks of `tests/test_hla_report.py` and `tests/test_mito_extended.py` are cargo tests in `rust/crates/og-render/tests/port_report_pgx.rs`, and both Python files are gone. What og does on purpose that the Python did not: + - A summary without the CYP2D6 block (`"cyp2d6"` missing or `null`) shows no CYP2D6 across the callers. The Python stopped with a KeyError or a TypeError and wrote no report. +- `og report render` writes most of the ancestry chapter and the polygenic score card in Rust: fine-grained ancestry with the painting and its legend, the mitochondrial haplogroup, the Y chromosome (Yleaf, the placement on the current public tree, the Y-STR markers in FamilyTreeDNA's order), archaic and ancient ancestry, and the PRS table with its ancestry and sex notes. Runs of homozygosity are not written yet, so both Rust reports leave them out even when the summary has them. Both reports read as `bin/render_report.py` writes them, except: + - A PRS row whose score, matched or total the summary has as null (a short row of pgsc_calc's table) reads "not available" in both reports, and in the HTML result cell when there is no score either. The Python wrote "None" in the text report and stopped the whole HTML report on that row. + - The painting and the ancient PCA plot are escaped in their `src` attribute. A real picture is base64 and does not change; a value that is not text renders instead of stopping the report. + - A haplogroup's explore address that is not a web address with a host (`http://` or `https://`), such as a `javascript:` one, shows as text in the HTML report. The Python made it a link, which would run it when clicked. Only the two docs pages the collector links get the plain "opens a website" note; any other address gets the note that it carries the haplogroup. + - A painting legend colour that is not `#rrggbb` shows as a fixed grey, so a summary cannot add CSS to the legend. The collector always writes `#rrggbb`. + - The charts are not part of these cards yet; `og report charts` will write them. + The tests of these sections moved from Python to cargo (`rust/crates/og-render/tests/ancestry_*.rs`): `tests/test_ancient_report.py`, `test_archaic_report.py`, `test_haplogroup_explore.py` and `test_ystr_report.py` are gone, and `test_ancestry_fine_report.py` no longer renders the reports. The IBDmix runs of the archaic test need docker and amd64; they run in CI. +- `og report render` now writes five more sections of both reports: the traits panel with its genotype rules, the blood groups, Lp(a), the paralog genes (Parascopy and step 35's depth counts for HBA, CYP21A2, the dosage loci and the copy-number intervals) and the repeat expansions. No step calls the verb yet, so nothing a run writes changes. On 106 summaries (the 99 report-render parity cases before, and 7 new ones that drive each branch of these sections in each state) its output is byte for byte the Python's for the sections it writes. + - The tests of these sections and of the tools behind them are now cargo tests: `tests/test_traits_report.py` in og-steps (`tests/genome_traits.rs`), `tests/test_copy_number.py` and `tests/test_lpa_kiv2.py` in og-cn (`tests/genome_cn.rs`, `genome_lpa.rs`, `genome_markers.rs`), `tests/test_str_ranges.py` in og-setup (`tests/genome_str_ranges.rs`), and the checks that cross the steps, the summary and the report in `rust/og/tests/genome_steps.rs` and `genome_traits_report.rs`. The four Python files are gone, and `tests/test_collect_summary.py` and `tests/test_render_report.py` no longer import `tests/test_traits_report.py`. + - The guard's check that a changed reference-block label in `assets/genotype_rules.tsv` fails its expectation is now a cargo feature, `og-steps/fault-wrong-reference`. The guard runs the tests with it and needs exactly the two planted blocks to fail. +- `og summary collect` refuses a sample folder it cannot list. A mistyped `--sample-dir`, a file, or a folder without read permission read as a folder where no step ran, so the collect wrote a summary.json with every section missing and exited 0, over the good summary when `--out` named it. It now exits 1 with `ERROR: : not a readable folder: ` and writes nothing. An empty folder that exists is still collected, and a step folder inside it that cannot be read still counts as empty. No output file or format changes. - A new page, [User journeys](docs/user-journeys.md), records where a new user gets stuck on each surface: the docs, the demo, the shell and the local site. It gives the outcome of each journey in the first test-as-user run (2026-11-17, commit `65095e97`), the fix for each finding, and what the run could not reach. A fix that lands has a journey to walk again. - VCF intake now refuses a record with GT after another FORMAT key, such as `DP:GT:GQ`, before a reader can mistake depth for the genotype. The check exits 2 and names the sample, file, first offending position and FORMAT field. It tells users to reorder the keys and sample values together or get a corrected VCF. Records without GT remain allowed by this ordering check. Valid input and output formats are unchanged. Part of #411 (clin-37). - The `archaic-same-person` job of `public-evidence.yml` puts step 43's Neanderthal totals beside Chen et al.'s released calls with `og archaic compare` and `og archaic genomes`, in Rust. They replace `bin/archaic_compare.py`, which is gone with its test, `tests/test_archaic_compare.py`; those checks are now cargo tests in og-ancestry. The tables are the same, byte for byte, on 20 parity cases and on the inputs of the last green run of the job (NA12878, chromosomes 19 to 22). No step of the pipeline runs them, and no option or output changes. What changes on purpose: a missing file, an unreadable number or an empty spans file stops with an `ERROR:` line and exit 1 instead of a Python traceback; an interval with a negative start or an end before its start stops the command, where the Python counted a negative length; and a `--mine` value without `=` is a usage error (exit 2). @@ -37,6 +50,9 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - Each report is written under a temporary name of its own and renamed into place. - An `-o` name must end in `.txt` or `.html`, or the verb exits 2 and writes nothing. The Python wrote the HTML report for any name that did not end in `.txt`. - A summary with no section for a step reads that section as not run. The Python stopped with a KeyError when the `variants` section was absent. +- `og report render` writes the clinical chapters of both reports in Rust, one module per section under `rust/crates/og-render/src/sections/`: gene callability, the ClinVar screen (the hits, the call floor, the frequency marks, the cDNA and construct marks, the known deep intronic sites and the cDNA scan), CPSR, the ACMG secondary findings and the clinical filter counts, the carrier list with its two-hit lines, slivar's compound-het phase and the couple section. The bytes are the Python's on every parity case. AnnotSV's rows are in `sections/annotsv.rs`, but the structural variants section that shows them is not switched on yet: half of it is the SV caller support, which another change ports. + - A number in a clinical table cell is escaped in the HTML. The Python put it into the page as it was, so a text value there could break the page. + - `tests/test_acmg_sf_rules.py`, `tests/test_cross_engine_biallelic.py` and `tests/test_callability.py` are gone. Their checks are cargo tests: `rust/crates/og-render/tests/clinical.rs`, with the couple screen checks in `rust/crates/og-couple/tests/port_report_clinical.rs` and the ALT contig refusal in `rust/crates/og-cn/tests/port_report_clinical.rs`. `bin/collect_summary.py` loses `parse_alleles`, `allele_key` and `SEVERITY_KINDS`, which only the ACMG test used. - Step 04c (Sniffles2) now refuses a short-read BAM. Sniffles2 finds no SV in short reads and still exits 0, so the step reported success with an empty VCF. It now reads the mean length of the first 10,000 primary reads with the pinned samtools image and stops with exit 1 below 1,000 bp, naming the long-read BAM to use. Part of #411. - `setup.sh` installs GIAB's GRCh38 tandem-repeat stratification (v3.6, `TANDEM_REPEATS_URL` and `TANDEM_REPEATS_SHA256`) and keeps it unpacked as `reference/GRCh38_AllTandemRepeats.bed`. Step 04c passes it to Sniffles2 as `--tandem-repeats`, which improves breakpoints and merging inside repeats, and says so when it is missing. On a public HG002 HiFi slice of chr20 it found one more benchmark SV (50 of 64 instead of 49) at the same precision. Sniffles' own GRCh38 file comes from PacBio's pbsv under a licence limited to PacBio data, so we use GIAB's. - The step's closing hint no longer says duphold adds a GQ to long-read SVs. Sniffles2 writes its own QUAL and GQ, and step 15 reads Manta's calls only. The long-read guide's duphold row and its uncited accuracy lines are corrected. diff --git a/bin/collect_summary.py b/bin/collect_summary.py index b84a58775..570dc9f01 100755 --- a/bin/collect_summary.py +++ b/bin/collect_summary.py @@ -147,9 +147,6 @@ def collect(sample, sample_dir, declared_sex=None, out=None): "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53"} -SEVERITY_KINDS = ("hypomorph", "pseudodeficiency") - - def open_text(path): if path.endswith((".gz", ".bgz")): return gzip.open(path, "rt", errors="replace") @@ -214,28 +211,3 @@ def get(v, k): freq = "none" if not fields else "unknown" if not afs else "rare" if max(afs) < COMMON_AF else "common" return cls, [{"impact": get(v, "IMPACT"), "gene": get(v, "SYMBOL"), "consequence": get(v, "Consequence").replace("&", ",")} for v in chosen], freq - - -def allele_key(locus): - """'chr6:26092913:G:A' -> ('chr6', 26092913, 'G', 'A').""" - c, pos, ref, alt = locus.split(":") - return (c, int(pos), ref, alt) - - -def parse_alleles(text, kinds=False): - """The variants or severity cell of assets/acmg_sf.tsv and - assets/carrier_genes.tsv: '.' or entries separated by ';', each - '[kind ]chrom:pos:ref:alt name'. {key: {'kind', 'name'}}.""" - out = {} - for e in (text or ".").split(";"): - e = e.strip() - if e in ("", "."): - continue - parts = e.split(" ", 2 if kinds else 1) - if kinds and len(parts) < 2: - raise ValueError(f"severity entry needs a kind and a chrom:pos:ref:alt: {e}") - kind = parts.pop(0) if kinds else None - if kinds and kind not in SEVERITY_KINDS: - raise ValueError(f"severity '{kind}' is not one of {', '.join(SEVERITY_KINDS)}: {e}") - out[allele_key(parts[0])] = {"kind": kind, "name": parts[1] if len(parts) > 1 else parts[0]} - return out diff --git a/bin/hla_alleles.tsv b/bin/hla_alleles.tsv index e8da1c385..076e6c38b 100644 --- a/bin/hla_alleles.tsv +++ b/bin/hla_alleles.tsv @@ -4,7 +4,7 @@ # allele: an allele group at a field boundary: B*27 matches B*27:05:02, B*57:01 matches B*57:01:01. # kind: drug (a drug reaction) or disease (a condition). association: what the study in pmids reports. # pmids: the primary papers, comma-separated PubMed ids. citation: first author, year, journal. -# tests/test_hla_report.py checks the schema. +# rust/crates/og-render/tests/port_report_pgx.rs checks the schema. allele kind association pmids citation A*31:01 drug carbamazepine hypersensitivity reactions in Europeans 21428769 McCormack 2011, N Engl J Med B*15:02 drug carbamazepine Stevens-Johnson syndrome in Han Chinese 15057820 Chung 2004, Nature diff --git a/docs/08-hla-typing.md b/docs/08-hla-typing.md index 5926f8849..029f2e707 100644 --- a/docs/08-hla-typing.md +++ b/docs/08-hla-typing.md @@ -120,7 +120,7 @@ docker run --rm \ ## What the Report Derives -The report (step 24 and the text report) lists each T1K row and then derives three things from it. The code is the HLA section of `og summary collect` (`rust/crates/og-summary/src/collect/group_c/hla.rs`); `tests/test_hla_report.py` checks every rule on synthetic T1K rows. The genotypes are unphased: T1K gives two alleles per gene, not which chromosome each sits on. +The report (step 24 and the text report) lists each T1K row and then derives three things from it. The code is the HLA section of `og summary collect` (`rust/crates/og-summary/src/collect/group_c/hla.rs`); `rust/crates/og-render/tests/port_report_pgx.rs` checks every rule on synthetic T1K rows. The genotypes are unphased: T1K gives two alleles per gene, not which chromosome each sits on. **Coeliac DQ.** From the DQA1 and DQB1 rows: diff --git a/rust/Cargo.lock b/rust/Cargo.lock index ec8e92fcc..8a634cd7f 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -848,8 +848,12 @@ dependencies = [ name = "og-render" version = "0.1.0" dependencies = [ + "flate2", "indexmap", + "og-ancestry", + "og-dev", "og-io", + "og-reports", "og-summary", "pycompat", ] diff --git a/rust/crates/og-cn/tests/data/genome_gold.json b/rust/crates/og-cn/tests/data/genome_gold.json new file mode 100644 index 000000000..4b3b86e4b --- /dev/null +++ b/rust/crates/og-cn/tests/data/genome_gold.json @@ -0,0 +1,466 @@ +{ + "hba": [ + [ + [ + 30, + 30, + 30, + 30, + 0.25 + ], + { + "hba2_depth": 30, + "hba1_depth": 30, + "flank_left_depth": 30, + "flank_right_depth": 30, + "mapq0_share": "0.250", + "estimate": "4.00", + "alpha_globin_genes": "4", + "status": "called", + "message": "four alpha-globin genes (aa/aa): no deletion seen" + } + ], + [ + [ + 15, + 30, + 30, + 30, + 0.25 + ], + { + "hba2_depth": 15, + "hba1_depth": 30, + "flank_left_depth": 30, + "flank_right_depth": 30, + "mapq0_share": "0.250", + "estimate": "3.00", + "alpha_globin_genes": "3", + "status": "called", + "message": "three alpha-globin genes: one gene lost on one chromosome (-a/aa, as -a3.7 or -a4.2), a silent carrier; unconfirmed: this short-read count has been checked on one real genome only (HG002, four genes), so a count under four needs a clinical test before it means anything" + } + ], + [ + [ + 15, + 15, + 30, + 30, + 0.25 + ], + { + "hba2_depth": 15, + "hba1_depth": 15, + "flank_left_depth": 30, + "flank_right_depth": 30, + "mapq0_share": "0.250", + "estimate": "2.00", + "alpha_globin_genes": "2", + "status": "called", + "message": "two alpha-globin genes: alpha-thalassaemia trait, either both genes lost on one chromosome (--/aa, as --SEA; a child with a partner who carries the same can have Hb Bart's) or one on each (-a/-a); depth cannot tell these apart; unconfirmed: this short-read count has been checked on one real genome only (HG002, four genes), so a count under four needs a clinical test before it means anything" + } + ], + [ + [ + 45, + 30, + 30, + 30, + 0.25 + ], + { + "hba2_depth": 45, + "hba1_depth": 30, + "flank_left_depth": 30, + "flank_right_depth": 30, + "mapq0_share": "0.250", + "estimate": "5.00", + "alpha_globin_genes": "5", + "status": "called", + "message": "5 alpha-globin genes: more than four, a triplication (as aaa/aa)" + } + ], + [ + [ + 22.5, + 30, + 30, + 30, + 0.25 + ], + { + "hba2_depth": 22.5, + "hba1_depth": 30, + "flank_left_depth": 30, + "flank_right_depth": 30, + "mapq0_share": "0.250", + "estimate": "3.50", + "alpha_globin_genes": ".", + "status": "not assessed", + "message": "not assessed: the estimate 3.50 lies between 3 and 4 alpha-globin genes" + } + ], + [ + [ + 5, + 5, + 6, + 6, + 0.25 + ], + { + "hba2_depth": 5, + "hba1_depth": 5, + "flank_left_depth": 6, + "flank_right_depth": 6, + "mapq0_share": "0.250", + "estimate": ".", + "alpha_globin_genes": ".", + "status": "not assessed", + "message": "not assessed: flank depth under 8x (6.0x and 6.0x), too low to compare" + } + ], + [ + [ + 30, + 30, + 30, + 15, + 0.25 + ], + { + "hba2_depth": 30, + "hba1_depth": 30, + "flank_left_depth": 30, + "flank_right_depth": 15, + "mapq0_share": "0.250", + "estimate": ".", + "alpha_globin_genes": ".", + "status": "not assessed", + "message": "not assessed: the two flanks disagree (30.0x and 15.0x); a deletion larger than --SEA, or uneven coverage, would skew the count" + } + ], + [ + [ + 15.5, + 25.68, + 30.11, + 31.25, + 0 + ], + { + "hba2_depth": 15.5, + "hba1_depth": 25.68, + "flank_left_depth": 30.11, + "flank_right_depth": 31.25, + "mapq0_share": "0.000", + "estimate": "2.68", + "alpha_globin_genes": ".", + "status": "not assessed", + "message": "not assessed: reads of mapping quality 0 make up 0.0% of the depth over HBA1 and HBA2, so the aligner left out the reads that fit both genes (as novoalign does by default) and the count would come out low; realign with minimap2 (step 02) or BWA-MEM" + } + ], + [ + [ + 30, + 30, + 30, + 30, + 0.01 + ], + { + "hba2_depth": 30, + "hba1_depth": 30, + "flank_left_depth": 30, + "flank_right_depth": 30, + "mapq0_share": "0.010", + "estimate": "4.00", + "alpha_globin_genes": ".", + "status": "not assessed", + "message": "not assessed: reads of mapping quality 0 make up 1.0% of the depth over HBA1 and HBA2, so the aligner left out the reads that fit both genes (as novoalign does by default) and the count would come out low; realign with minimap2 (step 02) or BWA-MEM" + } + ], + [ + [ + 0, + 0, + 30, + 30, + 0 + ], + { + "hba2_depth": 0, + "hba1_depth": 0, + "flank_left_depth": 30, + "flank_right_depth": 30, + "mapq0_share": "0.000", + "estimate": "0.00", + "alpha_globin_genes": "0", + "status": "called", + "message": "no alpha-globin gene: the pattern of Hb Bart's hydrops fetalis (--/--); unconfirmed: this short-read count has been checked on one real genome only (HG002, four genes), so a count under four needs a clinical test before it means anything" + } + ], + [ + [ + 7.4, + 7.6, + 30, + 30, + 0.25 + ], + { + "hba2_depth": 7.4, + "hba1_depth": 7.6, + "flank_left_depth": 30, + "flank_right_depth": 30, + "mapq0_share": "0.250", + "estimate": "1.00", + "alpha_globin_genes": "1", + "status": "called", + "message": "one alpha-globin gene: the pattern of HbH disease (--/-a); unconfirmed: this short-read count has been checked on one real genome only (HG002, four genes), so a count under four needs a clinical test before it means anything" + } + ], + [ + [ + 30.2, + 29.1, + 29.4, + 31.9, + 0.2 + ], + { + "hba2_depth": 30.2, + "hba1_depth": 29.1, + "flank_left_depth": 29.4, + "flank_right_depth": 31.9, + "mapq0_share": "0.200", + "estimate": "3.87", + "alpha_globin_genes": "4", + "status": "called", + "message": "four alpha-globin genes (aa/aa): no deletion seen" + } + ] + ], + "lpa": [ + [ + [ + 12, + 30, + 0, + 0, + 0.9 + ], + { + "status": "estimated", + "kiv2_copies": "12.0", + "ci95_low": "12.0", + "ci95_high": "12.0", + "class": "low", + "kiv2_depth": "30.00", + "flank_depth": "30.00", + "mapq0_share": "0.900", + "message": "about 12 KIV-2 repeats in total (95% interval 12.0 to 12.0): fewer KIV-2 repeats than most people (the whole 95% interval is under 30; about 70% of the 1000 Genomes samples had 30 to 45): a low count goes with small apo(a) isoforms and a higher Lp(a) level" + } + ], + [ + [ + 20, + 30, + 0.1, + 0.15, + 0.9 + ], + { + "status": "estimated", + "kiv2_copies": "19.8", + "ci95_low": "17.9", + "ci95_high": "22.0", + "class": "low", + "kiv2_depth": "47.45", + "flank_depth": "30.43", + "mapq0_share": "0.900", + "message": "about 20 KIV-2 repeats in total (95% interval 17.9 to 22.0): fewer KIV-2 repeats than most people (the whole 95% interval is under 30; about 70% of the 1000 Genomes samples had 30 to 45): a low count goes with small apo(a) isoforms and a higher Lp(a) level" + } + ], + [ + [ + 29, + 30, + 0.03, + 0.03, + 0.9 + ], + { + "status": "estimated", + "kiv2_copies": "29.0", + "ci95_low": "28.3", + "ci95_high": "29.7", + "class": "low", + "kiv2_depth": "66.56", + "flank_depth": "30.09", + "mapq0_share": "0.900", + "message": "about 29 KIV-2 repeats in total (95% interval 28.3 to 29.7): fewer KIV-2 repeats than most people (the whole 95% interval is under 30; about 70% of the 1000 Genomes samples had 30 to 45): a low count goes with small apo(a) isoforms and a higher Lp(a) level" + } + ], + [ + [ + 29, + 30, + 0.25, + 0.25, + 0.9 + ], + { + "status": "estimated", + "kiv2_copies": "28.8", + "ci95_low": "23.5", + "ci95_high": "35.1", + "class": "typical", + "kiv2_depth": "67.50", + "flank_depth": "30.72", + "mapq0_share": "0.900", + "message": "about 29 KIV-2 repeats in total (95% interval 23.5 to 35.1): within or overlapping the range most people have (30 to 45 for about 70% of the 1000 Genomes samples)" + } + ], + [ + [ + 47, + 30, + 0.25, + 0.25, + 0.9 + ], + { + "status": "estimated", + "kiv2_copies": "46.6", + "ci95_low": "38.3", + "ci95_high": "56.6", + "class": "typical", + "kiv2_depth": "106.69", + "flank_depth": "30.72", + "mapq0_share": "0.900", + "message": "about 47 KIV-2 repeats in total (95% interval 38.3 to 56.6): within or overlapping the range most people have (30 to 45 for about 70% of the 1000 Genomes samples)" + } + ], + [ + [ + 60, + 30, + 0.03, + 0.03, + 0.9 + ], + { + "status": "estimated", + "kiv2_copies": "59.9", + "ci95_low": "58.6", + "ci95_high": "61.4", + "class": "high", + "kiv2_depth": "133.11", + "flank_depth": "30.09", + "mapq0_share": "0.900", + "message": "about 60 KIV-2 repeats in total (95% interval 58.6 to 61.4): more KIV-2 repeats than most people (the whole 95% interval is over 45): a high count goes with large apo(a) isoforms and, on average, a lower Lp(a) level" + } + ], + [ + [ + 20, + 8, + 0, + 0, + 0.9 + ], + { + "status": "too little depth", + "kiv2_copies": ".", + "ci95_low": ".", + "ci95_high": ".", + "class": ".", + "kiv2_depth": "12.57", + "flank_depth": "8.00", + "mapq0_share": "0.900", + "message": "too little depth: 8.0x over the single-copy LPA exons, under the 10x the count needs" + } + ], + [ + [ + 20, + 30, + 0, + 0, + 0 + ], + { + "status": "not assessed", + "kiv2_copies": ".", + "ci95_low": ".", + "ci95_high": ".", + "class": ".", + "kiv2_depth": "47.14", + "flank_depth": "30.00", + "mapq0_share": "0.000", + "message": "not assessed: reads of mapping quality 0 make up 0.0% of the depth over the KIV-2 exons, so the aligner left out the reads that fit several KIV-2 copies (as novoalign does by default) and the count would come out low; realign with minimap2 (step 02) or BWA-MEM" + } + ], + [ + [ + -1.5, + 30, + 0, + 0, + 0.9 + ], + { + "status": "not assessed", + "kiv2_copies": ".", + "ci95_low": ".", + "ci95_high": ".", + "class": ".", + "kiv2_depth": "1.07", + "flank_depth": "30.00", + "mapq0_share": "0.900", + "message": "not assessed: the KIV-2 exons have 1.1x against 30.0x over the single-copy exons, too little for any repeat count (a deletion or a mapping problem)" + } + ], + [ + [ + 38, + 12, + 0.3, + 0.2, + 0.5 + ], + { + "status": "estimated", + "kiv2_copies": "38.0", + "ci95_low": "31.0", + "ci95_high": "46.5", + "class": "typical", + "kiv2_depth": "34.95", + "flank_depth": "12.23", + "mapq0_share": "0.500", + "message": "about 38 KIV-2 repeats in total (95% interval 31.0 to 46.5): within or overlapping the range most people have (30 to 45 for about 70% of the 1000 Genomes samples)" + } + ], + [ + [ + -2, + 30, + 0, + 0, + 0.9 + ], + { + "status": "not assessed", + "kiv2_copies": ".", + "ci95_low": ".", + "ci95_high": ".", + "class": ".", + "kiv2_depth": "0.00", + "flank_depth": "30.00", + "mapq0_share": "0.000", + "message": "not assessed: reads of mapping quality 0 make up 0.0% of the depth over the KIV-2 exons, so the aligner left out the reads that fit several KIV-2 copies (as novoalign does by default) and the count would come out low; realign with minimap2 (step 02) or BWA-MEM" + } + ] + ] +} diff --git a/rust/crates/og-cn/tests/genome_cn.rs b/rust/crates/og-cn/tests/genome_cn.rs new file mode 100644 index 000000000..9834c24ac --- /dev/null +++ b/rust/crates/og-cn/tests/genome_cn.rs @@ -0,0 +1,1982 @@ +//! `og cn` and `bin/cn_loci.tsv`: one depth-ratio engine for every +//! copy-number locus. These are the library and command-line checks of the +//! old `tests/test_copy_number.py`, by meaning; its checks of step 35's +//! shell blocks, of the Nextflow wiring, of `og dev build cn-intervals`, of +//! the summary and of the couple screen are `rust/og/tests/genome_steps.rs`. +//! +//! 1. the catalog: the gene, dosage and CYP21A2 rows first, then the +//! interval rows; every row is well formed; a broken row is refused, and +//! `og cn bed --catalog` refuses it with the same message; +//! 2. one owner per locus: no row's targets or controls overlap a region of +//! a model the pinned Parascopy data holds, except the pairs EXEMPT +//! lists; planted AMY1C, C4B, GYPA and DEFB4 rows are caught; +//! 3. parity: made-up HBA and LPA depths give, through `og cn call`, the +//! tables the code before the engine wrote (`tests/data/genome_gold.json`, +//! frozen from it); +//! 4. synthetic depths at 0 to 4 copies of a made-up gene are called right; +//! the MAPQ 0 ceiling; a kind=interval row with pooled depths; +//! 5. a control with no read beside covered controls is BROKEN_CONTROL +//! (exit 3 through `call`); no read anywhere is not assessed; +//! 6. the command line: `bed`, `call --table`, the bedcov reader's refusals; +//! 8. the gene-dosage rows (HP, LCE3BC, CCL3L1): 0 to 4 copies; CCL3L1's +//! readings name no disease; a row shifted by 1 kb refuses its fixture; +//! 9. the interval rows: their content, their calls by sex, and +//! `og cn intervals` over one bedcov table; +//! 10. the CYP21A2 row (call psv): its PSV sites, the refused variants, the +//! committed fixtures as the generator here makes them, and the calls. + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::path::PathBuf; + +use indexmap::IndexMap; +use og_cn::catalog::{COLUMNS, CallRule, Catalog, Kind, Locus, PSV_FINDINGS, parse_row}; +use og_cn::engine::{AssessError, Block, Means, Reason, Sex, Status, assess}; +use pycompat::json::{self, Value}; + +const REPO: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../.."); + +fn repo(rel: &str) -> String { + format!("{REPO}/{rel}") +} + +fn catalog_path() -> String { + repo("bin/cn_loci.tsv") +} + +fn cat() -> Catalog { + Catalog::load(&catalog_path()).expect("bin/cn_loci.tsv loads") +} + +fn read(path: &str) -> String { + std::fs::read_to_string(path).unwrap_or_default() +} + +/// A scratch folder of its own, removed when dropped. +struct Scratch(PathBuf); + +impl Scratch { + fn new(name: &str) -> Scratch { + let d = std::env::temp_dir().join(format!("og-genome-cn-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + Scratch(d) + } + + fn path(&self, rel: &str) -> String { + self.0.join(rel).to_string_lossy().into_owned() + } + + fn put(&self, rel: &str, text: &str) -> String { + let p = self.0.join(rel); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(&p, text).unwrap(); + p.to_string_lossy().into_owned() + } +} + +impl Drop for Scratch { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +/// `og cn ARGS`: (exit code, stdout, stderr). +fn og_cn(args: &[&str]) -> (u8, String, String) { + let args: Vec = args.iter().map(|a| (*a).to_owned()).collect(); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = og_cn::cli::main(&args, &mut out, &mut err); + ( + code, + String::from_utf8(out).unwrap(), + String::from_utf8(err).unwrap(), + ) +} + +/// The catalog's rows as written, by locus: column -> cell. +fn raw_rows() -> IndexMap> { + read(&catalog_path()) + .lines() + .filter(|l| !l.starts_with('#') && !l.trim().is_empty()) + .skip(1) + .map(|l| { + let m: HashMap = COLUMNS + .iter() + .map(|c| (*c).to_owned()) + .zip(l.split('\t').map(str::to_owned)) + .collect(); + (m["locus"].clone(), m) + }) + .collect() +} + +/// A made-up bedcov row: one 1 kb gene between two 10 kb flanks, called by +/// rounding, with a MAPQ 0 ceiling of 0.5; `kw` replaces columns. +fn row(kw: &[(&str, &str)]) -> HashMap { + let mut r: HashMap = [ + ("locus", "TESTG"), + ("kind", "gene"), + ("chrom", "chr3"), + ("targets", "TESTG:50000-51000"), + ("controls", "fl_a:30000-40000;fl_b:60000-70000"), + ("details", "."), + ("target_depth", "sum"), + ("control_depth", "flanks"), + ("scale", "2"), + ("offset", "0"), + ("bedcov", ".,-Q 1"), + ("mapq0_min", "."), + ("mapq0_max", "0.5"), + ("min_control_depth", "8"), + ("max_control_ratio", "1.5"), + ("call", "round:0.3"), + ("interval", "."), + ("readings", "0=none|1=one|2=two|more={n} copies"), + ("input", "bedcov"), + ("owner", "cn-depth"), + ("cross_check", "none"), + ("pmids", "."), + ] + .iter() + .map(|(k, v)| ((*k).to_owned(), (*v).to_owned())) + .collect(); + for (k, v) in kw { + r.insert((*k).to_owned(), (*v).to_owned()); + } + r +} + +fn line(r: &HashMap) -> String { + COLUMNS + .iter() + .map(|c| r[*c].as_str()) + .collect::>() + .join("\t") +} + +fn write_catalog(path: &str, rows: &[HashMap]) { + let mut text = COLUMNS.join("\t") + "\n"; + for r in rows { + text += &(line(r) + "\n"); + } + std::fs::write(path, text).unwrap(); +} + +fn parsed(r: &HashMap) -> Locus { + let l = line(r); + let cells: Vec<&str> = l.split('\t').collect(); + parse_row(&cells).unwrap_or_else(|e| panic!("{e}")) +} + +/// `Catalog::load` on these rows: the loaded catalog's size, or its error. +fn load_rows(w: &Scratch, rows: &[HashMap]) -> Result { + let p = w.path("c.tsv"); + write_catalog(&p, rows); + Catalog::load(&p).map(|c| c.loci.len()).map_err(|e| e.0) +} + +fn means(pairs: &[(&str, f64)]) -> Means { + pairs.iter().map(|(k, v)| ((*k).to_owned(), *v)).collect() +} + +/// Python's `round()` of a float to a whole number (ties to even). +fn round(v: f64) -> i64 { + v.round_ties_even() as i64 +} + +/// samtools bedcov over a row's segments at these mean depths: whole sums, +/// or with `exact` the sums themselves, so each mean comes back as given. +fn bedcov(path: &str, row: &Locus, depth: &Means, exact: bool) { + let mut text = String::new(); + for (c, s, e, n) in row.regions() { + let v = depth[n] * (e - s) as f64; + if exact { + text += &format!("{c}\t{s}\t{e}\t{n}\t{v:?}\n"); + } else { + text += &format!("{c}\t{s}\t{e}\t{n}\t{}\n", round(v)); + } + } + std::fs::write(path, text).unwrap(); +} + +/// A `metricvalue` table, in its order; empty when it is not there. +fn table(path: &str) -> Vec<(String, String)> { + read(path) + .lines() + .skip(1) + .filter_map(|l| l.split_once('\t')) + .map(|(k, v)| (k.to_owned(), v.to_owned())) + .collect() +} + +fn get<'a>(t: &'a [(String, String)], k: &str) -> &'a str { + t.iter() + .find(|(m, _)| m == k) + .map_or("", |(_, v)| v.as_str()) +} + +fn status(b: &Block) -> String { + b.status.to_string() +} + +/// The change column of an interval row for a block. +fn change(b: &Block) -> &'static str { + match (b.copies, b.expected) { + (Some(c), Some(e)) if c < e => "deletion", + (Some(c), Some(e)) if c > e => "duplication", + (Some(_), Some(_)) => "no change", + _ => ".", + } +} + +// 1. the catalog + +const DOSAGE: [&str; 3] = ["HP", "LCE3BC", "CCL3L1"]; + +/// `-X` or `-X N` options, one or more, space separated. +fn samtools_options(p: &str) -> bool { + let mut words = p.split(' ').peekable(); + let mut any = false; + while let Some(w) = words.next() { + let b = w.as_bytes(); + if !(b.len() == 2 && b[0] == b'-' && b[1].is_ascii_alphabetic()) { + return false; + } + any = true; + if words + .peek() + .is_some_and(|n| !n.is_empty() && n.bytes().all(|c| c.is_ascii_digit())) + { + words.next(); + } + } + any +} + +/// `none` or `tool:Name[,tool:Name...]`. +fn cross_check_ok(x: &str) -> bool { + x == "none" + || x.split(',').all(|p| { + p.split_once(':').is_some_and(|(a, b)| { + !a.is_empty() + && a.bytes().all(|c| c.is_ascii_lowercase()) + && !b.is_empty() + && b.bytes().all(|c| c.is_ascii_alphanumeric() || c == b'_') + }) + }) +} + +/// `.` or comma-separated numbers. +fn pmids_ok(x: &str, dot: bool) -> bool { + (dot && x == ".") + || x.split(',') + .all(|n| !n.is_empty() && n.bytes().all(|c| c.is_ascii_digit())) +} + +#[test] +fn the_gene_dosage_and_cyp21a2_rows_come_first_then_the_interval_rows() { + let c = cat(); + let ids: Vec<&str> = c.loci.keys().map(String::as_str).collect(); + assert_eq!( + &ids[..7], + [ + "HBA", "LPA_KIV2", "RHD", "HP", "LCE3BC", "CCL3L1", "CYP21A2" + ] + ); + for r in c.loci.values().skip(7) { + assert!( + r.kind == Kind::Interval && r.depth().passes == ["."], + "{}", + r.id + ); + } +} + +#[test] +fn every_catalog_row_is_well_formed() { + let c = cat(); + let raw = raw_rows(); + for (id, r) in &c.loci { + let w = &raw[id]; + assert_eq!(w["owner"], "cn-depth", "{id}"); + assert!( + cross_check_ok(&w["cross_check"]), + "{id}: {}", + w["cross_check"] + ); + assert!(pmids_ok(&w["pmids"], true), "{id}: {}", w["pmids"]); + if w["input"] != "bedcov" { + continue; + } + let passes = &r.depth().passes; + assert!( + passes.iter().all(|p| p == "." || samtools_options(p)), + "{id}: {passes:?}" + ); + if passes.len() > 1 { + let strip = |p: &str| p.replace('.', "").replace("-Q 1", "").trim().to_owned(); + assert_eq!( + strip(&passes[1]), + passes[0].replace('.', "").trim(), + "{id}: a second pass adds -Q 1" + ); + } + for s in r.targets.iter().chain(&r.controls).chain(&r.details) { + assert!( + 0 < s.end - s.start && s.end - s.start < 1_000_000, + "{id}: {}", + s.name + ); + } + for ctl in &r.controls { + assert!( + !r.targets + .iter() + .any(|t| ctl.start < t.end && t.start < ctl.end), + "{id}: control {} overlaps a target", + ctl.name + ); + } + let want: Option> = match &r.call { + CallRule::Class { .. } => Some(["low", "typical", "high"].map(str::to_owned).to_vec()), + CallRule::Cut(cuts) => Some((0..=cuts.len()).map(|i| i.to_string()).collect()), + _ => None, + }; + if let Some(want) = want { + for k in want { + assert!(r.readings.contains_key(&k), "{id}: no reading for {k}"); + } + } + } +} + +/// A broken row: what, the changed columns, a part of the error. +type Broken = ( + &'static str, + Vec<(&'static str, &'static str)>, + &'static str, +); + +/// The broken rows. +fn broken() -> Vec { + vec![ + ("a bad kind", vec![("kind", "exon")], "kind"), + ( + "a segment with no span", + vec![("targets", "TESTG:51000-50000")], + "empty span", + ), + ( + "a class call without an interval", + vec![("call", "class:30,45")], + "needs interval cv", + ), + ( + "a MAPQ 0 guard with one bedcov pass", + vec![("bedcov", ".")], + "two bedcov passes", + ), + ( + "a segment name used twice", + vec![("details", "fl_a:1-2")], + "used twice", + ), + ( + "a NaN call threshold", + vec![("call", "round:nan")], + "is not round", + ), + ( + "an infinite cut threshold", + vec![("call", "cut:0.5,inf")], + "is not round", + ), + ( + "a NaN number column", + vec![("mapq0_max", "nan")], + "not a number", + ), + ] +} + +#[test] +fn broken_rows_are_refused_by_the_reader_and_by_og_cn_bed() { + let w = Scratch::new("broken"); + assert_eq!( + load_rows(&w, &[row(&[])]), + Ok(1), + "the plain made-up row loads" + ); + for (desc, kw, err) in broken() { + let got = load_rows(&w, &[row(&kw)]); + let msg = got.clone().err().unwrap_or_else(|| panic!("{desc} loaded")); + assert!(msg.contains(err), "{desc}: {msg}"); + let (rc, _, stderr) = og_cn(&["bed", "--catalog", &w.path("c.tsv")]); + assert_eq!( + (rc, stderr), + (2, format!("ERROR: {msg}\n")), + "{desc} through og cn bed" + ); + } + let two = row(&[ + ("locus", "TESTH"), + ("targets", "TESTG:80000-81000"), + ("controls", "fl_c:1-2;fl_d:3-4"), + ]); + let got = load_rows(&w, &[row(&[]), two]); + assert!( + got.clone().is_err_and(|e| e.contains("used by two loci")), + "{got:?}" + ); +} + +// 2. one owner per locus + +/// The only rows allowed to overlap a Parascopy model, each for a stated +/// reason: step 41 runs by default and Parascopy is opt-in, so the RhD +/// deletion stays a depth row; RHCE's model holds RHD as a paralog copy and +/// is its cross-check. A cross_check in the catalog does not exempt a row. +const EXEMPT: [(&str, &str); 1] = [("RHD", "RHCE")]; + +type Models = BTreeMap>; + +fn models() -> Models { + let mut out: Models = BTreeMap::new(); + for l in read(&repo("assets/parascopy_models_v1.7.txt")).lines() { + if l.starts_with('#') || l.trim().is_empty() { + continue; + } + let f: Vec<&str> = l.split_whitespace().collect(); + let (c, span) = f[1].split_once(':').unwrap(); + let (s, e) = span.split_once('-').unwrap(); + out.entry(f[0].to_owned()).or_default().push(( + c.to_owned(), + s.parse().unwrap(), + e.parse().unwrap(), + f[2].to_owned(), + )); + } + out +} + +/// [(model, segment)] of a row's targets and controls that overlap any +/// region of a model (1-based, both ends included, against 0-based +/// segments). +fn overlaps(r: &Locus, models: &Models) -> Vec<(String, String)> { + let mut hits = Vec::new(); + for s in r.targets.iter().chain(&r.controls) { + for (name, regs) in models { + if regs + .iter() + .any(|(c, ms, me, _)| *c == r.chrom && s.start < *me && ms - 1 < s.end) + { + hits.push((name.clone(), s.name.clone())); + } + } + } + hits +} + +fn owner_violations(rows: &[&Locus], models: &Models) -> Vec<(String, String, String)> { + rows.iter() + .flat_map(|r| { + overlaps(r, models) + .into_iter() + .map(|(m, s)| (r.id.clone(), m, s)) + }) + .filter(|(l, m, _)| !EXEMPT.contains(&(l.as_str(), m.as_str()))) + .collect() +} + +#[test] +fn the_parascopy_model_list_is_the_pinned_one() { + let m = models(); + assert_eq!(m.len(), 167); + for g in [ + "AMY1C", "C4A", "CFH", "CFHR4", "FCGR3A", "CHRNA7", "APOBEC3A", "GYPA", "RHCE", + ] { + assert!(m.contains_key(g), "{g}"); + } + assert!( + m.values() + .all(|r| r.iter().filter(|x| x.3 == "main").count() == 1 + && r.iter().all(|x| x.3 == "main" || x.3 == "copy")) + ); + assert!(m.values().map(Vec::len).sum::() > 1000); +} + +#[test] +fn no_row_counts_a_gene_a_pinned_parascopy_model_owns() { + let (c, m) = (cat(), models()); + let raw = raw_rows(); + for (id, w) in &raw { + for x in w["cross_check"].split(',') { + if let Some(model) = x.strip_prefix("parascopy:") { + assert!( + m.contains_key(model), + "{id}: cross_check {x} is not a pinned model" + ); + } + } + } + let rows: Vec<&Locus> = c.loci.values().collect(); + assert_eq!(owner_violations(&rows, &m), Vec::new()); + for (l, model) in EXEMPT { + let r = c + .loci + .get(l) + .unwrap_or_else(|| panic!("the exempt row {l} is gone")); + assert!( + overlaps(r, &m).iter().any(|(x, _)| x == model), + "the exemption {l}/{model} is stale" + ); + } + let rhd: HashSet = overlaps(&c.loci["RHD"], &m) + .into_iter() + .map(|(x, _)| x) + .collect(); + assert_eq!( + rhd, + HashSet::from(["RHCE".to_owned()]), + "RHD overlaps RHCE's model only" + ); +} + +#[test] +fn planted_rows_on_a_parascopy_model_are_caught() { + let (c, m) = (cat(), models()); + for (kw, want) in [ + ( + vec![ + ("locus", "AMY1"), + ("chrom", "chr1"), + ("targets", "AMY1C:103745500-103760000"), + ( + "controls", + "amy_l:103000000-103010000;amy_r:104000000-104010000", + ), + ], + ("AMY1", "AMY1C", "AMY1C"), + ), + ( + vec![ + ("locus", "C4B"), + ("chrom", "chr6"), + ("targets", "C4B:32014800-32035400"), + ("controls", "c4_l:31000000-31010000;c4_r:33500000-33510000"), + ], + ("C4B", "C4A", "C4B"), + ), + ( + vec![ + ("locus", "GYPA"), + ("chrom", "chr4"), + ("cross_check", "parascopy:GYPA"), + ("targets", "GYPA_T:144140000-144150000"), + ( + "controls", + "gy_l:140000000-140010000;gy_r:150000000-150010000", + ), + ], + ("GYPA", "GYPA", "GYPA_T"), + ), + // DEFB4 has no row: the beta-defensin repeat lies in SPAG11A's copies. + ( + vec![ + ("locus", "DEFB4"), + ("chrom", "chr8"), + ("targets", "DEFB_UNIT_B:7414854-7430348"), + ("controls", "defb_l:7622000-7662000;defb_r:8250000-8290000"), + ], + ("DEFB4", "SPAG11A", "DEFB_UNIT_B"), + ), + ] { + let planted = parsed(&row(&kw)); + let mut rows: Vec<&Locus> = c.loci.values().collect(); + rows.push(&planted); + let v = owner_violations(&rows, &m); + let want = (want.0.to_owned(), want.1.to_owned(), want.2.to_owned()); + assert!(v.contains(&want), "{want:?} not caught: {v:?}"); + if want.0 != "GYPA" { + assert_eq!(v, vec![want]); + } + } +} + +// 3. parity with the code before the engine + +fn gold() -> Value { + json::loads(&read(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/data/genome_gold.json" + ))) + .unwrap() +} + +fn num(v: &Value) -> f64 { + match v { + Value::Int(i) => *i as f64, + Value::Float(f) => *f, + _ => panic!("{v:?}"), + } +} + +fn arr(v: &Value) -> &[Value] { + match v { + Value::Array(a) => a, + _ => panic!("{v:?}"), + } +} + +/// Every field of the frozen row: numbers as numbers, text as it is. +fn same(got: &[(String, String)], want: &Value) -> Result<(), String> { + let Value::Object(want) = want else { panic!() }; + for (k, v) in want { + let g = get(got, k); + let ok = match v { + Value::Str(s) => g == s, + _ => g.parse::().is_ok_and(|x| (x - num(v)).abs() < 1e-9), + }; + if !ok { + return Err(format!("{k}: got {g:?}, want {v:?}")); + } + } + Ok(()) +} + +#[test] +fn parity_hba_tables_are_the_ones_before_the_engine() { + let (c, w) = (cat(), Scratch::new("gold-hba")); + let (a, q, tab) = (w.path("a.tsv"), w.path("q.tsv"), w.path("t.tsv")); + let hrow = &c.loci["HBA"]; + let g = gold(); + let Value::Object(g) = &g else { panic!() }; + let cases = arr(&g["hba"]); + assert_eq!(cases.len(), 12); + for case in cases { + let (inp, want) = (arr(&arr(case)[0]), &arr(case)[1]); + let [h2, h1, left, right, share] = [0, 1, 2, 3, 4].map(|i| num(&inp[i])); + let m = means(&[ + ("flank_left", left), + ("HBA2", h2), + ("HBA1", h1), + ("flank_right", right), + ]); + let mq: Means = m + .iter() + .map(|(n, v)| { + ( + n.clone(), + if n.starts_with("HBA") { + v * (1.0 - share) + } else { + *v + }, + ) + }) + .collect(); + bedcov(&a, hrow, &m, true); + bedcov(&q, hrow, &mq, true); + let (rc, _, err) = og_cn(&[ + "call", + "--catalog", + &catalog_path(), + "--locus", + "HBA", + "--bedcov", + &a, + "--bedcov-mapq1", + &q, + "--table", + &tab, + ]); + assert_eq!(rc, 0, "{err}"); + if let Err(e) = same(&table(&tab), want) { + panic!("HBA {h2}/{h1} flanks {left}/{right} MAPQ0 {share}: {e}"); + } + } +} + +#[test] +fn parity_lpa_tables_are_the_ones_before_the_engine() { + let (c, w) = (cat(), Scratch::new("gold-lpa")); + let (a, q, tab) = (w.path("a.tsv"), w.path("q.tsv"), w.path("t.tsv")); + let lrow = &c.loci["LPA_KIV2"]; + let g = gold(); + let Value::Object(g) = &g else { panic!() }; + let cases = arr(&g["lpa"]); + assert_eq!(cases.len(), 11); + for case in cases { + let (inp, want) = (arr(&arr(case)[0]), &arr(case)[1]); + let [n, flank, ks, fs, share] = [0, 1, 2, 3, 4].map(|i| num(&inp[i])); + let k = flank * (n + 2.0) / 14.0; + let mut m = Means::new(); + for (i, s) in lrow.targets.iter().enumerate() { + m.insert( + s.name.clone(), + k * (1.0 + if i % 2 == 0 { ks } else { -ks }), + ); + } + for (i, s) in lrow.controls.iter().enumerate() { + m.insert( + s.name.clone(), + flank * (1.0 + if i % 2 == 0 { fs } else { -fs }), + ); + } + let mq: Means = m + .iter() + .map(|(x, v)| { + ( + x.clone(), + if x.starts_with("kiv2_") { + v * (1.0 - share) + } else { + *v + }, + ) + }) + .collect(); + bedcov(&a, lrow, &m, true); + bedcov(&q, lrow, &mq, true); + let (rc, _, err) = og_cn(&[ + "call", + "--catalog", + &catalog_path(), + "--locus", + "LPA_KIV2", + "--bedcov", + &a, + "--bedcov-mapq1", + &q, + "--table", + &tab, + ]); + assert_eq!(rc, 0, "{err}"); + if let Err(e) = same(&table(&tab), want) { + panic!("LPA {n} copies at {flank}x: {e}"); + } + } +} + +// 4. synthetic copies, the MAPQ 0 ceiling, a kind=interval row + +#[test] +fn synthetic_depths_at_0_to_4_copies_are_called_right() { + let g = parsed(&row(&[])); + for copies in 0..=4 { + let m = means(&[ + ("fl_a", 30.0), + ("fl_b", 30.0), + ("TESTG", 15.0 * copies as f64), + ]); + let q: Means = m.iter().map(|(n, v)| (n.clone(), v * 0.9)).collect(); + let b = assess(&g, &m, Some(&q), Sex::Unknown).unwrap(); + assert_eq!( + (b.status, b.copies), + (Status::Called, Some(copies)), + "{copies} copies" + ); + } +} + +#[test] +fn the_mapq0_ceiling_between_counts_and_no_control_depth() { + let g = parsed(&row(&[])); + let m = means(&[("fl_a", 30.0), ("fl_b", 30.0), ("TESTG", 30.0)]); + let b = assess( + &g, + &m, + Some(&means(&[("fl_a", 30.0), ("fl_b", 30.0), ("TESTG", 12.0)])), + Sex::Unknown, + ) + .unwrap(); + assert_eq!( + (b.status, b.reason), + (Status::NotAssessed, Some(Reason::HighMapq0Share)), + "60% from MAPQ 0" + ); + let b = assess( + &g, + &m, + Some(&means(&[("fl_a", 30.0), ("fl_b", 30.0), ("TESTG", 16.0)])), + Sex::Unknown, + ) + .unwrap(); + assert_eq!( + (b.status, b.copies), + (Status::Called, Some(2)), + "47% from MAPQ 0" + ); + let m25 = means(&[("fl_a", 30.0), ("fl_b", 30.0), ("TESTG", 37.5)]); + let b = assess(&g, &m25, Some(&m25), Sex::Unknown).unwrap(); + assert_eq!(b.reason, Some(Reason::BetweenCounts), "2.5 copies"); + // No minimum control depth set: no depth, or one empty flank, is a reason, not a crash. + let z = parsed(&row(&[("min_control_depth", ".")])); + let m0 = means(&[("fl_a", 0.0), ("fl_b", 0.0), ("TESTG", 30.0)]); + let b = assess(&z, &m0, Some(&m0), Sex::Unknown).unwrap(); + assert_eq!(b.reason, Some(Reason::LowControlDepth)); + let m1 = means(&[("fl_a", 0.0), ("fl_b", 30.0), ("TESTG", 30.0)]); + let b = assess(&z, &m1, Some(&m1), Sex::Unknown).unwrap(); + assert_eq!(b.reason, Some(Reason::ControlsDisagree)); +} + +#[test] +fn an_interval_row_with_pooled_depths_and_the_mapq1_depths_rule() { + let iv = parsed(&row(&[ + ("kind", "interval"), + ("target_depth", "pooled"), + ("control_depth", "pooled"), + ("mapq0_max", "."), + ("bedcov", "."), + ("targets", "seg1:50000-50500;seg2:50500-51000"), + ])); + let m = means(&[ + ("fl_a", 20.0), + ("fl_b", 40.0), + ("seg1", 45.0), + ("seg2", 45.0), + ]); + let b = assess(&iv, &m, None, Sex::Unknown).unwrap(); + assert_eq!( + (b.status, b.copies), + (Status::Called, Some(3)), + "pooled 45x over pooled 30x" + ); + let c = cat(); + let rhd = &c.loci["RHD"]; + let m: Means = rhd + .regions() + .iter() + .map(|r| (r.3.to_owned(), 30.0)) + .collect(); + let b = assess(rhd, &m, None, Sex::Unknown).unwrap(); + assert!( + b.status == Status::Called && b.mapq0_share.is_none(), + "{b:?}" + ); + let g = parsed(&row(&[])); + let m = means(&[("fl_a", 30.0), ("fl_b", 30.0), ("TESTG", 37.5)]); + match assess(&g, &m, None, Sex::Unknown) { + Err(AssessError::NeedsMapq1(e)) => { + assert!(e.contains("MAPQ >= 1 depths are needed"), "{e}") + } + other => panic!("{other:?}"), + } +} + +// 5. a broken control + +#[test] +fn a_control_with_no_read_is_broken_and_no_read_anywhere_is_not_assessed() { + let g = parsed(&row(&[])); + let bad = means(&[("fl_a", 0.0), ("fl_b", 30.0), ("TESTG", 30.0)]); + match assess(&g, &bad, Some(&bad), Sex::Unknown) { + Err(AssessError::BrokenControl(e)) => { + assert!( + e.starts_with("BROKEN_CONTROL: control segment fl_a of TESTG"), + "{e}" + ); + } + other => panic!("{other:?}"), + } + let b = assess( + &g, + &means(&[("fl_a", 0.0), ("fl_b", 0.0), ("TESTG", 0.0)]), + Some(&Means::new()), + Sex::Unknown, + ) + .unwrap(); + assert_eq!( + (b.status, b.reason), + (Status::NotAssessed, Some(Reason::LowControlDepth)) + ); + let b = assess( + &g, + &means(&[("fl_a", 0.0), ("fl_b", 5.0), ("TESTG", 5.0)]), + Some(&Means::new()), + Sex::Unknown, + ) + .unwrap(); + assert_eq!( + b.reason, + Some(Reason::LowControlDepth), + "one flank empty beside a thin one" + ); +} + +#[test] +fn call_exits_3_on_a_broken_control_and_writes_nothing() { + let (c, w) = (cat(), Scratch::new("broken-call")); + let lrow = &c.loci["LPA_KIV2"]; + let mut d: Means = lrow + .regions() + .iter() + .map(|r| (r.3.to_owned(), 30.0)) + .collect(); + d.insert("flank_e19".into(), 0.0); + let (a, q, tab) = (w.path("a.tsv"), w.path("q.tsv"), w.path("t.tsv")); + bedcov(&a, lrow, &d, false); + bedcov(&q, lrow, &d, false); + let (rc, _, err) = og_cn(&[ + "call", + "--catalog", + &catalog_path(), + "--locus", + "LPA_KIV2", + "--bedcov", + &a, + "--bedcov-mapq1", + &q, + "--table", + &tab, + ]); + assert_eq!(rc, 3, "{err}"); + assert!( + err.contains("BROKEN_CONTROL: control segment flank_e19 of LPA_KIV2"), + "{err}" + ); + assert!(!std::path::Path::new(&tab).exists()); +} + +// 6. the command line + +#[test] +fn the_bedcov_reader_refuses_a_region_twice_or_an_empty_region() { + let w = Scratch::new("reader"); + let p = w.path("c.tsv"); + write_catalog(&p, &[row(&[])]); + let g = parsed(&row(&[])); + let b = w.path("dup.bedcov"); + bedcov( + &b, + &g, + &means(&[("fl_a", 30.0), ("fl_b", 30.0), ("TESTG", 30.0)]), + false, + ); + let first = read(&b).lines().next().unwrap().to_owned(); + let mut text = read(&b); + text += &(first.rsplit_once('\t').unwrap().0.to_owned() + "\t0\n"); + std::fs::write(&b, text).unwrap(); + let (rc, _, err) = og_cn(&[ + "call", + "--catalog", + &p, + "--locus", + "TESTG", + "--bedcov", + &b, + "--bedcov-mapq1", + &b, + ]); + assert!(rc == 2 && err.contains("appears twice"), "{rc} {err}"); + std::fs::write( + &b, + format!( + "{}\t50000\t50000\tTESTG\t0\n", + first.split('\t').next().unwrap() + ), + ) + .unwrap(); + let (rc, _, err) = og_cn(&[ + "call", + "--catalog", + &p, + "--locus", + "TESTG", + "--bedcov", + &b, + "--bedcov-mapq1", + &b, + ]); + assert!( + rc == 2 && err.contains("is empty or reversed"), + "{rc} {err}" + ); +} + +#[test] +fn bed_prints_every_segment_of_the_bedcov_loci_once() { + let c = cat(); + let (rc, out, err) = og_cn(&["bed", "--catalog", &catalog_path()]); + assert_eq!(rc, 0, "{err}"); + let names: Vec<&str> = out.lines().map(|l| l.split('\t').nth(3).unwrap()).collect(); + let uniq: HashSet<&&str> = names.iter().collect(); + let want: usize = c + .loci + .values() + .filter(|r| r.input == og_cn::catalog::Input::Bedcov) + .map(|r| r.regions().len()) + .sum(); + assert_eq!((names.len(), uniq.len()), (want, want)); +} + +/// Step 46's table, in its order. +const LPA_ORDER: [&str; 10] = [ + "status", + "kiv2_copies", + "ci95_low", + "ci95_high", + "class", + "kiv2_depth", + "flank_depth", + "mapq0_share", + "message", + "advice", +]; + +#[test] +fn call_writes_step_46s_table_needs_mapq1_and_reads_empty_tables() { + let (c, w) = (cat(), Scratch::new("cli")); + let lrow = &c.loci["LPA_KIV2"]; + let d: Means = lrow + .regions() + .iter() + .map(|r| { + ( + r.3.to_owned(), + if r.3.starts_with("kiv2_") { + 30.0 * 22.0 / 14.0 + } else { + 30.0 + }, + ) + }) + .collect(); + let q: Means = d + .iter() + .map(|(n, v)| (n.clone(), if n.starts_with("kiv2_") { v * 0.1 } else { *v })) + .collect(); + let (a, b, tab) = (w.path("a.tsv"), w.path("q.tsv"), w.path("t.tsv")); + bedcov(&a, lrow, &d, false); + bedcov(&b, lrow, &q, false); + let cp = catalog_path(); + let (rc, _, err) = og_cn(&[ + "call", + "--catalog", + &cp, + "--locus", + "LPA_KIV2", + "--bedcov", + &a, + "--bedcov-mapq1", + &b, + "--table", + &tab, + ]); + let t = table(&tab); + assert_eq!(rc, 0, "{err}"); + assert_eq!( + (get(&t, "status"), get(&t, "kiv2_copies")), + ("estimated", "20.0") + ); + assert_eq!( + t.iter().map(|(k, _)| k.as_str()).collect::>(), + LPA_ORDER + ); + let (rc, _, err) = og_cn(&[ + "call", + "--catalog", + &cp, + "--locus", + "LPA_KIV2", + "--bedcov", + &a, + ]); + assert!(rc == 2 && err.contains("--bedcov-mapq1"), "{rc} {err}"); + let empty = w.put("empty.tsv", ""); + for (loc, key) in [("HBA", "alpha_globin_genes"), ("LPA_KIV2", "kiv2_copies")] { + let _ = std::fs::remove_file(&tab); + let (rc, _, err) = og_cn(&[ + "call", + "--catalog", + &cp, + "--locus", + loc, + "--bedcov", + &empty, + "--bedcov-mapq1", + &empty, + "--table", + &tab, + ]); + let t = table(&tab); + assert_eq!(rc, 0, "{loc}: {err}"); + assert_eq!( + (get(&t, "status"), get(&t, key)), + ("not assessed", "."), + "{loc}" + ); + assert!(get(&t, "message").contains("no chr"), "{loc}: {t:?}"); + } +} + +// 8. the gene-dosage rows + +/// Mean depths of a dosage row at a copy number: controls at 30x, the +/// target copies spread evenly over the row's targets, and the MAPQ >= 1 +/// depths with half of the targets'. +fn dosage_depths(r: &Locus, copies: f64) -> (Means, Means) { + let tg: HashSet<&str> = r.targets.iter().map(|s| s.name.as_str()).collect(); + let m: Means = r + .regions() + .iter() + .map(|x| { + ( + x.3.to_owned(), + if tg.contains(x.3) { + 15.0 * copies / tg.len() as f64 + } else { + 30.0 + }, + ) + }) + .collect(); + let q = m + .iter() + .map(|(n, v)| { + ( + n.clone(), + if tg.contains(n.as_str()) { v * 0.5 } else { *v }, + ) + }) + .collect(); + (m, q) +} + +/// Words a reading of CCL3L1 must not carry: its disease links are disputed. +const DENY: [&str; 16] = [ + "hiv", + "aids", + "infect", + "disease", + "risk", + "suscept", + "progress", + "malaria", + "hepatitis", + "arthritis", + "lupus", + "kawasaki", + "psoria", + "diabet", + "crohn", + "vaccin", +]; + +fn denied(text: &str) -> Vec<&'static str> { + let low = text.to_lowercase(); + DENY.iter().copied().filter(|w| low.contains(w)).collect() +} + +#[test] +fn the_dosage_rows_count_0_to_4_copies_with_their_readings() { + let (c, raw) = (cat(), raw_rows()); + for loc in DOSAGE { + let r = &c.loci[loc]; + assert_eq!( + r.depth().passes, + [".", "-Q 1"], + "{loc}: all reads, then MAPQ >= 1" + ); + assert!(matches!(r.call, CallRule::Round { .. }), "{loc}"); + for k in ["0", "1", "more"] { + assert!(r.readings.contains_key(k), "{loc}: no reading for {k}"); + } + assert!(pmids_ok(&raw[loc]["pmids"], false), "{loc}: a PMID"); + for copies in 0..=4 { + let (m, q) = dosage_depths(r, copies as f64); + let b = assess(r, &m, Some(&q), Sex::Unknown).unwrap(); + assert_eq!( + (b.status, b.copies), + (Status::Called, Some(copies)), + "{loc} {copies}" + ); + assert_eq!(b.reading, r.reading_count(copies), "{loc} {copies}"); + assert!(!b.reading.contains("{n}"), "{loc}: {}", b.reading); + } + let (m, q) = dosage_depths(r, 2.5); + assert_eq!( + assess(r, &m, Some(&q), Sex::Unknown).unwrap().reason, + Some(Reason::BetweenCounts), + "{loc}" + ); + } + let hp = &c.loci["HP"]; + let (mut m, q) = dosage_depths(hp, 4.0); + m.insert("HP_UNIT1".into(), 60.0); + m.insert("HP_UNIT2".into(), 0.0); + assert_eq!( + assess(hp, &m, Some(&q), Sex::Unknown).unwrap().copies, + Some(4), + "both copies on one reference copy" + ); +} + +#[test] +fn no_ccl3l1_reading_names_a_disease() { + let c = cat(); + let readings: Vec<&str> = c.loci["CCL3L1"] + .readings + .values() + .map(String::as_str) + .collect(); + let all = readings.join(" "); + assert_eq!(denied(&all), Vec::<&str>::new()); + assert!( + !denied(&(all + " more copies go with slower HIV progression")).is_empty(), + "control: a planted HIV" + ); +} + +#[test] +fn a_dosage_row_shifted_by_1_kb_refuses_its_fixture() { + let w = Scratch::new("shift"); + let (c, raw) = (cat(), raw_rows()); + let p = w.path("c.tsv"); + for loc in DOSAGE { + let mut r = raw[loc].clone(); + r.insert( + "targets".into(), + c.loci[loc] + .targets + .iter() + .map(|s| format!("{}:{}-{}", s.name, s.start + 1000, s.end + 1000)) + .collect::>() + .join(";"), + ); + write_catalog(&p, &[r]); + let f = repo(&format!("tests/fixtures/cn/{loc}.bedcov.tsv")); + let (rc, _, err) = og_cn(&[ + "call", + "--catalog", + &p, + "--locus", + loc, + "--bedcov", + &f, + "--bedcov-mapq1", + &f, + ]); + assert!( + rc == 2 && err.contains("regions differ"), + "{loc}: {rc} {err}" + ); + // and the unshifted row reads it + let (rc, _, err) = og_cn(&[ + "call", + "--catalog", + &catalog_path(), + "--locus", + loc, + "--bedcov", + &f, + "--bedcov-mapq1", + &f, + ]); + assert_eq!(rc, 0, "{loc}: {err}"); + } +} + +// 9. the interval rows + +const HAND: [&str; 10] = [ + "1p36_terminal", + "Xq28_MECP2", + "GJB6_D13S1830", + "CFTR_dele2_3", + "HBB_exon3_del", + "HBB_619bp_del", + "HBD_HBB_Sicilian_del", + "AZFa", + "AZFb", + "AZFc_DAZ", +]; + +fn interval_rows(c: &Catalog) -> Vec<&Locus> { + c.loci + .values() + .filter(|r| r.kind == Kind::Interval && r.depth().passes == ["."]) + .collect() +} + +/// Targets at `target`, every other segment at `control`. +fn depths(r: &Locus, target: f64, control: f64) -> Means { + let tg: HashSet<&str> = r.targets.iter().map(|s| s.name.as_str()).collect(); + r.regions() + .iter() + .map(|x| { + ( + x.3.to_owned(), + if tg.contains(x.3) { target } else { control }, + ) + }) + .collect() +} + +#[test] +fn the_interval_rows_are_the_clingen_regions_then_the_hand_rows() { + let (c, raw) = (cat(), raw_rows()); + let iv = interval_rows(&c); + let names: Vec<&str> = iv.iter().map(|r| r.id.as_str()).collect(); + assert!( + iv.len() > 80 && names[names.len() - 10..] == HAND, + "{} {:?}", + iv.len(), + &names[names.len() - 10..] + ); + let spans = |l: &str| { + c.loci[l] + .targets + .iter() + .map(|s| (s.start, s.end)) + .collect::>() + }; + assert_eq!(spans("HBB_619bp_del"), [(5_225_388, 5_226_007)]); + assert_eq!(spans("HBD_HBB_Sicilian_del"), [(5_219_877, 5_233_280)]); + assert!( + iv.iter() + .all(|r| raw[&r.id]["cross_check"] == "cnvpytor:cnvs") + ); + let q15: Vec<&&Locus> = iv.iter().filter(|r| r.id.starts_with("15q13.3")).collect(); + assert!(!q15.is_empty()); + assert!( + q15.iter().all(|r| r + .targets + .iter() + .all(|s| !(32_154_858 < s.end && s.start < 32_178_018))), + "CHRNA7" + ); + let mut hits = Vec::new(); + for r in &iv { + for ctl in &r.controls { + for o in &iv { + if o.id != r.id + && o.chrom == r.chrom + && o.targets + .iter() + .any(|t| t.start < ctl.end && ctl.start < t.end) + { + hits.push((r.id.clone(), o.id.clone())); + } + } + } + } + assert_eq!( + hits, + Vec::<(String, String)>::new(), + "a control on another row's target" + ); +} + +#[test] +fn interval_rows_call_deletions_duplications_and_follow_the_sex() { + let c = cat(); + let (a_d, p17) = ( + &c.loci["22q11.2_DGS_proximal_A-D"], + &c.loci["17p12_HNPP_CMT1A"], + ); + for (r, target, want, ch) in [ + (a_d, 15.0, 1, "deletion"), + (p17, 45.0, 3, "duplication"), + (a_d, 30.0, 2, "no change"), + (p17, 30.0, 2, "no change"), + ] { + let b = assess(r, &depths(r, target, 30.0), None, Sex::Unknown).unwrap(); + assert_eq!( + (status(&b).as_str(), b.copies, change(&b)), + ("called", Some(want), ch), + "{} at {target}x", + r.id + ); + } + let az = &c.loci["AZFc_DAZ"]; + assert_eq!( + assess(az, &depths(az, 15.0, 15.0), None, Sex::Male) + .unwrap() + .copies, + Some(4) + ); + let b = assess(az, &depths(az, 7.5, 15.0), None, Sex::Male).unwrap(); + assert_eq!( + (b.copies, change(&b)), + (Some(2), "deletion"), + "AZFc at half its DAZ depth" + ); + let a = &c.loci["AZFa"]; + let b = assess(a, &depths(a, 15.0, 15.0), None, Sex::Male).unwrap(); + assert_eq!((b.copies, b.expected), (Some(1), Some(1))); + let b = assess(a, &depths(a, 0.0, 15.0), None, Sex::Male).unwrap(); + assert_eq!((b.copies, change(&b)), (Some(0), "deletion")); + for loc in ["AZFa", "AZFb", "AZFc_DAZ"] { + let r = &c.loci[loc]; + let b = assess(r, &depths(r, 15.0, 15.0), None, Sex::Female).unwrap(); + assert_eq!( + (b.status, b.reason), + (Status::NotApplicable, Some(Reason::NoY)), + "{loc} female" + ); + let b = assess(r, &depths(r, 15.0, 15.0), None, Sex::Unknown).unwrap(); + assert_eq!( + (b.status, b.reason), + (Status::NotAssessed, Some(Reason::SexUnknown)), + "{loc} unknown" + ); + } + let xp = &c.loci["Xp22.31"]; + let b = assess(xp, &depths(xp, 15.0, 15.0), None, Sex::Male).unwrap(); + assert_eq!((b.copies, b.expected), (Some(1), Some(1))); + let b = assess(xp, &depths(xp, 0.0, 15.0), None, Sex::Male).unwrap(); + assert_eq!(b.copies, Some(0), "the STS deletion"); +} + +/// samtools bedcov over the lines of a BED at 30x, the segments in `half` +/// at 15x and those in `zero` at 0. +fn fakecov(bed: &str, half: &HashSet, zero: &HashSet) -> String { + bed.lines() + .map(|l| { + let c: Vec<&str> = l.split('\t').collect(); + let d = if zero.contains(c[3]) { + 0.0 + } else if half.contains(c[3]) { + 15.0 + } else { + 30.0 + }; + let len: i64 = c[2].parse::().unwrap() - c[1].parse::().unwrap(); + format!("{l}\t{}\n", (d * len as f64) as i64) + }) + .collect() +} + +pub const INTERVAL_COLUMNS: [&str; 11] = [ + "locus", "name", "chrom", "start", "end", "status", "copies", "expected", "change", "estimate", + "reason", +]; + +#[test] +fn og_cn_intervals_writes_one_line_per_interval_row() { + let (c, w) = (cat(), Scratch::new("intervals")); + let iv = interval_rows(&c); + let names: Vec<&str> = iv.iter().map(|r| r.id.as_str()).collect(); + let cp = catalog_path(); + let (rc, bed, err) = og_cn(&["bed", "--catalog", &cp, "--kind", "interval"]); + assert_eq!(rc, 0, "{err}"); + assert_eq!( + bed.lines().count(), + iv.iter().map(|r| r.regions().len()).sum::() + ); + let (a_d, p17) = ( + &c.loci["22q11.2_DGS_proximal_A-D"], + &c.loci["17p12_HNPP_CMT1A"], + ); + let half: HashSet = a_d.targets.iter().map(|s| s.name.clone()).collect(); + let zero = HashSet::from([p17.controls[0].name.clone()]); + let cov = w.put("cov.tsv", &fakecov(&bed, &half, &zero)); + let tab = w.path("iv.tsv"); + let (rc, _, err) = og_cn(&[ + "intervals", + "--catalog", + &cp, + "--bedcov", + &cov, + "--sex", + "female", + "--out", + &tab, + ]); + assert_eq!(rc, 0, "{err}"); + let text = read(&tab); + let mut lines = text.lines(); + assert_eq!(lines.next(), Some(INTERVAL_COLUMNS.join("\t").as_str())); + let rows: IndexMap<&str, Vec<&str>> = lines + .map(|l| (l.split('\t').next().unwrap(), l.split('\t').collect())) + .collect(); + assert_eq!(rows.keys().copied().collect::>(), names); + assert_eq!( + rows[a_d.id.as_str()][5..9], + ["called", "1", "2", "deletion"] + ); + assert_eq!( + rows[p17.id.as_str()][10], + "broken_control", + "a control with no read" + ); + assert_eq!( + rows["3q29"][5..9], + ["called", "2", "2", "no change"], + "the rest are called" + ); + for loc in ["AZFa", "AZFb", "AZFc_DAZ"] { + assert_eq!(rows[loc][5], "not applicable", "{loc} on a female"); + } + let (rc, _, err) = og_cn(&["intervals", "--catalog", &cp, "--failed", "--out", &tab]); + assert_eq!(rc, 0, "{err}"); + let text = read(&tab); + let failed: Vec<&str> = text.lines().skip(1).collect(); + assert_eq!(failed.len(), iv.len()); + assert!( + failed + .iter() + .all(|l| l.split('\t').nth(10) == Some("check_failed")) + ); +} + +// 10. the CYP21A2 row (call psv) + +const CYP_SITES: [&str; 6] = [ + "PSV_In2G", + "PSV_I172N", + "PSV_V237E", + "PSV_V281L", + "PSV_Q318X", + "PSV_R356W", +]; + +#[test] +fn the_cyp21a2_row_and_its_psv_sites() { + let c = cat(); + let r = &c.loci["CYP21A2"]; + let names: Vec<&str> = r.psv.iter().map(|s| s.name.as_str()).collect(); + assert_eq!(names, CYP_SITES); + assert!(matches!(r.call, CallRule::Psv { .. })); + for f in PSV_FINDINGS { + assert!(r.readings.contains_key(f), "a reading for {f}"); + } + let tg: Vec<&str> = r.targets.iter().map(|s| s.name.as_str()).collect(); + assert_eq!(tg, ["CYP21A2_BODY", "CYP21A1P_BODY"]); + let (g, p) = (&r.targets[0], &r.targets[1]); + for s in &r.psv { + assert!(g.start <= s.gene_pos && s.gene_pos < g.end, "{}", s.name); + assert!( + p.start <= s.pseudo_pos && s.pseudo_pos < p.end, + "{}", + s.name + ); + assert!( + (32_700..32_800).contains(&(s.gene_pos - s.pseudo_pos)), + "{}", + s.name + ); + } + let mut gb = r.psv[0].gene.clone(); + gb.sort_unstable(); + assert_eq!( + (gb.as_slice(), r.psv[0].pseudo.as_slice()), + (&b"AC"[..], &b"G"[..]), + "In2G c.293-13A/C>G" + ); + let d = r.depth(); + assert_eq!(d.passes, [".", "-Q 1"]); + assert_eq!((d.mapq0_min, d.mapq0_max), (Some(0.02), None)); +} + +#[test] +fn broken_cyp21a2_rows_are_refused() { + let w = Scratch::new("cyp-broken"); + let base = raw_rows()["CYP21A2"].clone(); + let det = base["details"].clone(); + let readings: String = base["readings"] + .split('|') + .filter(|x| !x.starts_with("conversion=")) + .collect::>() + .join("|"); + for (desc, col, val, err) in [ + ( + "a PSV site of two bases", + "details", + det.replace("PSV_In2G:32039080-32039081", "PSV_In2G:32039080-32039082"), + "must be one base", + ), + ( + "a site with the same base on both genes", + "details", + det.replace("32006352 AC>G", "32006352 AC>C"), + "distinct gene and pseudogene", + ), + ( + "a site label without its pseudogene position", + "details", + det.replace("32006352 AC>G", "here AC>G"), + "is not 'PSEUDO_POS", + ), + ( + "a psv row without a conversion reading", + "readings", + readings, + "a reading for each", + ), + ] { + let mut r = base.clone(); + assert_ne!(r[col], val, "{desc}: nothing planted"); + r.insert(col.into(), val); + let got = load_rows(&w, &[r]); + assert!( + got.clone().is_err_and(|e| e.contains(err)), + "{desc}: {got:?}" + ); + } +} + +#[test] +fn bed_psv_prints_twelve_one_base_positions_and_refuses_a_row_without_sites() { + let cp = catalog_path(); + let (rc, out, err) = og_cn(&["bed", "--catalog", &cp, "--psv", "--locus", "CYP21A2"]); + assert_eq!(rc, 0, "{err}"); + let starts: Vec = out + .lines() + .map(|l| l.split('\t').nth(1).unwrap().parse().unwrap()) + .collect(); + assert_eq!(starts.len(), 12); + assert!(out.lines().all(|l| { + let c: Vec = l + .split('\t') + .skip(1) + .take(2) + .map(|x| x.parse().unwrap()) + .collect(); + c[1] - c[0] == 1 + })); + let mut sorted = starts.clone(); + sorted.sort_unstable(); + assert_eq!(starts, sorted); + let (rc, _, err) = og_cn(&["bed", "--catalog", &cp, "--psv", "--locus", "HBA"]); + assert!(rc == 2 && err.contains("no psv locus"), "{rc} {err}"); +} + +/// (bedcov text, mpileup text) of a CYP21A2 genome: 15x per copy, flanks at +/// 30x, two CYP21A2 copies on CYP21A2's body and the rest of the module +/// total on CYP21A1P's. At each site the reads carrying CYP21A2's base +/// (`a2` copies, 2 by default) and CYP21A1P's (the rest) are split 4 to 1 +/// between the CYP21A2 and the CYP21A1P position, with read starts, ends and +/// an indel in the bases column. +fn cyp_fixture(r: &Locus, total: f64, a2: &[(&str, f64)]) -> (String, String) { + let mut depth: Means = r.regions().iter().map(|x| (x.3.to_owned(), 30.0)).collect(); + depth.insert("CYP21A2_BODY".into(), 30.0); + depth.insert("CYP21A1P_BODY".into(), 15.0 * (total - 2.0)); + let bed: String = r + .regions() + .iter() + .map(|(c, s, e, n)| { + format!( + "{c}\t{s}\t{e}\t{n}\t{}\n", + round(depth[*n] * (e - s) as f64) + ) + }) + .collect(); + let mut lines: Vec<(i64, String)> = Vec::new(); + for site in &r.psv { + let copies = a2 + .iter() + .find(|(n, _)| *n == site.name) + .map_or(2.0, |x| x.1); + let (gene, pseudo) = (15.0 * copies, 15.0 * (total - copies)); + let gbase = *site.gene.iter().max().unwrap() as char; + let pbase = *site.pseudo.iter().min().unwrap() as char; + for (pos, share) in [(site.gene_pos, 0.8), (site.pseudo_pos, 0.2)] { + let (ng, np) = (round(gene * share), round(pseudo * share)); + let mut reads: Vec = std::iter::repeat_n(gbase, ng as usize) + .chain(std::iter::repeat_n(pbase, np as usize)) + .enumerate() + .map(|(i, x)| { + if i % 2 == 1 { + x.to_ascii_lowercase().to_string() + } else { + x.to_string() + } + }) + .collect(); + if !reads.is_empty() { + reads[0] = format!("^]{}", reads[0]); + let last = reads.len() - 1; + reads[last] += "$"; + } + if reads.len() > 2 { + reads[1] += "+2AG"; + } + let n = reads.len(); + lines.push(( + pos + 1, + format!( + "{}\t{}\tN\t{n}\t{}\t{}\n", + r.chrom, + pos + 1, + reads.concat(), + "I".repeat(n) + ), + )); + } + } + lines.sort(); + (bed, lines.into_iter().map(|x| x.1).collect()) +} + +/// What `og cn call --locus CYP21A2` wrote for a genome, read back. +#[derive(Debug)] +struct Cyp { + table: Vec<(String, String)>, + line: String, + sites: HashMap, +} + +impl Cyp { + fn v(&self, k: &str) -> Option<&str> { + Some(get(&self.table, k)).filter(|v| !v.is_empty() && *v != ".") + } + + fn copies(&self) -> Option { + self.v("cyp21a2_copies").map(|v| v.parse().unwrap()) + } + + fn total(&self) -> Option { + self.v("module_total").map(|v| v.parse().unwrap()) + } + + fn conversion_sites(&self) -> Vec<&str> { + self.v("conversion_sites") + .map_or(Vec::new(), |v| v.split(',').collect()) + } +} + +/// `copies X (EST); ratio RATIO` in a psv_ row: (EST, RATIO). +fn site_numbers(v: &str) -> Option<(f64, f64)> { + let est = v + .split_once("copies ")? + .1 + .split_once(" (")? + .1 + .split_once(')')? + .0 + .parse() + .ok()?; + let ratio = v + .split_once("ratio ")? + .1 + .split(|c: char| !(c.is_ascii_digit() || c == '.')) + .next()? + .parse() + .ok()?; + Some((est, ratio)) +} + +/// og cn call on a CYP21A2 genome; `mapq0` is the share of every segment's +/// depth from MAPQ 0 reads (0.1 as HG002's reads give over the two targets). +fn cyp_call(bed: &str, pileup: &str, mapq0: f64) -> Cyp { + static N: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + let w = Scratch::new(&format!( + "cyp-{}", + N.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + )); + let b = w.put("b.tsv", bed); + let q = w.put( + "q.tsv", + &bed.lines() + .map(|l| { + let c: Vec<&str> = l.split('\t').collect(); + format!( + "{}\t{:?}\n", + c[..4].join("\t"), + c[4].parse::().unwrap() * (1.0 - mapq0) + ) + }) + .collect::(), + ); + let p = w.put("p.tsv", pileup); + let tab = w.path("t.tsv"); + let (_, out, _) = og_cn(&[ + "call", + "--catalog", + &catalog_path(), + "--locus", + "CYP21A2", + "--bedcov", + &b, + "--bedcov-mapq1", + &q, + "--pileup", + &p, + "--table", + &tab, + ]); + let table = table(&tab); + let sites = table + .iter() + .filter_map(|(k, v)| Some((k.strip_prefix("psv_")?.to_owned(), site_numbers(v)?))) + .collect(); + Cyp { + table, + line: out.trim().to_owned(), + sites, + } +} + +fn cyp_row() -> Locus { + cat().loci["CYP21A2"].clone() +} + +/// A CYP21A2 case: its name, the module total and the site readings. +type CypCase = (&'static str, f64, &'static [(&'static str, f64)]); + +const CYP_CASES: [CypCase; 3] = [ + ("normal", 4.0, &[]), + // the 30 kb chimeric deletion: one module fewer, CYP21A1P's sequence at the 5' sites + ("deletion", 3.0, &[("PSV_In2G", 1.0), ("PSV_I172N", 1.0)]), + // In2G converted to the pseudogene base on one of the two CYP21A2 copies + ("conversion", 4.0, &[("PSV_In2G", 1.0)]), +]; + +#[test] +fn the_committed_cyp21a2_fixtures_are_the_generators_and_call_right() { + let r = cyp_row(); + let mut got = HashMap::new(); + for (case, total, a2) in CYP_CASES { + let (bed, pile) = cyp_fixture(&r, total, a2); + let fb = read(&repo(&format!( + "tests/fixtures/cn/CYP21A2_{case}.bedcov.tsv" + ))); + let fp = read(&repo(&format!( + "tests/fixtures/cn/CYP21A2_{case}.pileup.tsv" + ))); + assert!( + fb == bed && fp == pile, + "tests/fixtures/cn/CYP21A2_{case} differs from the generator" + ); + got.insert(case, cyp_call(&bed, &pile, 0.1)); + } + let b = &got["normal"]; + assert_eq!( + (b.v("status"), b.copies(), b.total(), b.v("finding")), + (Some("called"), Some(2), Some(4), Some("none")) + ); + assert_eq!(b.v("reading").unwrap_or_default(), r.reading_word("none")); + let b = &got["deletion"]; + assert_eq!( + (b.v("status"), b.copies(), b.total(), b.v("finding")), + (Some("called"), Some(1), Some(3), Some("deletion")) + ); + let b = &got["conversion"]; + let ratio = b.sites.get("PSV_In2G").map(|x| x.1).unwrap_or(f64::NAN); + assert!((ratio - 0.5).abs() <= 0.1, "{b:?}"); + assert_eq!( + (b.v("finding"), b.conversion_sites(), b.total()), + (Some("conversion"), vec!["PSV_In2G"], Some(4)) + ); + let reading = b.v("reading").unwrap_or_default(); + assert!( + reading.contains("c.293-13A/C>G (In2G") && reading.contains("possible conversion"), + "{reading}" + ); +} + +#[test] +fn the_cyp21a2_call_rests_on_the_psv_reads() { + let r = cyp_row(); + let (bed, _) = cyp_fixture(&r, 4.0, &[("PSV_In2G", 1.0)]); + let (_, even) = cyp_fixture(&r, 4.0, &[]); + assert_eq!( + cyp_call(&bed, &even, 0.1).v("finding"), + Some("none"), + "the conversion's depths, every site even" + ); + let (bed, pile) = cyp_fixture(&r, 3.0, &[]); + let b = cyp_call(&bed, &pile, 0.1); + assert_eq!( + (b.copies(), b.v("finding")), + (Some(2), Some("pseudogene_deletion")) + ); + // One noisy 3' site (V281L at 1.47, rounds to 1) inside the 0.6 margin. + for (total, want) in [(3.0, "pseudogene_deletion"), (4.0, "none")] { + let (bed, pile) = cyp_fixture(&r, total, &[("PSV_V281L", 1.47)]); + let b = cyp_call(&bed, &pile, 0.1); + let est = b.sites.get("PSV_V281L").map_or(f64::NAN, |x| x.0); + assert!(1.4 < est && est < 1.5, "{b:?}"); + assert_eq!( + (b.copies(), b.v("finding")), + (Some(2), Some(want)), + "module total {total}" + ); + } + let all: Vec<(&str, f64)> = CYP_SITES.iter().map(|n| (*n, 1.0)).collect(); + let (bed, pile) = cyp_fixture(&r, 4.0, &all); + let b = cyp_call(&bed, &pile, 0.1); + assert_eq!( + (b.v("finding"), b.conversion_sites(), b.copies()), + (Some("conversion"), CYP_SITES.to_vec(), Some(1)) + ); + let (bed, _) = cyp_fixture(&r, 4.0, &[]); + let thin: String = even + .lines() + .map(|l| { + let c: Vec<&str> = l.split('\t').collect(); + format!("{}\t{}\tN\t1\tC\tI\n", c[0], c[1]) + }) + .collect(); + let b = cyp_call(&bed, &thin, 0.1); + assert_eq!( + (b.v("status"), b.v("reason")), + (Some("not assessed"), Some("psv_low_depth")) + ); + let (bed, pile) = cyp_fixture(&r, 4.0, &[]); + let from = format!("\t{}\n", round(30.0 * 2689.0)); + assert!(bed.contains(&from)); + let b = cyp_call( + &bed.replace(&from, &format!("\t{}\n", round(37.5 * 2689.0))), + &pile, + 0.1, + ); + assert_eq!( + (b.v("status"), b.v("reason"), b.v("finding")), + (Some("not assessed"), Some("between_counts"), None) + ); +} + +#[test] +fn the_cyp21a2_mapq0_guard() { + let r = cyp_row(); + let (bed, pile) = cyp_fixture(&r, 3.0, &[("PSV_In2G", 1.0), ("PSV_I172N", 1.0)]); + for (share, want) in [ + ( + 0.0, + (Some("not assessed"), Some("low_mapq0_share"), None, None), + ), + ( + 0.019, + (Some("not assessed"), Some("low_mapq0_share"), None, None), + ), + (0.021, (Some("called"), None, Some("deletion"), Some(1))), + (0.1, (Some("called"), None, Some("deletion"), Some(1))), + ] { + let b = cyp_call(&bed, &pile, share); + assert_eq!( + (b.v("status"), b.v("reason"), b.v("finding"), b.copies()), + want, + "share {share}" + ); + } + let b = cyp_call(&bed, &pile, 0.0); + let msg = get(&b.table, "message"); + assert!( + msg.starts_with("not assessed: the BAM has few or no multi-mapping reads here"), + "{msg}" + ); + assert!(msg.contains("CYP21A2 and CYP21A1P"), "{msg}"); + assert_eq!(get(&b.table, "mapq0_share"), "0.000"); + assert_eq!( + ( + get(&b.table, "cyp21a2_copies"), + get(&b.table, "module_total"), + get(&b.table, "finding") + ), + (".", ".", ".") + ); + assert!(b.line.contains("low_mapq0_share"), "{}", b.line); + let (bed, pile) = cyp_fixture(&r, 4.0, &[]); + let t = cyp_call(&bed, &pile, 0.1).table; + assert_eq!( + ( + get(&t, "status"), + get(&t, "finding"), + get(&t, "mapq0_share"), + get(&t, "message") + ), + ("called", "none", "0.100", ".") + ); +} diff --git a/rust/crates/og-cn/tests/genome_lpa.rs b/rust/crates/og-cn/tests/genome_lpa.rs new file mode 100644 index 000000000..d5b2abcaf --- /dev/null +++ b/rust/crates/og-cn/tests/genome_lpa.rs @@ -0,0 +1,673 @@ +//! `og cn call --locus LPA_KIV2`: step 46's LPA KIV-2 repeat count from +//! short-read depth, and KIV-2 4925G>A beside it (`og cn marker-read`, `og cn +//! call --marker`). These are the tool checks of the old +//! `tests/test_lpa_kiv2.py`, by meaning; its report checks are +//! `rust/og/tests/genome_steps.rs` and its traits-table check is +//! og-steps' `tests/genome_traits.rs`. +//! +//! On `samtools bedcov` tables made up for each case (summed depth = mean +//! depth x exon length), 30x over the single-copy LPA exons, with 90% of the +//! KIV-2 depth from reads of MAPQ 0 unless a case says otherwise: +//! 1. the arithmetic: a block depth of 30 x (N + 2) / 14 gives N, for N = +//! 12, 20 and 40; +//! 2. the interval: equal depths give a zero-width interval; planted spread +//! gives sqrt(cv_kiv2^2 / 14 + cv_flank^2 / 15) on N + 2, computed here +//! by hand; twice the spread gives a wider interval; +//! 3. the class: low, typical and high by the whole interval against 30 +//! and 45; +//! 4. too little depth, no MAPQ 0 read over KIV-2, no chr6 and a depth under +//! the floor of any count are not counts; +//! 5. the command line: 29 regions, the table, and the refused inputs; +//! 8. KIV-2 4925G>A at its seven GRCh38 copies: present, not seen, +//! uncertain, too little depth, not assessed; other positions and indel +//! bases are not counted; step 46's table is left byte for byte; +//! `call --marker` writes what `marker-read` writes; +//! 9. the demo sample's step 46 table (`tests/demo/make_demo_sample.py`) is +//! the one `og cn call` writes for its depths. + +use std::collections::HashMap; +use std::path::PathBuf; + +use og_cn::catalog::{Catalog, Locus}; + +const REPO: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../.."); + +/// The seven copies of KIV-2 4925G>A on GRCh38. +const COPIES: [i64; 7] = [ + 160_646_214, + 160_640_667, + 160_635_123, + 160_629_574, + 160_624_030, + 160_618_484, + 160_612_932, +]; + +const ADVICE: &str = "A research estimate from read depth, not a diagnostic test. Lp(a) is measured in blood; a 2022 \ + European Atherosclerosis Society consensus advises measuring it at least once in every adult."; + +fn catalog_path() -> String { + format!("{REPO}/bin/cn_loci.tsv") +} + +fn lpa_row() -> Locus { + Catalog::load(&catalog_path()).unwrap().loci["LPA_KIV2"].clone() +} + +fn read(path: &str) -> String { + std::fs::read_to_string(path).unwrap_or_default() +} + +struct Scratch(PathBuf); + +impl Scratch { + fn new(name: &str) -> Scratch { + let d = std::env::temp_dir().join(format!("og-genome-lpa-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + Scratch(d) + } + + fn path(&self, rel: &str) -> String { + let p = self.0.join(rel); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + p.to_string_lossy().into_owned() + } +} + +impl Drop for Scratch { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +fn og_cn(args: &[&str]) -> (u8, String, String) { + let args: Vec = args.iter().map(|a| (*a).to_owned()).collect(); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = og_cn::cli::main(&args, &mut out, &mut err); + ( + code, + String::from_utf8(out).unwrap(), + String::from_utf8(err).unwrap(), + ) +} + +type Depth = HashMap; + +/// Mean depth per region: the flanks at `flank`, the KIV-2 block at the +/// depth `n` copies give; a spread alternates +x and -x around each mean. +fn depths(r: &Locus, n: f64, flank: f64, ks: f64, fs: f64) -> Depth { + let k = flank * (n + 2.0) / 14.0; + let mut m = Depth::new(); + for (i, s) in r.targets.iter().enumerate() { + m.insert( + s.name.clone(), + k * (1.0 + if i % 2 == 0 { ks } else { -ks }), + ); + } + for (i, s) in r.controls.iter().enumerate() { + m.insert( + s.name.clone(), + flank * (1.0 + if i % 2 == 0 { fs } else { -fs }), + ); + } + m +} + +/// The MAPQ >= 1 depths: `share` of the KIV-2 depth is from MAPQ 0. +fn q1(m: &Depth, share: f64) -> Depth { + m.iter() + .map(|(n, v)| { + ( + n.clone(), + if n.starts_with("kiv2_") { + v * (1.0 - share) + } else { + *v + }, + ) + }) + .collect() +} + +fn bedcov(path: &str, r: &Locus, m: &Depth, exact: bool) { + let text: String = r + .regions() + .iter() + .map(|(c, s, e, n)| { + let v = m[*n] * (e - s) as f64; + if exact { + format!("{c}\t{s}\t{e}\t{n}\t{v:?}\n") + } else { + format!("{c}\t{s}\t{e}\t{n}\t{}\n", v.round_ties_even() as i64) + } + }) + .collect(); + std::fs::write(path, text).unwrap(); +} + +fn table(path: &str) -> HashMap { + read(path) + .lines() + .skip(1) + .filter_map(|l| l.split_once('\t')) + .map(|(k, v)| (k.to_owned(), v.to_owned())) + .collect() +} + +fn g<'a>(t: &'a HashMap, k: &str) -> &'a str { + t.get(k).map_or("", String::as_str) +} + +fn estimate(a: &str, b: &str, out: &str) -> (u8, String) { + let (rc, _, err) = og_cn(&[ + "call", + "--catalog", + &catalog_path(), + "--locus", + "LPA_KIV2", + "--bedcov", + a, + "--bedcov-mapq1", + b, + "--table", + out, + ]); + (rc, err) +} + +/// Step 46's table for mean depths m (every read) and q (MAPQ >= 1); empty +/// tables when `m` is None. +fn assess(name: &str, mq: Option<(&Depth, &Depth)>) -> HashMap { + let w = Scratch::new(name); + let r = lpa_row(); + let (a, b, o) = (w.path("a.tsv"), w.path("b.tsv"), w.path("o.tsv")); + match mq { + Some((m, q)) => { + bedcov(&a, &r, m, true); + bedcov(&b, &r, q, true); + } + None => { + std::fs::write(&a, "").unwrap(); + std::fs::write(&b, "").unwrap(); + } + } + let (rc, err) = estimate(&a, &b, &o); + assert_eq!(rc, 0, "{err}"); + table(&o) +} + +fn plain(name: &str, n: f64, flank: f64, share: f64) -> HashMap { + let m = depths(&lpa_row(), n, flank, 0.0, 0.0); + assess(name, Some((&m, &q1(&m, share)))) +} + +fn f(t: &HashMap, k: &str) -> f64 { + g(t, k).parse().unwrap_or_else(|_| panic!("{k}: {t:?}")) +} + +fn cv(v: &[f64]) -> f64 { + let mu = v.iter().sum::() / v.len() as f64; + (v.iter().map(|x| (x - mu).powi(2)).sum::() / (v.len() - 1) as f64).sqrt() / mu +} + +#[test] +fn the_arithmetic_a_block_at_30_n_plus_2_over_14_is_n() { + for n in [12.0, 20.0, 40.0] { + let t = plain(&format!("n{n}"), n, 30.0, 0.9); + assert_eq!( + (g(&t, "status"), g(&t, "kiv2_copies")), + ("estimated", format!("{n:.1}").as_str()), + "{t:?}" + ); + } +} + +#[test] +fn the_interval_by_hand() { + let r = lpa_row(); + let t = plain("zero", 20.0, 30.0, 0.9); + assert_eq!( + (g(&t, "ci95_low"), g(&t, "ci95_high")), + ("20.0", "20.0"), + "equal depths: zero width" + ); + let m = depths(&r, 20.0, 30.0, 0.10, 0.15); + let t = assess("spread", Some((&m, &q1(&m, 0.9)))); + let kv: Vec = r.targets.iter().map(|s| m[&s.name]).collect(); + let fv: Vec = r.controls.iter().map(|s| m[&s.name]).collect(); + let rel = (cv(&kv).powi(2) / 14.0 + cv(&fv).powi(2) / 15.0).sqrt(); + let pooled = |segs: &[og_cn::catalog::Segment]| { + segs.iter() + .map(|s| m[&s.name] * (s.end - s.start) as f64) + .sum::() + / segs.iter().map(|s| (s.end - s.start) as f64).sum::() + }; + let total = 14.0 * pooled(&r.targets) / pooled(&r.controls); + let want = ( + format!("{:.1}", total - 2.0), + format!("{:.1}", total * (-1.96 * rel).exp() - 2.0), + format!("{:.1}", total * (1.96 * rel).exp() - 2.0), + ); + let got = ( + g(&t, "kiv2_copies").to_owned(), + g(&t, "ci95_low").to_owned(), + g(&t, "ci95_high").to_owned(), + ); + assert_eq!(got, want); + assert!(f(&t, "ci95_low") < f(&t, "kiv2_copies") && f(&t, "kiv2_copies") < f(&t, "ci95_high")); + let m2 = depths(&r, 20.0, 30.0, 0.20, 0.30); + let t2 = assess("spread2", Some((&m2, &q1(&m2, 0.9)))); + assert!( + f(&t2, "ci95_high") - f(&t2, "ci95_low") > f(&t, "ci95_high") - f(&t, "ci95_low"), + "twice the spread" + ); +} + +#[test] +fn the_class_by_the_whole_interval() { + let r = lpa_row(); + for (n, sp, cls) in [ + (20.0, 0.03, "low"), + (29.0, 0.03, "low"), + (29.0, 0.25, "typical"), + (38.0, 0.03, "typical"), + (47.0, 0.25, "typical"), + (60.0, 0.03, "high"), + ] { + let m = depths(&r, n, 30.0, sp, sp); + let t = assess(&format!("class-{n}-{sp}"), Some((&m, &q1(&m, 0.9)))); + assert_eq!(g(&t, "class"), cls, "{n} copies, spread {sp}: {t:?}"); + } + let t = plain("lowmsg", 20.0, 30.0, 0.9); + assert!( + g(&t, "message") + .contains("a low count goes with small apo(a) isoforms and a higher Lp(a) level") + ); +} + +#[test] +fn what_is_not_a_count() { + let t = plain("8x", 20.0, 8.0, 0.9); + assert!( + g(&t, "status") == "too little depth" + && g(&t, "kiv2_copies") == "." + && g(&t, "message").contains("8.0x"), + "{t:?}" + ); + assert_eq!(g(&plain("10x", 20.0, 10.0, 0.9), "status"), "estimated"); + let t = plain("nomq0", 20.0, 30.0, 0.0); + assert!( + g(&t, "status") == "not assessed" + && g(&t, "kiv2_copies") == "." + && g(&t, "message").contains("novoalign"), + "{t:?}" + ); + assert_eq!( + g(&plain("quarter", 20.0, 30.0, 0.25), "status"), + "estimated" + ); + let t = assess("nochr6", None); + assert!( + g(&t, "status") == "not assessed" && g(&t, "message").contains("no chr6"), + "{t:?}" + ); + let t = plain("floor", -1.5, 30.0, 0.9); + assert!( + g(&t, "status") == "not assessed" + && g(&t, "kiv2_copies") == "." + && g(&t, "message").contains("too little for any repeat count"), + "{t:?}" + ); +} + +#[test] +fn the_command_line() { + let (rc, out, _) = og_cn(&["bed", "--catalog", &catalog_path(), "--locus", "LPA_KIV2"]); + let rows: Vec> = out.lines().map(|l| l.split('\t').collect()).collect(); + assert!(rc == 0 && rows.len() == 29); + let kiv: Vec<&Vec<&str>> = rows.iter().filter(|r| r[3].starts_with("kiv2_")).collect(); + assert_eq!(kiv.len(), 14); + assert_eq!( + kiv.iter() + .map(|r| r[2].parse::().unwrap() - r[1].parse::().unwrap()) + .sum::(), + 2394 + ); + assert_eq!( + rows.iter().filter(|r| r[3].starts_with("flank_")).count(), + 15 + ); + let (r, w) = (lpa_row(), Scratch::new("cli")); + let (a, b, o) = (w.path("a.tsv"), w.path("b.tsv"), w.path("o.tsv")); + let m = depths(&r, 20.0, 30.0, 0.0, 0.0); + bedcov(&a, &r, &m, false); + bedcov(&b, &r, &q1(&m, 0.9), false); + let (rc, err) = estimate(&a, &b, &o); + let t = table(&o); + assert_eq!(rc, 0, "{err}"); + assert_eq!( + (g(&t, "kiv2_copies"), g(&t, "status")), + ("20.0", "estimated") + ); + assert!(g(&t, "advice").contains("blood")); + std::fs::write(&a, "").unwrap(); + std::fs::write(&b, "").unwrap(); + let (rc, _) = estimate(&a, &b, &o); + assert_eq!( + (rc, g(&table(&o), "status")), + (0, "not assessed"), + "no chr6" + ); + bedcov(&a, &r, &m, false); + let (rc, err) = estimate(&a, &b, &o); + assert!( + rc == 2 && err.contains("is empty but the other table is not"), + "{rc} {err}" + ); + bedcov(&b, &r, &q1(&m, 0.9), false); + let mut text = read(&a); + text += "chr6\t1\t100\tother\t500\n"; + std::fs::write(&a, text).unwrap(); + let (rc, err) = estimate(&a, &b, &o); + assert!( + rc == 2 && err.contains("differ"), + "other regions: {rc} {err}" + ); + std::fs::write(&a, "chr6\t160646213\t160646395\n").unwrap(); + let (rc, err) = estimate(&a, &b, &o); + assert!( + rc == 2 && err.contains("expected chrom"), + "a short row: {rc} {err}" + ); +} + +// 8. KIV-2 4925G>A + +/// mpileup lines at the seven copies; `(c, t)` reads with C and T at each. +fn pileup(path: &str, c: usize, t: usize, extra: &str) { + let mut text: String = COPIES + .iter() + .map(|p| { + format!( + "chr6\t{p}\tN\t{}\t^]{}{}$\t{}\n", + c + t, + "C".repeat(c), + "t".repeat(t), + "I".repeat(c + t) + ) + }) + .collect(); + text += extra; + std::fs::write(path, text).unwrap(); +} + +/// Step 46's table at 38 copies, a pileup, then marker-read: (exit, the +/// 4925G>A table, paths of the LPA table, the pileup and the 4925G>A table). +fn after( + w: &Scratch, + name: &str, + c: usize, + t: usize, + extra: &str, +) -> (u8, HashMap, [String; 3]) { + let r = lpa_row(); + let (a, b, o, p, n) = ( + w.path(&format!("{name}/a.tsv")), + w.path(&format!("{name}/b.tsv")), + w.path(&format!("{name}/S_lpa_kiv2.tsv")), + w.path(&format!("{name}/p.tsv")), + w.path(&format!("{name}/S_lpa_kiv2_4925.tsv")), + ); + let m = depths(&r, 38.0, 30.0, 0.0, 0.0); + bedcov(&a, &r, &m, false); + bedcov(&b, &r, &q1(&m, 0.9), false); + assert_eq!(estimate(&a, &b, &o).0, 0); + pileup(&p, c, t, extra); + let (rc, _, _) = og_cn(&[ + "marker-read", + "LPA_KIV2_4925", + "--catalog", + &catalog_path(), + "--pileup", + &p, + "--copy-table", + &o, + "--out", + &n, + ]); + (rc, table(&n), [o, p, n]) +} + +#[test] +fn kiv2_4925_present_reads_copies_and_the_same_table_through_call() { + let w = Scratch::new("4925"); + let (rc, t, [o, p, n]) = after(&w, "present", 95, 5, ""); + assert_eq!( + (rc, g(&t, "status"), g(&t, "alt_reads"), g(&t, "depth")), + (0, "present", "35", "700") + ); + assert_eq!( + (g(&t, "fraction"), g(&t, "copies"), g(&t, "kiv2_copies")), + ("0.0500", "2.0", "38.0"), + "0.05 x 40" + ); + assert!(g(&t, "message").contains("cannot say which chromosome or which repeat")); + assert_eq!( + read(&n).lines().take(3).collect::>(), + [ + "metric\tvalue", + "marker\tLPA KIV-2 4925G>A", + "rsid\trs1484919544 (first KIV-2 copy)" + ] + ); + // Step 46's table is left byte for byte. + let r = lpa_row(); + let (a0, b0, o0) = ( + w.path("plain/a.tsv"), + w.path("plain/b.tsv"), + w.path("plain/S_lpa_kiv2.tsv"), + ); + let m = depths(&r, 38.0, 30.0, 0.0, 0.0); + bedcov(&a0, &r, &m, false); + bedcov(&b0, &r, &q1(&m, 0.9), false); + assert_eq!(estimate(&a0, &b0, &o0).0, 0); + assert_eq!(read(&o), read(&o0)); + let (n3, o3) = (w.path("viacall.tsv"), w.path("viacall_lpa.tsv")); + let (rc, out, err) = og_cn(&[ + "call", + "--catalog", + &catalog_path(), + "--locus", + "LPA_KIV2", + "--bedcov", + &a0, + "--bedcov-mapq1", + &b0, + "--table", + &o3, + "--marker", + "LPA_KIV2_4925", + "--marker-pileup", + &p, + "--marker-out", + &n3, + ]); + assert_eq!(rc, 0, "{err}"); + assert_eq!( + read(&o3), + read(&o0), + "call --marker writes the same count table" + ); + assert_eq!( + read(&n3), + read(&n), + "and the same 4925G>A table as marker-read" + ); + assert!( + out.lines() + .last() + .unwrap_or_default() + .starts_with("LPA LPA KIV-2 4925G>A"), + "{out}" + ); + // Without step 46's table the reading stands and the copies are '.'. + let (rc, _, _) = og_cn(&[ + "marker-read", + "LPA_KIV2_4925", + "--catalog", + &catalog_path(), + "--pileup", + &p, + "--copy-table", + &w.path("none.tsv"), + "--out", + &n3, + ]); + let t = table(&n3); + assert_eq!( + (rc, g(&t, "status"), g(&t, "copies"), g(&t, "kiv2_copies")), + (0, "present", ".", ".") + ); + for (share, want) in [("0.050", "not assessed"), ("0.900", "present")] { + let lt = w.path(&format!("share{share}.tsv")); + std::fs::write( + <, + format!("metric\tvalue\nstatus\tnot assessed\nkiv2_copies\t.\nmapq0_share\t{share}\n"), + ) + .unwrap(); + let (rc, _, _) = og_cn(&[ + "marker-read", + "LPA_KIV2_4925", + "--catalog", + &catalog_path(), + "--pileup", + &p, + "--copy-table", + <, + "--out", + &n3, + ]); + let t = table(&n3); + assert_eq!((rc, g(&t, "status")), (0, want), "MAPQ 0 share {share}"); + if want != "present" { + assert!( + g(&t, "message").contains("left out the reads that fit several copies"), + "{t:?}" + ); + } + } +} + +#[test] +fn kiv2_4925_statuses_and_what_is_not_counted() { + let w = Scratch::new("4925s"); + let (_, t, _) = after(&w, "ref", 100, 0, ""); + assert_eq!((g(&t, "status"), g(&t, "copies")), ("not seen", ".")); + assert!(g(&t, "message").contains("not proof of absence")); + let (_, t, _) = after(&w, "unc", 100, 0, "chr6\t160640667\tN\t3\tTTT\tIII\n"); + assert_eq!((g(&t, "status"), g(&t, "alt_reads")), ("uncertain", "3")); + let (_, t, _) = after(&w, "thin", 10, 1, ""); + assert_eq!(g(&t, "status"), "too little depth"); + let other = format!( + "chr6\t160640668\tN\t50\t{t}\t.\nchr7\t160640667\tN\t50\t{t}\t.\nchr6\t160635123\tN\t2\tC+2TTC-1T\tII\n", + t = "T".repeat(50) + ); + let (_, t, _) = after(&w, "other", 100, 0, &other); + assert_eq!( + (g(&t, "status"), g(&t, "alt_reads"), g(&t, "depth")), + ("not seen", "0", "702") + ); + let (_, _, [o, p, n]) = after(&w, "bad", 95, 5, ""); + std::fs::write(&p, "not a pileup\n").unwrap(); + let (rc, _, _) = og_cn(&[ + "marker-read", + "LPA_KIV2_4925", + "--catalog", + &catalog_path(), + "--pileup", + &p, + "--copy-table", + &o, + "--out", + &n, + ]); + assert_eq!( + (rc, g(&table(&n), "status")), + (0, "not assessed"), + "an unreadable pileup" + ); + let (rc, _, _) = og_cn(&[ + "marker-read", + "LPA_KIV2_4925", + "--catalog", + &catalog_path(), + "--pileup", + &p, + "--copy-table", + &o, + "--out", + &format!("{}/no/such/dir.tsv", w.0.display()), + ]); + assert_eq!(rc, 2, "an output that cannot be written"); +} + +// 9. the demo's step 46 table + +/// The string the parenthesised literals assigned to `var` in `src` join to. +fn py_str(src: &str, var: &str) -> String { + let start = src + .find(&format!("\n{var} = (")) + .unwrap_or_else(|| panic!("no {var} literal")); + let rest = &src[start..]; + let body = &rest[rest.find('(').unwrap() + 1..rest.find(")\n").unwrap()]; + body.split('"') + .skip(1) + .step_by(2) + .map(|p| p.replace("\\t", "\t").replace("\\n", "\n")) + .collect() +} + +#[test] +fn the_demo_step_46_table_is_what_og_cn_call_writes_for_its_depths() { + let src = read(&format!("{REPO}/tests/demo/make_demo_sample.py")); + let want = py_str(&src, "LPA_TABLE"); + assert!( + want.starts_with("metric\tvalue\nstatus\testimated\n"), + "{want:?}" + ); + // lpa_depth: about 37 KIV-2 repeats at a flank depth near 31, half the + // KIV-2 depth at MAPQ 0. + assert!(src.contains("m = {n: 86.5 * (1.02 if i % 2 == 0 else 0.98)")); + assert!(src.contains("m.update({n: 31.0 * (1.03 if i % 2 == 0 else 0.97)")); + assert!(src.contains("return m, {n: v * (0.5 if n.startswith(\"kiv2_\") else 0.98)")); + let r = lpa_row(); + let mut m = Depth::new(); + for (i, s) in r.targets.iter().enumerate() { + m.insert(s.name.clone(), 86.5 * if i % 2 == 0 { 1.02 } else { 0.98 }); + } + for (i, s) in r.controls.iter().enumerate() { + m.insert(s.name.clone(), 31.0 * if i % 2 == 0 { 1.03 } else { 0.97 }); + } + let q: Depth = m + .iter() + .map(|(n, v)| { + ( + n.clone(), + v * if n.starts_with("kiv2_") { 0.5 } else { 0.98 }, + ) + }) + .collect(); + let w = Scratch::new("demo"); + let (a, b, o) = (w.path("a.tsv"), w.path("b.tsv"), w.path("o.tsv")); + bedcov(&a, &r, &m, true); + bedcov(&b, &r, &q, true); + assert_eq!(estimate(&a, &b, &o).0, 0); + assert_eq!(read(&o), want); +} + +#[test] +fn the_advice_row() { + let t = plain("advice", 20.0, 30.0, 0.9); + assert_eq!(g(&t, "advice"), ADVICE); +} diff --git a/rust/crates/og-cn/tests/genome_markers.rs b/rust/crates/og-cn/tests/genome_markers.rs new file mode 100644 index 000000000..cc65e4d18 --- /dev/null +++ b/rust/crates/og-cn/tests/genome_markers.rs @@ -0,0 +1,348 @@ +//! The copy-aware markers (`og cn marker-regions`, `og cn marker-read +//! SMN1_27134`): the marker checks of the old `tests/test_traits_report.py`, +//! by meaning. +//! +//! 1. Each marker's pinned dbSNP record is one of its copies with its ref +//! and alt; the controls are caught: ref and alt swapped, the minus-strand +//! bases, a dbSNP position one base off the copies (the planted record of +//! the old guard.yml control), and a context whose middle base is not the +//! ref. KIV-2 4925G>A has seven copies. +//! 2. `og cn marker-regions` prints one region per copy, as written here. +//! 3. SMN1 g.27134T>G read from the pileup at both copies beside step 35's +//! SMN table: present, not seen, uncertain, too little depth, not +//! assessed; reads elsewhere are not counted; the SMN1 psCN at exon 7 (or +//! through its homologous region) with its quality; and every message, +//! for every status and psCN, says the marker does not prove 2+0. + +use std::collections::HashMap; +use std::path::PathBuf; + +use og_cn::markers::{LPA_KIV2_4925, MARKERS, Marker, SMN1_27134, marker_errors}; + +/// The copies of each marker, written here as the test's own copy. +const COPIES: [(&str, &[i64]); 2] = [ + ( + "LPA_KIV2_4925", + &[ + 160_646_214, + 160_640_667, + 160_635_123, + 160_629_574, + 160_624_030, + 160_618_484, + 160_612_932, + ], + ), + ("SMN1_27134", &[70_952_074, 70_076_654]), +]; + +const SMN_HEADER: &str = "chrom\tstart\tend\tlocus\tagCN_filter\tagCN\tagCN_qual\tpsCN_filter\tpsCN\tpsCN_qual\thomologous_regions\n"; +const NOT_PROOF: &str = "This marker does not prove 2+0, and its absence does not rule 2+0 out."; + +fn og_cn(args: &[&str]) -> (u8, String, String) { + let args: Vec = args.iter().map(|a| (*a).to_owned()).collect(); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = og_cn::cli::main(&args, &mut out, &mut err); + ( + code, + String::from_utf8(out).unwrap(), + String::from_utf8(err).unwrap(), + ) +} + +fn complement(b: u8) -> u8 { + match b { + b'A' => b'T', + b'C' => b'G', + b'G' => b'C', + b'T' => b'A', + x => x, + } +} + +/// The context with its middle base replaced. +fn with_middle(m: &Marker, base: u8) -> &'static str { + let mut c = m.context.as_bytes().to_vec(); + let mid = c.len() / 2; + c[mid] = base; + Box::leak(String::from_utf8(c).unwrap().into_boxed_str()) +} + +#[test] +fn each_marker_agrees_with_its_dbsnp_record_and_the_controls_are_caught() { + for m in &MARKERS { + assert_eq!(marker_errors(m), Vec::::new(), "{}", m.key); + let mut swapped = *m; + (swapped.reference, swapped.alt) = (m.alt, m.reference); + assert!( + marker_errors(&swapped) + .iter() + .any(|e| e.contains("swapped")), + "{}: swap", + m.key + ); + let mut minus = *m; + (minus.reference, minus.alt) = (complement(m.reference), complement(m.alt)); + minus.context = with_middle(m, complement(m.reference)); + assert!( + marker_errors(&minus) + .iter() + .any(|e| e.contains("minus-strand")), + "{}: minus strand", + m.key + ); + let mut off = *m; + off.dbsnp.2 += 1; + let e = marker_errors(&off); + assert!( + e.iter() + .any(|e| e.contains("not at one of the marker's copies")), + "{}: pos + 1: {e:?}", + m.key + ); + let mut ctx = *m; + ctx.context = with_middle(m, m.alt); + assert!( + marker_errors(&ctx) + .iter() + .any(|e| e.contains("middle base")), + "{}: context", + m.key + ); + } + assert_eq!(LPA_KIV2_4925.copies.len(), 7, "six KIV-2 units and KIV-1"); +} + +#[test] +fn marker_regions_prints_one_region_per_copy() { + for (key, copies) in COPIES { + let m = MARKERS.iter().find(|m| m.key == key).unwrap(); + let (rc, out, err) = og_cn(&["marker-regions", key]); + let want: Vec = copies + .iter() + .map(|p| format!("{}:{p}-{p}", m.chrom)) + .collect(); + assert_eq!( + ( + rc, + out.split_whitespace() + .map(str::to_owned) + .collect::>() + ), + (0, want), + "{key}: {err}" + ); + } +} + +struct Scratch(PathBuf); + +impl Scratch { + fn new(name: &str) -> Scratch { + let d = + std::env::temp_dir().join(format!("og-genome-markers-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + Scratch(d) + } + + fn put(&self, rel: &str, text: &str) -> String { + let p = self.0.join(rel); + std::fs::write(&p, text).unwrap(); + p.to_string_lossy().into_owned() + } + + fn path(&self, rel: &str) -> String { + self.0.join(rel).to_string_lossy().into_owned() + } +} + +impl Drop for Scratch { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +/// A pileup with `t` T reads and `g` G reads at each SMN1 copy. +fn pileup(w: &Scratch, t: usize, g: usize, extra: &str) -> String { + let text: String = SMN1_27134 + .copies + .iter() + .map(|p| { + format!( + "chr5\t{p}\tN\t{}\t^I{}{}$\t.\n", + t + g, + "T".repeat(t), + "g".repeat(g) + ) + }) + .collect(); + w.put("p.tsv", &(text + extra)) +} + +/// Step 35's SMN table with one row over SMN1's exon 7. +fn smn(w: &Scratch, pscn: &str, pqual: &str) -> String { + w.put( + "smn.tsv", + &format!( + "{SMN_HEADER}chr5\t70925000\t70953000\tSMN1\tPASS\t4\t50\tPASS\t{pscn}\t{pqual}\tchr5:70049000-70078000\n" + ), + ) +} + +/// marker-read SMN1_27134 on a pileup and an SMN table: exit and the table. +fn read_marker(w: &Scratch, p: &str, smn_path: &str) -> (u8, HashMap) { + let out = w.path("o.tsv"); + let _ = std::fs::remove_file(&out); + let (rc, _, _) = og_cn(&[ + "marker-read", + "SMN1_27134", + "--pileup", + p, + "--copy-table", + smn_path, + "--out", + &out, + ]); + let t = std::fs::read_to_string(&out) + .unwrap_or_default() + .lines() + .skip(1) + .filter_map(|l| l.split_once('\t')) + .map(|(k, v)| (k.to_owned(), v.to_owned())) + .collect(); + (rc, t) +} + +fn g<'a>(t: &'a HashMap, k: &str) -> &'a str { + t.get(k).map_or("", String::as_str) +} + +#[test] +fn smn1_marker_statuses_and_the_smn1_pscn() { + let w = Scratch::new("smn1"); + let s = smn(&w, "2,2", "30,30"); + let (rc, t) = read_marker(&w, &pileup(&w, 15, 5, ""), &s); + assert_eq!( + ( + rc, + g(&t, "status"), + g(&t, "alt_reads"), + g(&t, "depth"), + g(&t, "fraction") + ), + (0, "present", "10", "40", "0.2500") + ); + assert_eq!( + (g(&t, "copies"), g(&t, "agcn")), + ("1.0", "4"), + "copies = fraction x agCN" + ); + assert_eq!(g(&t, "smn1_pscn"), "2 (quality 30, filter PASS)"); + assert!( + g(&t, "message").contains("higher chance") + && g(&t, "message").contains("does not prove 2+0"), + "{t:?}" + ); + let (_, t) = read_marker(&w, &pileup(&w, 20, 0, ""), &s); + assert!( + g(&t, "status") == "not seen" + && g(&t, "copies") == "." + && g(&t, "message").contains("lower"), + "{t:?}" + ); + let (_, t) = read_marker(&w, &pileup(&w, 20, 0, "chr5\t70952074\tN\t2\tGG\t.\n"), &s); + assert_eq!(g(&t, "status"), "uncertain"); + let (_, t) = read_marker(&w, &pileup(&w, 3, 1, ""), &s); + assert_eq!(g(&t, "status"), "too little depth"); + let other = format!("chr5\t70952075\tN\t30\t{}\t.\n", "G".repeat(30)); + let (_, t) = read_marker(&w, &pileup(&w, 15, 5, &other), &s); + assert_eq!( + g(&t, "alt_reads"), + "10", + "reads at another position are not counted" + ); + let bad = w.put("bad.tsv", "nonsense\n"); + let (rc, t) = read_marker(&w, &bad, &s); + assert_eq!( + (rc, g(&t, "status")), + (0, "not assessed"), + "an unreadable pileup" + ); + let (_, t) = read_marker(&w, &pileup(&w, 15, 5, ""), &smn(&w, "1,3", "30,30")); + assert!(g(&t, "message").contains("SMN1 has 1 copy here"), "{t:?}"); + let (_, t) = read_marker(&w, &pileup(&w, 15, 5, ""), &smn(&w, "2,2", "12,30")); + assert_eq!( + g(&t, "smn1_pscn"), + "2 (quality 12, filter PASS), not resolved" + ); + assert!( + g(&t, "message").contains("SMN1 copy number: 2 (quality 12, filter PASS), not resolved;"), + "{t:?}" + ); + // SMN2's region first: exon 7 of SMN1 is its homologous region. + let s2 = w.put( + "smn2.tsv", + &format!("{SMN_HEADER}chr5\t70049000\t70078000\tSMN1\tPASS\t4\t50\tPASS\t2,2\t30,40\tchr5:70925000-70953000\n"), + ); + let (_, t) = read_marker(&w, &pileup(&w, 15, 5, ""), &s2); + assert_eq!( + (g(&t, "smn1_pscn"), g(&t, "agcn")), + ("2 (quality 40, filter PASS)", "4") + ); + let (_, t) = read_marker(&w, &pileup(&w, 15, 5, ""), &w.path("missing.tsv")); + assert!(g(&t, "smn1_pscn").contains("no SMN table"), "{t:?}"); + let dir = w.0.to_string_lossy().into_owned(); + let (rc, t) = read_marker(&w, &pileup(&w, 15, 5, ""), &dir); + assert_eq!((rc, g(&t, "status"), g(&t, "agcn")), (0, "present", ".")); + assert!(g(&t, "smn1_pscn").contains("could not be read"), "{t:?}"); +} + +#[test] +fn every_smn1_marker_message_says_it_does_not_prove_2_plus_0() { + let w = Scratch::new("smn1-msg"); + let bad = w.put("bad.tsv", "nonsense\n"); + let mut msgs = Vec::new(); + for counts in [ + Some((30, 10)), + Some((40, 0)), + Some((38, 2)), + Some((3, 1)), + None, + ] { + let p = match counts { + None => bad.clone(), + Some((t, g)) => w.put( + "all.tsv", + &format!( + "chr5\t{}\tN\t{}\t{}{}\t.\n", + SMN1_27134.copies[0], + t + g, + "T".repeat(t), + "G".repeat(g) + ), + ), + }; + for path in [Some("2,2"), Some("1,3"), Some("5,5"), None] { + let s = match path { + None => w.path("missing.tsv"), + Some("5,5") => smn(&w, "2,2", "5,5"), + Some(ps) => smn(&w, ps, "30,30"), + }; + msgs.push(g(&read_marker(&w, &p, &s).1, "message").to_owned()); + } + } + assert_eq!(msgs.len(), 20); + let without: Vec<&String> = msgs + .iter() + .filter(|m| !m.contains("does not prove 2+0")) + .collect(); + assert!(without.is_empty(), "{without:?}"); + assert!(msgs[0].contains(NOT_PROOF), "{}", msgs[0]); + assert!( + !msgs[0] + .replace(NOT_PROOF, "") + .contains("does not prove 2+0"), + "control: without the sentence" + ); +} diff --git a/rust/crates/og-cn/tests/port_report_clinical.rs b/rust/crates/og-cn/tests/port_report_clinical.rs new file mode 100644 index 000000000..7dcf9bc56 --- /dev/null +++ b/rust/crates/og-cn/tests/port_report_clinical.rs @@ -0,0 +1,101 @@ +//! The ALT contig refusal of `og cn check-bam` and `og cn call +//! --bam-header`: a header with an ALT or an HLA contig is refused (exit 4, +//! a REFUSED line), a header without one passes (the positive control). +//! +//! This was part 5 of `tests/test_callability.py`; its other parts are in +//! `rust/crates/og-render/tests/clinical.rs`. The Python fed the contig +//! names on stdin (`--header -`); in the process they come from a file, +//! which `check_bam` reads with the same line rule. + +const REPO: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../.."); + +/// `og cn ARGS` in the process: (exit code, stderr). +fn og_cn(args: &[&str]) -> (u8, String) { + let args: Vec = args.iter().map(|a| (*a).to_owned()).collect(); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = og_cn::cli::main(&args, &mut out, &mut err); + (code, String::from_utf8(err).unwrap()) +} + +/// A scratch folder, removed when dropped. +struct Dir(std::path::PathBuf); + +impl Dir { + fn new(name: &str) -> Dir { + let p = std::env::temp_dir().join(format!("og-cn-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&p); + std::fs::create_dir_all(&p).unwrap(); + Dir(p) + } + + fn put(&self, rel: &str, text: &str) -> String { + let p = self.0.join(rel); + std::fs::write(&p, text).unwrap(); + p.to_string_lossy().into_owned() + } +} + +impl Drop for Dir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +#[test] +fn a_header_with_an_alt_or_hla_contig_is_refused() { + let t = Dir::new("alt-refusal"); + let noalt = t.put( + "noalt.txt", + "@HD\tVN:1.6\tSO:coordinate\n@SQ\tSN:chr16\tLN:90338345\n@SQ\tSN:chrM\tLN:16569\n", + ); + let alt = t.put( + "alt.txt", + "@SQ\tSN:chr16\tLN:90338345\n@SQ\tSN:chr22_KI270879v1_alt\tLN:304135\n@RG\tID:x\tSM:y\n", + ); + let (rc, err) = og_cn(&["check-bam", "--header", &noalt]); + assert_eq!( + (rc, err.as_str()), + (0, ""), + "positive control: no ALT contig passes" + ); + let (rc, err) = og_cn(&["check-bam", "--header", &alt]); + assert_eq!(rc, 4, "{err}"); + assert!( + err.starts_with("REFUSED:") && err.contains("chr22_KI270879v1_alt"), + "{err}" + ); + let names = t.put("names.txt", "chr1\nHLA-A*01:01:01:01\n"); + let (rc, err) = og_cn(&["check-bam", "--header", &names]); + assert!( + rc == 4 && err.contains("HLA-A"), + "contig names with an HLA contig: {rc} {err}" + ); + + let bed = t.put("bedcov.tsv", ""); + let catalog = format!("{REPO}/bin/cn_loci.tsv"); + let call = |header: &str| { + og_cn(&[ + "call", + "--catalog", + &catalog, + "--locus", + "HBA", + "--bedcov", + &bed, + "--bedcov-mapq1", + &bed, + "--bam-header", + header, + ]) + }; + let (rc, err) = call(&alt); + assert!( + rc == 4 && err.starts_with("REFUSED:"), + "call refuses before any depth: {rc} {err}" + ); + let (rc, err) = call(&noalt); + assert_eq!( + rc, 0, + "two empty tables: not assessed, and no refusal: {err}" + ); +} diff --git a/rust/crates/og-couple/tests/port_report_clinical.rs b/rust/crates/og-couple/tests/port_report_clinical.rs new file mode 100644 index 000000000..b28fbd562 --- /dev/null +++ b/rust/crates/og-couple/tests/port_report_clinical.rs @@ -0,0 +1,156 @@ +//! The couple screen's set-aside alleles, on synthetic ClinVar hits at real +//! gene positions: a HEXA pseudodeficiency allele and the RBM8A +//! low-expression allele (rs139428292) are never a carrier finding on their +//! own. +//! +//! These were the couple screen checks of two Python tests whose other +//! checks are in `rust/crates/og-render/tests/clinical.rs`: +//! check 7 of `tests/test_acmg_sf_rules.py` and check 9 of +//! `tests/test_cross_engine_biallelic.py`. + +mod common; + +use common::{Dir, og, read, repo, rows}; + +const VCF_HEAD: &str = + "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n"; +const PEDIGREE: &str = + "sample\tsex\tfather\tmother\tpartner\nA\tfemale\t.\t.\tB\nB\tmale\t.\t.\tA\n"; + +/// A ClinVar P hit, reviewed by several submitters. +fn hit(chrom: &str, pos: i64, r: &str, a: &str, gene: &str, gt: &str) -> String { + format!( + "{chrom}\t{pos}\t.\t{r}\t{a}\t50\tPASS\tGENEINFO={gene}:1;CLNSIG=Pathogenic;\ + CLNREVSTAT=criteria_provided,_multiple_submitters,_no_conflicts\tGT:GQ:DP\t{gt}:50:30\n" + ) +} + +/// test_acmg_sf_rules.py check 7, the couple screen part. +#[test] +fn a_hexa_pseudodeficiency_allele_is_set_aside_and_the_screen_goes_on() { + let t = Dir::new("clinical-hexa"); + let hits = t.put( + "S_clinvar_hits.vcf", + &format!( + "{VCF_HEAD}{}{}{}", + hit("chr7", 117559590, "ATCT", "A", "CFTR", "0/1"), + hit("chr15", 72350584, "G", "A", "HEXA", "0/1"), + hit("chr15", 72346579, "G", "GGATA", "HEXA", "0/1") + ), + ); + let ped = t.put("pedigree.tsv", PEDIGREE); + let genes = repo("assets/carrier_genes.tsv"); + let out = t.path("out"); + let (a, b) = (format!("A={hits}"), format!("B={hits}")); + let (rc, log) = og( + "screen", + &[ + "--pedigree", + &ped, + "--genes", + &genes, + "--clinvar", + &a, + "--clinvar", + &b, + "--out-dir", + &out, + ], + ); + let md = read(&format!("{out}/couple_A_B.md")); + let mut found: Vec = rows(&format!("{out}/couple_A_B.tsv")) + .iter() + .map(|r| r["gene"].clone()) + .collect(); + found.sort(); + let aside: Vec<&str> = md + .lines() + .filter(|x| x.contains("Set aside for A")) + .collect(); + assert_eq!(found, ["CFTR", "HEXA"], "{log}"); + assert_eq!(aside.len(), 1, "{md}"); + assert!(aside[0].contains("pseudodeficiency"), "{}", aside[0]); + assert_eq!(rc, 0, "{log}"); + assert!(md.contains("Set aside for A, not a carrier finding on its own")); +} + +// RBM8A c.-21G>A (rs139428292), on the plus strand C>T. +const TAR: (&str, i64, &str, &str) = ("chr1", 145927447, "C", "T"); + +/// test_cross_engine_biallelic.py check 9. +#[test] +fn the_rbm8a_allele_counts_only_against_a_deletion_in_the_partner() { + let t = Dir::new("clinical-tar"); + let tar = t.put( + "tar.vcf", + &format!( + "{VCF_HEAD}{}", + hit(TAR.0, TAR.1, TAR.2, TAR.3, "RBM8A", "0/1") + ), + ); + let empty = t.put("empty.vcf", VCF_HEAD); + let ped = t.put("pedigree.tsv", PEDIGREE); + // B's null allele: a 300 kb Delly deletion over RBM8A's exons (no RDCN, + // so read depth is no question). + let null_sv = t.put( + "B_sv.vcf", + "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n\ + chr1\t145700000\tDEL0\tN\t\t.\tPASS\tSVTYPE=DEL;END=146000000\tGT\t0/1\n", + ); + let exons: String = (145921000..145928000) + .step_by(1200) + .map(|s| format!("chr1\t{s}\t{}\n", s + 100)) + .collect(); + let exons = t.put("exons.bed", &exons); + let genes = repo("assets/carrier_genes.tsv"); + let a = format!("A={tar}"); + let couple = |name: &str, b_hits: &str, extra: &[&str]| { + let out = t.path(name); + let b = format!("B={b_hits}"); + let mut args = vec![ + "--pedigree", + &ped, + "--genes", + &genes, + "--clinvar", + &a, + "--clinvar", + &b, + ]; + args.extend_from_slice(extra); + args.extend_from_slice(&["--out-dir", &out]); + let (rc, log) = og("screen", &args); + let (rows, md) = if rc == 0 { + ( + rows(&format!("{out}/couple_A_B.tsv")), + read(&format!("{out}/couple_A_B.md")), + ) + } else { + (Vec::new(), String::new()) + }; + (rc, log, rows, md) + }; + + let delly = format!("B={null_sv}"); + let (_, log, got, _) = couple("null", &empty, &["--delly", &delly, "--exons", &exons]); + let x: Vec<_> = got.iter().filter(|r| r["gene"] == "RBM8A").collect(); + assert_eq!(x.len(), 1, "{log}"); + assert_eq!(x[0]["result"], "shared"); + assert_eq!(x[0]["offspring_risk"], "1 in 4 per pregnancy"); + assert!(x[0]["findings_a"].contains("rs139428292")); + assert!(x[0]["explanation"].contains("hypomorph")); + + let (rc, log, got, _) = couple("both", &tar, &[]); + assert_eq!(rc, 0, "{log}"); + assert!( + got.iter().all(|r| r["gene"] != "RBM8A"), + "the allele in both partners is set aside" + ); + let (rc, log, got, md) = couple("one", &empty, &[]); + assert_eq!(rc, 0, "{log}"); + assert!(got.iter().all(|r| r["gene"] != "RBM8A"), "the allele alone"); + assert!( + md.contains("| A | 0 / 1 |") && md.contains("Set aside for A"), + "no finding of the listed genes (0 / 1), and a gap line: {md}" + ); +} diff --git a/rust/crates/og-render/Cargo.toml b/rust/crates/og-render/Cargo.toml index 5112b56a7..465c627d8 100644 --- a/rust/crates/og-render/Cargo.toml +++ b/rust/crates/og-render/Cargo.toml @@ -20,5 +20,14 @@ og-io.workspace = true og-summary.workspace = true indexmap.workspace = true +[dev-dependencies] +# The PGx tests validate their summaries with og dev schema's rule. +og-dev.workspace = true +# The ancestry chapter's tests (tests/ancestry_*.rs) run each step's own +# verb, then collect and render its result, as the run does. +og-ancestry.workspace = true +og-reports.workspace = true +flate2.workspace = true + [lints] workspace = true diff --git a/rust/crates/og-render/src/chapters.rs b/rust/crates/og-render/src/chapters.rs index 7f468bead..ea46924ed 100644 --- a/rust/crates/og-render/src/chapters.rs +++ b/rust/crates/og-render/src/chapters.rs @@ -159,17 +159,17 @@ pub fn writer(sid: &str) -> Option<(TextFn, HtmlFn)> { match sid { "qc" => Some((sections::qc::text, sections::qc::html)), - "callability" => None, + "callability" => Some((sections::callability::text, sections::callability::html)), "variants" => Some((sections::variants::text, sections::variants::html)), - "clinvar" => None, + "clinvar" => Some((sections::clinvar::text, sections::clinvar::html)), - "clinical-filter" => None, + "clinical-filter" => Some((sections::acmg_sf::text, sections::acmg_sf::html)), - "cpsr" => None, + "cpsr" => Some((sections::cpsr::text, sections::cpsr::html)), - "slivar" => None, + "slivar" => Some((sections::comphet::text, sections::comphet::html)), "exomiser" => Some((sections::exomiser::text, sections::exomiser::html)), @@ -177,13 +177,13 @@ pub fn writer(sid: &str) -> Option<(TextFn, HtmlFn)> { "mei" => Some((sections::mei::text, sections::mei::html)), - "expansions" => None, + "expansions" => Some((sections::str::text, sections::str::html)), "vntyper" => Some((sections::vntyper::text, sections::vntyper::html)), - "mito" => None, + "mito" => Some((sections::mito::text, sections::mito::html)), - "lpa" => None, + "lpa" => Some((sections::lpa::text, sections::lpa::html)), "chip" => Some((sections::chip::text, sections::chip::html)), @@ -191,39 +191,42 @@ pub fn writer(sid: &str) -> Option<(TextFn, HtmlFn)> { "mosaic-mca" => Some((sections::mocha::text, sections::mocha::html)), - "carrier-list" => None, + "carrier-list" => Some((sections::carrier::text, sections::carrier::html)), - "couple" => None, + "couple" => Some((sections::couple::text, sections::couple::html)), - "paralogs" => None, + "paralogs" => Some((sections::paralogs::text, sections::paralogs::html)), - "pharmcat" => None, + "pharmcat" => Some((sections::pgx::pharmcat_text, sections::pgx::pharmcat_html)), - "cpic" => None, + "cpic" => Some((sections::pgx::cpic_text, sections::pgx::cpic_html)), - "cyp2d6" => None, + "cyp2d6" => Some((sections::pgx::cyp2d6_text, sections::pgx::cyp2d6_html)), - "hla" => None, + "hla" => Some((sections::hla::text, sections::hla::html)), "locityper" => Some((sections::locityper::text, sections::locityper::html)), - "ancestry-fine" => None, + "ancestry-fine" => Some((sections::ancestry_fine::text, sections::ancestry_fine::html)), - "mt-haplogroup" => None, + "mt-haplogroup" => Some((sections::haplogroups::text, sections::haplogroups::html)), - "y-chromosome" => None, + "y-chromosome" => Some(( + sections::haplogroups::y_chromosome_text, + sections::haplogroups::y_chromosome_html, + )), - "archaic" => None, + "archaic" => Some((sections::archaic::text, sections::archaic::html)), - "ancient" => None, + "ancient" => Some((sections::ancient::text, sections::ancient::html)), "roh" => Some((sections::roh::text, sections::roh::html)), - "traits-panel" => None, + "traits-panel" => Some((sections::traits::text, sections::traits::html)), - "prs" => None, + "prs" => Some((sections::prs::text, sections::prs::html)), - "bloodtype" => None, + "bloodtype" => Some((sections::bloodtype::text, sections::bloodtype::html)), "telomere" => Some((sections::telomere::text, sections::telomere::html)), diff --git a/rust/crates/og-render/src/html.rs b/rust/crates/og-render/src/html.rs index 73014859f..efccdf40c 100644 --- a/rust/crates/og-render/src/html.rs +++ b/rust/crates/og-render/src/html.rs @@ -10,7 +10,8 @@ use crate::chapters::{CHAPTERS, FrameState, LIMITS, Sec, step_of, writer}; use crate::css::CSS; use crate::escape::e; use crate::model::{ - Card, Cell, Colour, HtmlPart, Inline, Line, Malformed, Section, State, Summary, Table, py, + Card, Cell, Colour, HtmlPart, Inline, Layout, Line, Malformed, Section, State, Summary, Table, + py, }; use crate::notes::{VIEWER_NOTE, stale_note}; use crate::sections::run; @@ -56,6 +57,11 @@ fn inline(v: &Inline) -> String { "{}", e(t) ), + Inline::Break => "
".to_owned(), + Inline::Small(t) => format!( + "
{}", + e(t) + ), } } @@ -74,6 +80,16 @@ fn line(out: &mut Vec, l: &Line) { out.extend(items.iter().map(|t| format!("
  • {}
  • ", e(t)))); out.push(" ".to_owned()); } + Line::Warn(t) => out.push(format!("
    {}
    ", e(t))), + Line::ViewerNote => out.push(VIEWER_NOTE.to_owned()), + Line::Folded(summary, t) => { + out.push(format!("
    {}", e(summary))); + table(out, t); + if let Some(last) = out.last_mut() { + last.push_str("
    "); + } + } + Line::Raw(t) => out.push(t.clone()), } } @@ -84,11 +100,8 @@ fn cell(c: &Cell) -> String { match c { Cell::Text(t) => format!("{}", e(t)), Cell::Badge(t, colour) => format!("{}", badge(t, *colour)), - Cell::Sub(t, sub) => format!( - "{}
    {}", - e(t), - e(sub) - ), + Cell::Sub(t, sub) => format!("{}{}", e(t), inline(&Inline::Small(sub.clone()))), + Cell::Parts(parts) => format!("{}", parts.iter().map(inline).collect::()), Cell::Viewer(None) => "".to_owned(), Cell::Viewer(Some(at)) => format!( "{0}", @@ -110,19 +123,23 @@ fn table(out: &mut Vec, t: &Table) { .rows .iter() .map(|r| format!("{}", r.iter().map(cell).collect::())); - if t.one_line { - out.push(format!( + match t.layout { + Layout::OneLine => out.push(format!( " {head}{}
    ", rows.collect::() - )); - } else { - out.push(" ".to_owned()); - out.push(format!(" {head}")); - out.extend(rows.map(|r| format!(" {r}"))); - out.push("
    ".to_owned()); - } - if t.viewer { - out.push(VIEWER_NOTE.to_owned()); + )), + Layout::Indented => { + out.push(" ".to_owned()); + out.push(format!(" {head}")); + out.extend(rows.map(|r| format!(" {r}"))); + out.push("
    ".to_owned()); + } + Layout::RowLines => { + out.push(" ".to_owned()); + out.push(format!(" {head}")); + out.push(rows.collect::>().join("\n")); + out.push("
    ".to_owned()); + } } } @@ -194,8 +211,23 @@ fn meta( Ok(out) } +/// A card that shows a Viewer column with rows also says how the viewer +/// opens: each section places [`Line::ViewerNote`] itself, so this catches +/// one that forgot. +fn viewer_note_placed(c: &Card) -> bool { + let shows_viewer = c.body.iter().any(|l| match l { + Line::Table(t) | Line::Folded(_, t) => t.viewer && !t.rows.is_empty(), + _ => false, + }); + !shows_viewer || c.body.contains(&Line::ViewerNote) +} + /// One section's frame. -fn frame(sid: &str, part: &HtmlPart, meta: Option, always: bool) -> Vec { +pub(crate) fn frame(sid: &str, part: &HtmlPart, meta: Option, always: bool) -> Vec { + debug_assert!( + part.cards.iter().all(viewer_note_placed), + "{sid}: a card shows a Viewer column without the line saying how it opens" + ); let wide = part.cards.len() > 1 || part.cards.iter().any(|c| c.full); let mut out = vec![format!( "
    ", diff --git a/rust/crates/og-render/src/lib.rs b/rust/crates/og-render/src/lib.rs index 839f2be52..83e46ab58 100644 --- a/rust/crates/og-render/src/lib.rs +++ b/rust/crates/og-render/src/lib.rs @@ -10,9 +10,9 @@ //! [`css`] the page's style. //! //! Ported so far: the frame (chapters, contents, section frames, the run -//! facts, what the report leaves out) and the quality-control and variant -//! sections. A section not ported yet is left out of both reports; the index -//! and charts verbs say they are not ported and exit 2. +//! facts, what the report leaves out) and the sections +//! [`chapters::writer`] names. A section not ported yet is left out of both +//! reports; the index and charts verbs say they are not ported and exit 2. pub mod chapters; pub mod charts; diff --git a/rust/crates/og-render/src/model.rs b/rust/crates/og-render/src/model.rs index 761afdb8b..939938781 100644 --- a/rust/crates/og-render/src/model.rs +++ b/rust/crates/og-render/src/model.rs @@ -3,8 +3,9 @@ //! `==`), and the document the section modules build from it, which the //! text and HTML writers lay out. //! -//! A section module never writes markup. It returns cards of typed lines; -//! every text in them is escaped once, by the HTML writer. +//! A section module returns cards of typed lines; every text in them is +//! escaped once, by the HTML writer. The one exception is [`Line::Raw`]: +//! markup the section lays out itself, escaping each text it puts in. use std::fmt; @@ -315,6 +316,11 @@ pub enum Inline { /// A fixed text written as it is, unescaped, as render_report.py's /// stat() writes its value (an apostrophe stays an apostrophe). Fixed(&'static str), + /// A line break. + Break, + /// A line break, then smaller grey text: the second line under a + /// table cell's main text. + Small(String), } /// One cell of a table row. @@ -326,6 +332,8 @@ pub enum Cell { Badge(String, Colour), /// Text with a second, smaller grey line under it. Sub(String, String), + /// Texts, badges and smaller lines one after another in one cell. + Parts(Vec), /// The Viewer cell: a link that opens the locus (`chrom:start` or /// `chrom:start-end`) in the genome viewer; empty without a locus. Viewer(Option), @@ -351,14 +359,30 @@ impl Cell { } /// A table: its column titles and rows. A table with a Viewer column (the -/// rows end in a [`Cell::Viewer`]) is followed by the line saying how the -/// viewer opens. `one_line` writes the whole table on one line of the page. +/// rows end in a [`Cell::Viewer`]) has `viewer`; the card places the line +/// saying how the viewer opens, [`Line::ViewerNote`], where the report has +/// it; a card with such a table and no such line fails a debug check in the +/// HTML writer. `layout` says how its tags and rows sit on the page's lines. #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct Table { pub head: Vec, pub rows: Vec>, pub viewer: bool, - pub one_line: bool, + pub layout: Layout, +} + +/// How a table is laid out in the page's source. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Layout { + /// The table tags and the head row on lines of their own, each row on a + /// line of its own, indented under the head. + #[default] + Indented, + /// The whole table on one line. + OneLine, + /// The table tags and the head row on lines of their own, the rows on + /// one line each with no indent. + RowLines, } /// One line of a card's body. @@ -374,6 +398,17 @@ pub enum Line { Para(String), /// A plain bulleted list. List(Vec), + /// A warning in the stale box's style, among the card's lines. + Warn(String), + /// The sentence that says how a table's Viewer column opens the reads. + ViewerNote, + /// A table folded away behind a one-line summary, which the reader opens. + Folded(String, Table), + /// Markup a section built itself, each text in it escaped with + /// [`crate::escape::e`]: a heading, a picture, a link or an anchor whose + /// layout the report fixes byte for byte. Written as it is. A table is a + /// [`Line::Table`], never markup. + Raw(String), } /// One card: a title, the caveats shown under it, the notes saying its diff --git a/rust/crates/og-render/src/notes.rs b/rust/crates/og-render/src/notes.rs index e94157408..e04bc59f7 100644 --- a/rust/crates/og-render/src/notes.rs +++ b/rust/crates/og-render/src/notes.rs @@ -17,7 +17,7 @@ pub const CALLABILITY_NOTE: &str = "A gene listed as not read could not be read pub const HLA_NOTE: &str = "HLA typing from short-read WGS is approximate. If either HLA-A or HLA-B allele has a \ T1K quality of 0 or less, that gene is not passed to PharmCAT (step 36)."; pub const EH_NOTE: &str = "Repeat-expansion calls from short reads can be wrong at some loci, and a long expansion \ - cannot be sized. Disease thresholds: docs/interpreting-results.md."; + cannot be sized. Disease thresholds: docs/results/structural-and-repeats.md."; pub const TELOMERE_NOTE: &str = "Relative telomere content (intratelomeric reads per million reads with 48-52% GC), not \ a telomere length: compare it only with samples of a similar age sequenced on the same \ platform (docs/10-telomere-analysis.md)."; diff --git a/rust/crates/og-render/src/sections/acmg_sf.rs b/rust/crates/og-render/src/sections/acmg_sf.rs index 70669f6e2..800535851 100644 --- a/rust/crates/og-render/src/sections/acmg_sf.rs +++ b/rust/crates/og-render/src/sections/acmg_sf.rs @@ -1,2 +1,329 @@ -//! The report section `acmg_sf`. Empty until its unit of the 1.0.0 plan -//! lands. +//! Step 23's clinical filter and the ACMG secondary-findings tier: the +//! counts of the filtered variants, then each SF gene's verdict by its +//! reporting rule (`assets/acmg_sf.tsv`): the ClinVar P/LP hits, the rare +//! HIGH-impact records and the class 4/5 SVs it reports, and the variants +//! it does not report, each with the reason. Every phrase comes from the +//! summary as `og summary collect` wrote it. + +use pycompat::fmt::ljust; + +use crate::model::{Card, Cell, HtmlPart, Inline, Layout, Line, Malformed, Summary, Table}; +use crate::notes::ACMG_NOTE; +use crate::sections::clinvar::{D, viewer_of}; +use crate::{html, text}; + +/// One row of the SF table: gene, variant, genotype, evidence. +pub type Row = (String, String, String, String); + +/// The HIGH-impact records counted as rare: neither a weak call nor of +/// unknown frequency (a summary written before those marks counts every +/// record). +pub fn high_rare(sf: &D<'_>) -> Result { + let all = sf.at("high_impact")?.len()?; + match sf.get("high_impact_rare")? { + Some(n) => n.text(), + None => Ok(all.to_string()), + } +} + +/// The HIGH-impact records listed but not counted as rare. +pub fn high_apart(sf: &D<'_>) -> Result { + let mut parts = Vec::new(); + if let Some(n) = sf.opt("high_impact_weak")? { + parts.push(format!("{} weak call(s)", n.text()?)); + } + if let Some(n) = sf.opt("high_impact_frequency_unknown")? { + parts.push(format!("{} of unknown frequency", n.text()?)); + } + Ok(if parts.is_empty() { + String::new() + } else { + format!(" (listed apart: {})", parts.join(", ")) + }) +} + +/// The marks of one HIGH-impact record (unknown frequency, weak call), each +/// after `lead`; `""` for a rare record with a sound call. +pub fn high_marks(x: &D<'_>, lead: &str) -> Result { + let mut marks = Vec::new(); + if x.flag("frequency_unknown")? { + marks.push("frequency unknown, not counted as rare".to_owned()); + } + if x.flag("weak_call")? { + marks.push(format!( + "weak call ({}): check the reads", + x.s_or_none("call_note")? + )); + } + if marks.is_empty() { + return Ok(String::new()); + } + let close = if lead.ends_with('(') { ")" } else { "" }; + Ok(format!("{lead}{}{close}", marks.join("; "))) +} + +/// The severity, rule and two-hit notes of a row. +fn extra(x: &D<'_>) -> Result { + let mut out = String::new(); + for k in ["severity", "rule_note", "two_hit"] { + if let Some(v) = x.opt(k)? { + out += &format!("; {}", v.text()?); + } + } + Ok(out) +} + +/// The secondary findings as rows, the same in both reports: ClinVar hits, +/// rare HIGH-impact records, then class 4 and 5 SVs. +pub fn rows(sf: &D<'_>) -> Result, Malformed> { + let mut out = Vec::new(); + for x in sf.at("clinvar_hits")?.items()? { + let low = if x.flag("low_confidence")? { + "; low-confidence call: check the reads" + } else { + "" + }; + out.push(( + x.s("gene")?, + x.s("variant")?, + x.s("genotype")?, + format!( + "ClinVar {}{low}{}", + x.at("significance")?.string()?, + extra(&x)? + ), + )); + } + for x in sf.at("high_impact")?.items()? { + out.push(( + x.s("gene")?, + x.s("variant")?, + x.s("genotype")?, + format!( + "{}{}{}", + x.at("consequence")?.string()?, + high_marks(&x, "; ")?, + extra(&x)? + ), + )); + } + if let Some(sv) = sf.get("structural")? { + for x in sv.items()? { + let support = x.at("support")?; + let support = if support.is_none() { + "caller support not known (no step 22 consensus)".to_owned() + } else if support.is_num(1) { + format!("single caller ({})", x.s("caller")?) + } else { + format!("{} callers", support.text()?) + }; + let via = match x.opt("via")? { + Some(v) => format!("; through {}", v.text()?), + None => String::new(), + }; + out.push(( + x.s("gene")?, + x.s("variant")?, + ".".to_owned(), + format!( + "AnnotSV class {}, {support}{via}{}", + x.s("acmg_class")?, + extra(&x)? + ), + )); + } + } + Ok(out) +} + +/// The variants in SF genes the gene's rule does not report, or why no +/// rule could be applied. +pub fn apart_lines(sf: &D<'_>) -> Result, Malformed> { + if sf.flag("unavailable")? { + return Ok(vec![format!( + "Secondary findings: not assessed, {}", + sf.s("unavailable")? + )]); + } + let mut out = Vec::new(); + if let Some(nr) = sf.get("not_reported")? { + for x in nr.items()? { + let gt = match x.opt("genotype")? { + Some(g) => g.text()?, + None => ".".to_owned(), + }; + out.push(format!( + "{} {} {gt} ({}): not reported, {}", + x.s("gene")?, + x.s("variant")?, + x.s("source")?, + x.s("reason")? + )); + } + } + if let Some(n) = sf.opt("two_hit_note")? { + out.push(n.text()?); + } + Ok(out) +} + +/// The text report's block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "clinical", Some("Clinical Variant Filter"))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if !head.ran { + return Ok(w); + } + let d = D::of(&s.section("clinical")); + let genes = if d.has("genes") { + format!(" in {} genes", d.s("genes")?) + } else { + String::new() + }; + w.push(format!(" Clinical variants: {}{genes}", d.s("variants")?)); + if let Some(by) = d.opt("by_impact")? { + let mut pairs = by.pairs()?; + pairs.sort_by(|a, b| a.0.cmp(b.0)); + let parts: Vec = pairs + .iter() + .map(|(k, v)| Ok(format!("{k} {}", v.text()?))) + .collect::>()?; + w.push(format!(" By impact: {}", parts.join(", "))); + } + if d.has("frequency_unknown") { + w.push(format!( + " Frequency unknown (not counted as rare): {}", + d.s("frequency_unknown")? + )); + } + if d.has("weak_calls") { + w.push(format!( + " Weak genotype calls (check the reads): {}", + d.s("weak_calls")? + )); + } + if let Some(sf) = d.opt("acmg_sf")? { + if sf.flag("unavailable")? { + w.push(format!(" {}", apart_lines(&sf)?[0])); + } else { + let mut line = format!( + " Secondary-findings genes ({}, {} genes): {} ClinVar P/LP, {} rare HIGH impact{}", + sf.s("version")?, + sf.s_or("genes_on_list", "?")?, + sf.at("clinvar_hits")?.len()?, + high_rare(&sf)?, + high_apart(&sf)? + ); + if sf.flag("structural_checked")? { + line += &format!(", {} class 4/5 SV(s)", sf.at("structural")?.len()?); + } + if let Some(nr) = sf.opt("not_reported")? { + line += &format!(", {} not reported by the gene's rule", nr.len()?); + } + w.push(line); + for (g, v, z, ev) in rows(&sf)? { + w.push(format!(" {} {v} {z} {ev}", ljust(&g, 8))); + } + for line in apart_lines(&sf)? { + w.push(format!(" {line}")); + } + w.push(format!(" {ACMG_NOTE}")); + } + } + w.push(String::new()); + Ok(w) +} + +/// The counts card, always there; a stat's missing count prints `None`. +fn counts_card(d: &D<'_>) -> Result { + let stat = |label: String, v: String| Line::Stat(label, Inline::Text(v)); + let mut body = vec![stat( + "Total interesting variants".to_owned(), + d.s_or_none("variants")?, + )]; + if d.has("genes") { + body.push(stat("Genes".to_owned(), d.s("genes")?)); + } + if let Some(by) = d.opt("by_impact")? { + let mut pairs = by.pairs()?; + pairs.sort_by(|a, b| a.0.cmp(b.0)); + for (k, v) in pairs { + body.push(stat(format!("{k} impact"), v.text()?)); + } + } + if d.has("frequency_unknown") { + body.push(stat( + "Frequency unknown (not rare)".to_owned(), + d.s("frequency_unknown")?, + )); + } + if d.has("weak_calls") { + body.push(stat("Weak genotype calls".to_owned(), d.s("weak_calls")?)); + } + Ok(Card { + title: "Clinical Variant Filter".to_owned(), + body, + ..Card::default() + }) +} + +/// The SF card, when the tier lists anything or says why it could not. +fn sf_card(sf: &D<'_>) -> Result, Malformed> { + let shown = sf.at("clinvar_hits")?.truthy() + || sf.at("high_impact")?.truthy() + || sf.flag("structural")? + || sf.flag("not_reported")? + || sf.flag("unavailable")?; + if !shown { + return Ok(None); + } + let mut cells = Vec::new(); + for (g, v, z, ev) in rows(sf)? { + let at = viewer_of(&v)?; + cells.push(vec![ + Cell::Text(g), + Cell::Text(v), + Cell::Text(z), + Cell::Text(ev), + at, + ]); + } + let listed = !cells.is_empty(); + let mut body = vec![Line::Table(Table { + head: ["Gene", "Variant", "Genotype", "Evidence"] + .map(str::to_owned) + .to_vec(), + rows: cells, + viewer: true, + layout: Layout::RowLines, + })]; + body.extend(apart_lines(sf)?.into_iter().map(Line::Para)); + if listed { + body.push(Line::ViewerNote); + } + Ok(Some(Card { + title: format!("Secondary-Findings Genes ({})", sf.s("version")?), + full: true, + caveats: vec![ACMG_NOTE], + body, + ..Card::default() + })) +} + +/// The HTML report's cards: the counts, then the SF table. +pub fn html(s: &Summary) -> Result { + let sec = s.section("clinical"); + let d = D::of(&sec); + let mut cards = vec![html::card(&sec, counts_card(&d)?)?]; + if let Some(sf) = d.opt("acmg_sf")? + && let Some(card) = sf_card(&sf)? + { + cards.push(html::card(&sec, card)?); + } + Ok(HtmlPart { + cards, + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/ancestry.rs b/rust/crates/og-render/src/sections/ancestry.rs index f882faadb..4f5d38853 100644 --- a/rust/crates/og-render/src/sections/ancestry.rs +++ b/rust/crates/og-render/src/sections/ancestry.rs @@ -1,2 +1,202 @@ -//! The report section `ancestry`. Empty until its unit of the 1.0.0 plan -//! lands. +//! What the ancestry chapter's sections share: reading a row of the +//! summary the way the Python report did (`r["key"]`, `f"{v:<35}"`, a list +//! walked with `for`), the percentages of the fine-grained ancestry, and the +//! links where a haplogroup can be explored further (steps 12 and 37). +//! +//! Used by the fine-grained ancestry, haplogroup, Y chromosome, archaic, +//! ancient and polygenic score sections. + +use pycompat::fmt::{fixed, ljust, rjust}; +use pycompat::json::Value; + +use crate::escape::e; +use crate::html::offsite_link; +use crate::model::{Line, Malformed, get, need, num, py, truthy}; +use crate::notes::EXPLORE_NOTE; + +/// Every explore address but the docs one carries the haplogroup in it. +pub const EXPLORE_DOCS: &str = "https://github.com/GeiserX/OwnGenome/blob/main/docs/"; + +/// The pages under [`EXPLORE_DOCS`] the collector links, one per haplogroup. +const EXPLORE_DOC_PAGES: [&str; 2] = [ + "37-y-haplogroup.md#explore-your-branch-further", + "12-mito-haplogroup.md#explore-your-branch-further", +]; + +/// The mark of an address that leaves this computer with the haplogroup in it. +pub const OFFSITE_HAPLOGROUP: &str = "opens a website; the address carries the haplogroup"; + +/// `str(v)` of an optional value: `None` when it is absent. +pub fn py_or(v: Option<&Value>, default: &str) -> Result { + v.map_or(Ok(default.to_owned()), py) +} + +/// `d.get(key, default)` as text: the default only when the key is absent. +pub fn get_text(d: &Value, key: &str, default: &str, path: &str) -> Result { + py_or(get(d, key, path)?, default) +} + +/// `str(d[key])`. +pub fn need_text(d: &Value, key: &str, path: &str) -> Result { + py(need(d, key, path)?) +} + +/// The items `for x in v` walks: a list's items, a string's characters, a +/// dict's keys. `None` and numbers cannot be walked. +pub fn items(v: &Value, path: &str) -> Result, Malformed> { + match v { + Value::Array(a) => Ok(a.clone()), + Value::Str(s) => Ok(s.chars().map(|c| Value::Str(c.to_string())).collect()), + Value::Object(o) => Ok(o.keys().map(|k| Value::Str(k.clone())).collect()), + _ => Err(Malformed(format!("{path} is not a list"))), + } +} + +/// `d.get(key, [])` walked with `for`: nothing when the key is absent. +pub fn list(d: &Value, key: &str, path: &str) -> Result, Malformed> { + match get(d, key, path)? { + None => Ok(Vec::new()), + Some(v) => items(v, &format!("{path}.{key}")), + } +} + +/// `len(v)`. +pub fn py_len(v: &Value, path: &str) -> Result { + match v { + Value::Array(a) => Ok(a.len()), + Value::Str(s) => Ok(s.chars().count()), + Value::Object(o) => Ok(o.len()), + _ => Err(Malformed(format!("{path} has no length"))), + } +} + +/// Which side `f"{v:N}"` (right) pads. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Align { + Left, + Right, +} + +/// `f"{v:N}"`: a string or a number padded to `width` +/// characters. A bool formats as the number it is, as in Python; `None`, a +/// list or a dict cannot take a width. +pub fn align(v: &Value, width: usize, side: Align, path: &str) -> Result { + let text = match v { + Value::Str(s) => s.clone(), + Value::Int(i) => i.to_string(), + Value::Float(_) => py(v)?, + Value::Bool(b) => u8::from(*b).to_string(), + _ => return Err(Malformed(format!("{path} cannot be padded"))), + }; + Ok(match side { + Align::Left => ljust(&text, width), + Align::Right => rjust(&text, width), + }) +} + +/// `f"{d[key]:N}"`. +pub fn need_align( + d: &Value, + key: &str, + width: usize, + side: Align, + path: &str, +) -> Result { + align(need(d, key, path)?, width, side, &format!("{path}.{key}")) +} + +/// A number as Python compares it with `>=`; anything else is an error. +pub fn number(v: &Value, path: &str) -> Result { + num(Some(v)).ok_or_else(|| Malformed(format!("{path} is not a number"))) +} + +/// A fraction as a whole percentage (`95%`), `.` for none. +pub fn share(v: Option<&Value>, path: &str) -> Result { + match v { + None | Some(Value::Null) => Ok(".".to_owned()), + Some(x) => Ok(format!("{}%", fixed(100.0 * number(x, path)?, 0))), + } +} + +/// A stat line whose value is markup built here. +pub fn stat_raw(label: &str, value: &str) -> Line { + Line::Raw(format!( + "
    {}{value}
    ", + e(label) + )) +} + +/// One explore entry of a haplogroup section: its name and address. +fn explore(d: &Value, path: &str) -> Result, Malformed> { + let at = format!("{path}.explore[]"); + list(d, "explore", path)? + .iter() + .map(|x| Ok((need_text(x, "name", &at)?, need(x, "url", &at)?.clone()))) + .collect() +} + +/// The text report's explore lines: `name: url` for each link, then the +/// note that nothing is uploaded; nothing when there is no link. +pub fn explore_lines(d: &Value, path: &str) -> Result, Malformed> { + let mut out = Vec::new(); + for (name, url) in explore(d, path)? { + out.push(format!("{name}: {}", py(&url)?)); + } + if !out.is_empty() { + out.push(EXPLORE_NOTE.to_owned()); + } + Ok(out) +} + +/// The HTML report's explore lines: one stat per link, each marked as +/// leaving this computer, and whether the card shows the explore caveat. +/// The address is escaped, so a branch name cannot open a tag or end the +/// attribute. +pub fn explore_html(d: &Value, path: &str) -> Result<(Vec, bool), Malformed> { + let mut out = Vec::new(); + for (name, url) in explore(d, path)? { + let Value::Str(url) = url else { + return Err(Malformed(format!("{path}.explore[].url is not a text"))); + }; + // Only the two fixed pages the collector links get the plain + // warning: any other address, even under the docs folder, may carry + // the haplogroup in its query. + let why = if url + .strip_prefix(EXPLORE_DOCS) + .is_some_and(|page| EXPLORE_DOC_PAGES.contains(&page)) + { + crate::html::OFFSITE + } else { + OFFSITE_HAPLOGROUP + }; + // Only a web address with a host becomes a link: a `javascript:` or + // `data:` address in a summary would run when clicked, and a bare + // `https://` leads nowhere, so both show as text. + let web = ["http://", "https://"].iter().any(|s| { + url.get(..s.len()) + .is_some_and(|head| head.eq_ignore_ascii_case(s)) + && url[s.len()..] + .split(['/', '?', '#']) + .next() + .is_some_and(|host| !host.is_empty()) + }); + let value = if web { + offsite_link(&url, &url, why) + } else { + e(&url).into_owned() + }; + out.push(stat_raw(&name, &value)); + } + let note = !out.is_empty(); + Ok((out, note)) +} + +/// `d.get(key, ".") not in (".", "")`: a value worth a line. +pub fn shown<'a>(d: &'a Value, key: &str, path: &str) -> Result, Malformed> { + Ok(get(d, key, path)?.filter(|v| !matches!(v, Value::Str(s) if s.is_empty() || s == "."))) +} + +/// `bool(d.get(key))`. +pub fn has(d: &Value, key: &str, path: &str) -> Result { + Ok(get(d, key, path)?.is_some_and(truthy)) +} diff --git a/rust/crates/og-render/src/sections/ancestry_fine.rs b/rust/crates/og-render/src/sections/ancestry_fine.rs index d87e67915..cc1366373 100644 --- a/rust/crates/og-render/src/sections/ancestry_fine.rs +++ b/rust/crates/og-render/src/sections/ancestry_fine.rs @@ -1,2 +1,293 @@ -//! The report section `ancestry_fine`. Empty until its unit of the 1.0.0 -//! plan lands. +//! Fine-grained ancestry (step 26b): the continental and regional groups +//! with their bootstrap ranges, the groups the model leaves out, the share +//! of the genome each group paints, and the chromosome painting with its +//! legend. + +use pycompat::json::Value; + +use crate::escape::e; +use crate::model::{ + Card, Cell, HtmlPart, Inline, Layout, Line, Malformed, State, Summary, Table, get, need, py, + truthy, +}; +use crate::notes::ANCESTRY_FINE_NOTE; +use crate::sections::ancestry::{get_text, has, list, need_text, number, share}; +use crate::sections::haplogroups::shows_body; +use crate::{html, text}; + +const PATH: &str = "sections.ancestry_fine.data"; + +/// The two levels the model reports, with their headings. +pub const LEVELS: [(&str, &str); 2] = [ + ("continent", "Continental groups"), + ("region", "Regional groups"), +]; + +/// The groups of one level worth a line: every continental group of 1% or +/// more, a regional one of 1% or more or whose bootstrap range reaches 5%. +pub fn rows(d: &Value, level: &str) -> Result, Malformed> { + let at = format!("{PATH}.{level}[]"); + let mut out = Vec::new(); + for r in list(d, level, PATH)? { + let f = need(&r, "fraction", &at)?; + if matches!(f, Value::Null) { + continue; + } + let keep = number(f, &format!("{at}.fraction"))? >= 0.01 + || (level == "region" + && match get(&r, "high", &at)?.filter(|v| truthy(v)) { + Some(h) => number(h, &format!("{at}.high"))? >= 0.05, + None => false, + }); + if keep { + out.push(r); + } + } + Ok(out) +} + +/// ` (93% to 97%)`: the bootstrap range of a row, when it has one. +pub fn spread(r: &Value, at: &str) -> Result { + match get(r, "low", at)? { + None | Some(Value::Null) => Ok(String::new()), + Some(low) => Ok(format!( + " ({} to {})", + share(Some(low), at)?, + share(Some(need(r, "high", at)?), at)? + )), + } +} + +/// `(.*) \((.*) group\)` matched against the whole name: the group and its +/// kind (`Oceania (continental group)`), or the name alone. +fn name_and_kind(group: &str) -> (&str, &str) { + let Some(inner) = group.strip_suffix(" group)") else { + return (group, ""); + }; + match inner.rfind(" (") { + Some(i) if !inner.contains('\n') => (&inner[..i], &inner[i + 2..]), + _ => (group, ""), + } +} + +/// The groups the model leaves out, as one sentence, or "". +pub fn left_out(d: &Value) -> Result { + if !has(d, "left_out", PATH)? { + return Ok(String::new()); + } + let at = format!("{PATH}.left_out[]"); + // One row per level ("Oceania (continental group)"); a group both + // levels leave out reads as one name. + let mut kinds: Vec<(String, Vec)> = Vec::new(); + for r in list(d, "left_out", PATH)? { + let Value::Str(group) = need(&r, "group", &at)? else { + return Err(Malformed(format!("{at}.group is not a text"))); + }; + let (name, kind) = name_and_kind(group); + let i = match kinds.iter().position(|(n, _)| n == name) { + Some(i) => i, + None => { + kinds.push((name.to_owned(), Vec::new())); + kinds.len() - 1 + } + }; + if !kind.is_empty() && !kinds[i].1.iter().any(|k| k == kind) { + kinds[i].1.push(kind.to_owned()); + } + } + let names: Vec = kinds + .iter() + .map(|(n, k)| { + if k.is_empty() { + n.clone() + } else { + format!("{n} ({} group)", k.join(" and ")) + } + }) + .collect(); + Ok(format!( + "Left out of the model: {}. The panel has fewer than {} unrelated reference people for \ + it, so its ancestry is spread over the groups shown.", + names.join(", "), + get_text(d, "min_references", "10", PATH)? + )) +} + +/// The painted shares, or why there are none, and the chromosomes the +/// painting failed on. +pub fn painted(d: &Value) -> Result { + let codes = list(d, "not_painted", PATH)? + .iter() + .map( + |r| match need(r, "code", &format!("{PATH}.not_painted[]"))? { + Value::Str(c) => Ok(c.clone()), + _ => Err(Malformed(format!( + "{PATH}.not_painted[].code is not a text" + ))), + }, + ) + .collect::, _>>()?; + let failed = codes.join(", "); + let mut text = if has(d, "painting", PATH)? { + let at = format!("{PATH}.painting[]"); + list(d, "painting", PATH)? + .iter() + .map(|r| { + Ok(format!( + "{} {}", + need_text(r, "group", &at)?, + share(Some(need(r, "fraction", &at)?), &at)? + )) + }) + .collect::, Malformed>>()? + .join(", ") + } else if failed.is_empty() { + "none (no chromosome long enough)".to_owned() + } else { + "none".to_owned() + }; + if !failed.is_empty() { + text += &format!("; not painted: RFMix failed on {failed}"); + } + Ok(text) +} + +/// The text report's block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head( + s, + "ancestry_fine", + Some("Fine-Grained Ancestry (ADMIXTURE, RFMix)"), + )? + else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if !head.ran { + return Ok(w); + } + let d = s.section("ancestry_fine").data; + for (level, title) in LEVELS { + let rs = rows(d, level)?; + if rs.is_empty() { + continue; + } + w.push(format!(" {title} (fraction, bootstrap range):")); + let at = format!("{PATH}.{level}[]"); + for r in &rs { + let group = crate::sections::ancestry::need_align( + r, + "group", + 50, + crate::sections::ancestry::Align::Left, + &at, + )?; + let frac = share(Some(need(r, "fraction", &at)?), &at)?; + w.push(format!( + " {group} {}{}", + pycompat::fmt::rjust(&frac, 5), + spread(r, &at)? + )); + } + } + let lo = left_out(d)?; + if !lo.is_empty() { + w.push(format!(" {lo}")); + } + let lead = if has(d, "painting", PATH)? { + " Chromosome painting, share of the painted genome: " + } else { + " Chromosome painting: " + }; + w.push(format!("{lead}{}", painted(d)?)); + w.push(format!(" {ANCESTRY_FINE_NOTE}")); + w.push(String::new()); + Ok(w) +} + +/// The HTML report's card, when step 26b has a result: one table per +/// level, the painted shares, and the painting with its legend. +pub fn html(s: &Summary) -> Result { + let sec = s.section("ancestry_fine"); + if sec.state == State::Missing { + return Ok(HtmlPart::default()); + } + let mut c = Card { + title: "Fine-Grained Ancestry".to_owned(), + full: true, + ..Card::default() + }; + if shows_body(&sec) { + let d = sec.data; + for (level, title) in LEVELS { + let at = format!("{PATH}.{level}[]"); + let mut cells = Vec::new(); + for r in rows(d, level)? { + let range = spread(&r, &at)?; + let range = range.trim_matches([' ', '(', ')']); + cells.push(vec![ + Cell::Text(need_text(&r, "group", &at)?), + Cell::Text(share(Some(need(&r, "fraction", &at)?), &at)?), + Cell::Text(if range.is_empty() { "." } else { range }.to_owned()), + ]); + } + if !cells.is_empty() { + c.body.extend([ + Line::Raw(format!("

    {}

    ", e(title))), + Line::Table(Table { + head: ["Group", "Fraction", "Bootstrap range"] + .map(str::to_owned) + .to_vec(), + rows: cells, + layout: Layout::RowLines, + ..Table::default() + }), + ]); + } + } + let lo = left_out(d)?; + if !lo.is_empty() { + c.body.push(Line::Para(lo)); + } + c.body.push(Line::Stat( + "Painted genome".to_owned(), + Inline::Text(painted(d)?), + )); + if let Some(png) = get(d, "painting_png", PATH)?.filter(|v| truthy(v)) { + c.body.push(Line::Raw(format!( + " \"Chromosome", + e(&py(png)?) + ))); + let at = format!("{PATH}.legend[]"); + let keys = list(d, "legend", PATH)? + .iter() + .map(|x| { + Ok(format!( + "■ {}", + css_colour(&need_text(x, "colour", &at)?), + e(&need_text(x, "group", &at)?) + )) + }) + .collect::, Malformed>>()?; + c.body.push(Line::Raw(format!( + "

    {} ■ not painted

    ", + keys.join(" ") + ))); + } + c.caveats.push(ANCESTRY_FINE_NOTE); + } + Ok(HtmlPart { + cards: vec![html::card(&sec, c)?], + files: Vec::new(), + }) +} + +/// A legend colour as CSS: the `#rrggbb` the collector writes, else a fixed +/// grey. Escaping keeps it inside the attribute but would still let a `;` +/// add a declaration, such as one that hides the swatch. +fn css_colour(c: &str) -> &str { + let hex = c.len() == 7 && c.starts_with('#') && c[1..].bytes().all(|b| b.is_ascii_hexdigit()); + if hex { c } else { "#808080" } +} diff --git a/rust/crates/og-render/src/sections/ancient.rs b/rust/crates/og-render/src/sections/ancient.rs index 0bcd3409a..7454e3816 100644 --- a/rust/crates/og-render/src/sections/ancient.rs +++ b/rust/crates/og-render/src/sections/ancient.rs @@ -1,2 +1,133 @@ -//! The report section `ancient`. Empty until its unit of the 1.0.0 plan -//! lands. +//! Ancient populations (step 44): where the genome falls on a PCA of +//! present-day West Eurasians, and the ancient groups of the AADR nearest +//! to it there, with the plot. + +use pycompat::json::Value; + +use crate::escape::e; +use crate::model::{ + Card, Cell, HtmlPart, Layout, Line, Malformed, State, Summary, Table, get, py, truthy, +}; +use crate::notes::ANCIENT_NOTE; +use crate::sections::ancestry::{Align, get_text, need_align, need_text}; +use crate::{html, text}; + +const PATH: &str = "sections.ancient.data"; +const ROW: &str = "sections.ancient.data.nearest[]"; + +/// `(d.get("nearest") or [])[:10]`: the ten nearest groups. +fn nearest(d: &Value) -> Result, Malformed> { + match get(d, "nearest", PATH)?.filter(|v| truthy(v)) { + None => Ok(Vec::new()), + Some(Value::Array(a)) => Ok(a.iter().take(10).collect()), + Some(_) => Err(Malformed(format!("{PATH}.nearest is not a list"))), + } +} + +/// Step 44's lines, the same in both reports: what the PCA was built from, +/// then the nearest groups. +pub fn lines(d: &Value) -> Result, Malformed> { + let g = |k: &str| get_text(d, k, ".", PATH); + let mut out = vec![ + format!( + "AADR {}: {} of {} sites used; axes from {} present-day West Eurasians, {} ancient \ + people in {} groups projected.", + g("release")?, + g("sites_used")?, + g("panel_sites")?, + g("present_day_people")?, + g("ancient_people")?, + g("ancient_groups")? + ), + "Nearest ancient groups:".to_owned(), + ]; + for r in nearest(d)? { + let n = |k: &str| need_text(r, k, ROW); + out.push(format!( + " {}. {} ({}, {}; {} people)", + need_align(r, "rank", 2, Align::Right, ROW)?, + n("group")?, + n("dates")?, + n("country")?, + n("n")? + )); + } + Ok(out) +} + +/// The text report's block, with the plot's file name. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head( + s, + "ancient", + Some("Ancient Populations (AADR PCA, step 44)"), + )? + else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if head.ran { + w.extend( + lines(s.section("ancient").data)? + .iter() + .map(|l| format!(" {l}")), + ); + w.push(format!( + " {ANCIENT_NOTE} Plot: ancient/{}_ancient_pca.svg", + py(s.need("sample")?)? + )); + w.push(String::new()); + } + Ok(w) +} + +/// The HTML report's card: the nearest groups as a table, and the plot as +/// an image (shown, never run as part of the page). +pub fn html(s: &Summary) -> Result { + let sec = s.section("ancient"); + if sec.state == State::Missing { + return Ok(HtmlPart::default()); + } + let mut c = Card { + title: "Ancient Populations".to_owned(), + full: true, + ..Card::default() + }; + if crate::sections::haplogroups::shows_body(&sec) { + let d = sec.data; + let first = lines(d)?.swap_remove(0); + let mut rows = Vec::new(); + for r in nearest(d)? { + rows.push( + ["rank", "group", "dates", "country", "locality", "n"] + .iter() + .map(|k| need_text(r, k, ROW).map(Cell::Text)) + .collect::, Malformed>>()?, + ); + } + c.body = vec![ + Line::Para(first), + Line::Table(Table { + head: ["#", "Ancient group", "Dates", "Country", "Place", "People"] + .map(str::to_owned) + .to_vec(), + rows, + layout: Layout::RowLines, + ..Table::default() + }), + ]; + if let Some(svg) = get(d, "plot_svg", PATH)?.filter(|v| truthy(v)) { + c.body.push(Line::Raw(format!( + " \"PC1", + e(&py(svg)?) + ))); + } + c.caveats.push(ANCIENT_NOTE); + } + Ok(HtmlPart { + cards: vec![html::card(&sec, c)?], + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/annotsv.rs b/rust/crates/og-render/src/sections/annotsv.rs index b9d5acde1..37caf236a 100644 --- a/rust/crates/og-render/src/sections/annotsv.rs +++ b/rust/crates/og-render/src/sections/annotsv.rs @@ -1,2 +1,384 @@ -//! The report section `annotsv`. Empty until its unit of the 1.0.0 plan -//! lands. +//! Step 05's AnnotSV class 4 and 5 SVs, with how many callers saw each +//! (step 22's consensus), the exon-edge and benign-set checks, and the +//! calls step 05's rules moved to class 3 with the reason. +//! +//! It is one block of the structural-variants section, beside the SV +//! callers' counts (`sv_callers`): [`text`] and [`html`] write the AnnotSV +//! block and card, and the section's writer puts them after the callers'. + +use pycompat::fmt::{general, ljust}; + +use crate::model::{ + Card, Cell, Colour, HtmlPart, Inline, Layout, Line, Malformed, State, Summary, Table, +}; +use crate::notes::ANNOTSV_NOTE; +use crate::sections::clinvar::{D, locus_text, viewer_of}; +use crate::{html, text}; + +const TITLE: &str = "Structural Variants, Class 4 and 5 (AnnotSV)"; + +/// How many callers saw the SV. +pub fn support_text(x: &D<'_>) -> Result { + let s = x.at("support")?; + if s.is_none() { + return Ok("not known (no step 22 consensus)".to_owned()); + } + if s.is_num(1) { + return Ok(format!("single caller ({})", x.s("caller")?)); + } + let callers = x.at("support_callers")?.items()?; + let names: Vec<&str> = callers.iter().map(D::string).collect::>()?; + Ok(format!("{} callers ({})", s.text()?, names.join(", "))) +} + +/// The first three genes, and how many more. +pub fn genes_text(x: &D<'_>) -> Result { + let g = x.at("genes")?; + if !g.truthy() { + return Ok("no gene".to_owned()); + } + let items = g.items()?; + let first: Vec<&str> = items + .iter() + .take(3) + .map(D::string) + .collect::>()?; + let mut out = first.join(", "); + if items.len() > 3 { + out += &format!(" and {} more", items.len() - 3); + } + Ok(out) +} + +/// The exon-edge, benign-set and rule flags of one class 4-5 SV. +pub fn flags(x: &D<'_>) -> Result, Malformed> { + let mut out = Vec::new(); + let edge = x.at("exon_edge")?; + if edge.is("yes") { + out.push(format!( + "on exon edges: {}", + x.at("exon_edge_detail")?.string()? + )); + } else if edge.is("not checked") { + out.push("exon edges not checked".to_owned()); + } + let af = x.at("population_af")?; + if !af.is_none() { + let mut f = format!("benign-set AF {}", general(af.num()?, 6)); + let src = x.at("population_source")?; + if src.truthy() { + f += &format!(" ({})", src.text()?); + } + if x.at("common")?.truthy() { + f += ", common"; + } + out.push(f); + } + if let Some(detail) = x.opt("rule_detail")? { + let class = x.s("acmg_class")?; + if class == "4" || class == "5" { + out.push(format!("rule {}: {}", x.s("rule")?, detail.text()?)); + } + } + Ok(out) +} + +/// `v or default`, as text. +fn or_text(x: &D<'_>, key: &str, default: &str) -> Result { + let v = x.at(key)?; + if v.truthy() { + v.text() + } else { + Ok(default.to_owned()) + } +} + +/// One SV step 05's rules moved to class 3: where, from which class, and +/// why. +pub fn moved_text(x: &D<'_>) -> Result { + Ok(format!( + "{} {} {} AnnotSV class {} -> {} ({}: {})", + x.s("type")?, + x.s("sv")?, + genes_text(x)?, + x.s("annotsv_class")?, + x.s("acmg_class")?, + or_text(x, "rule", "rule not named")?, + or_text(x, "rule_detail", "no detail")? + )) +} + +/// The class 4-5 lines, the same in both reports. +pub fn lines(d: &D<'_>) -> Result, Malformed> { + let count = d.at("count")?; + let mut head = format!("Class 4-5 SVs: {}", count.text()?); + if count.truthy() { + let consensus = d.at("consensus")?.truthy(); + let single = if consensus { + d.s("single_caller")? + } else { + "not known".to_owned() + }; + let edge = if d.at("exon_edge_checked")?.truthy() { + d.s("exon_edge")? + } else { + "not checked".to_owned() + }; + head += &format!( + " (single caller: {single}; deletions on exon edges: {edge}; common: {})", + d.s("common")? + ); + } + let mut out = vec![head]; + if let Some(down) = d.opt("downgraded")? { + out.push(format!( + "Moved to class 3 by step 05's rules, not counted above: {}", + down.len()? + )); + } + if count.truthy() && !d.at("consensus")?.truthy() { + out.push(match d.opt("consensus_note")? { + Some(n) => n.text()?, + None => "Caller support: step 22's consensus did not run, so how many callers saw each SV is \ + not known." + .to_owned(), + }); + } + Ok(out) +} + +/// The AnnotSV block of the text report. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "annotsv", Some(TITLE))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if !head.ran { + return Ok(w); + } + let d = D::of(&s.section("annotsv")); + w.extend(lines(&d)?.iter().map(|l| format!(" {l}"))); + for x in d.at("rows")?.items()? { + w.push(format!( + " {} {} class {} {} support: {}", + ljust(&x.s("type")?, 4), + x.s("sv")?, + x.s("acmg_class")?, + genes_text(&x)?, + support_text(&x)? + )); + for f in flags(&x)? { + w.push(format!(" {f}")); + } + } + let down = d.opt("downgraded")?; + if let Some(down) = &down { + w.push( + " Moved to class 3 by step 05's rules (from AnnotSV's class, with the reason):" + .to_owned(), + ); + for x in down.items()? { + w.push(format!(" {}", moved_text(&x)?)); + } + } + if d.at("count")?.truthy() || down.is_some() { + w.push(format!(" {ANNOTSV_NOTE}")); + } + w.push(String::new()); + Ok(w) +} + +/// `type sv` of a row, as the tables show it. +fn sv_cell(x: &D<'_>) -> Result { + Ok(format!( + "{} {}", + x.at("type")?.string()?, + x.at("sv")?.string()? + )) +} + +/// The AnnotSV card of the HTML report, when the step ran or failed. +pub fn html(s: &Summary) -> Result { + let sec = s.section("annotsv"); + if sec.state == State::Missing { + return Ok(HtmlPart::default()); + } + let d = D::of(&sec); + let done = if sec.state == State::Stale { + Inline::Badge("Stale".to_owned(), Colour::Yellow) + } else { + Inline::Badge("Complete".to_owned(), Colour::Green) + }; + let mut body = vec![Line::Stat("Status".to_owned(), done)]; + if d.truthy() { + body.extend(lines(&d)?.into_iter().map(Line::Note)); + } + let rows = d.opt("rows")?; + if let Some(rows) = &rows { + let mut cells = Vec::new(); + for x in rows.items()? { + let checks = flags(&x)?.join("; "); + cells.push(vec![ + Cell::Text(sv_cell(&x)?), + Cell::Text(x.s("acmg_class")?), + Cell::Text(genes_text(&x)?), + Cell::Text(support_text(&x)?), + Cell::Text(if checks.is_empty() { + "none".to_owned() + } else { + checks + }), + viewer_of(&locus_text(&x.at("sv")?)?)?, + ]); + } + body.push(Line::Table(Table { + head: ["SV", "Class", "Genes", "Callers", "Checks"] + .map(str::to_owned) + .to_vec(), + rows: cells, + viewer: true, + layout: Layout::OneLine, + })); + } + let down = d.opt("downgraded")?; + if let Some(down) = &down { + body.push(Line::Raw( + "

    Moved to class 3 by step 05's rules

    ".to_owned(), + )); + let mut cells = Vec::new(); + for x in down.items()? { + cells.push(vec![ + Cell::Text(sv_cell(&x)?), + Cell::Text(x.s("annotsv_class")?), + Cell::Text(x.s("acmg_class")?), + Cell::Text(genes_text(&x)?), + Cell::Text(or_text(&x, "rule", "not named")?), + Cell::Text(or_text(&x, "rule_detail", "no detail")?), + viewer_of(&locus_text(&x.at("sv")?)?)?, + ]); + } + body.push(Line::Table(Table { + head: ["SV", "AnnotSV class", "Class", "Genes", "Rule", "Why"] + .map(str::to_owned) + .to_vec(), + rows: cells, + viewer: true, + layout: Layout::OneLine, + })); + } + let mut caveats = Vec::new(); + if rows.is_some() || down.is_some() { + body.push(Line::ViewerNote); + caveats.push(ANNOTSV_NOTE); + } + let card = html::card( + &sec, + Card { + title: TITLE.to_owned(), + full: true, + caveats, + body, + ..Card::default() + }, + )?; + Ok(HtmlPart { + cards: vec![card], + files: Vec::new(), + }) +} + +#[cfg(test)] +mod tests { + use super::html; + use crate::escape::e; + use crate::model::Summary; + use crate::notes::{ANNOTSV_NOTE, VIEWER_NOTE}; + + /// The structural-variants writer is not wired yet, so no report case + /// reaches this card: this renders it through the section frame. + fn page(data: &str) -> String { + let s = Summary::parse(&format!( + r#"{{"sections": {{"annotsv": {{"state": "ok", "data": {data}}}}}}}"# + )) + .unwrap(); + let part = html(&s).unwrap(); + crate::html::frame("structural-variants", &part, None, false).join("\n") + } + + const ROWS: &str = r#""rows": [ + {"type": "DEL", "sv": "chr1:1000-5000", "acmg_class": "5", "genes": ["A function & more"}, + {"type": "INS", "sv": "chr3:7", "acmg_class": "4", "genes": [], "support": null, + "exon_edge": "no", "population_af": null}]"#; + + const DOWN: &str = r#""downgraded": [ + {"type": "DUP", "sv": "chr2:10-90", "annotsv_class": "4", "acmg_class": "3", "genes": [], + "rule": "", "rule_detail": ""}, + {"type": "DEL", "sv": "chr2:500-900", "annotsv_class": "5", "acmg_class": "3", "genes": ["X&Y"], + "rule": "B2", "rule_detail": "common & benign"}]"#; + + const HEAD: &str = r#""count": 2, "consensus": true, "single_caller": 0, "exon_edge_checked": true, + "exon_edge": 1, "common": 0"#; + + fn viewer(at: &str) -> String { + format!( + "{at}" + ) + } + + #[test] + fn both_tables_escape_their_cells_and_the_viewer_line_follows_them_once() { + let p = page(&format!("{{{HEAD}, {ROWS}, {DOWN}}}")); + let rows = format!( + " \ + \ + \ + {}\ + \ + {}
    SVClassGenesCallersChecksViewer
    DEL chr1:1000-50005A<B, C&D, G1 and 1 more2 callers (manta, delly)on exon edges: exon 2; benign-set AF 0.001 (gnomAD); \ + rule R1: loss <of> function & more
    INS chr3:74no genenot known (no step 22 consensus)none
    ", + viewer("chr1:1000-5000"), + viewer("chr3:7") + ); + let down = format!( + " \ + \ + \ + {}\ + \ + {}
    SVAnnotSV classClassGenesRuleWhyViewer
    DUP chr2:10-9043no genenot namedno detail
    DEL chr2:500-90053X&YB2common <AF> & benign
    ", + viewer("chr2:10-90"), + viewer("chr2:500-900") + ); + let lines: Vec<&str> = p.lines().collect(); + let at = |l: &str| lines.iter().position(|x| *x == l); + let (r, h, d, v) = ( + at(&rows).expect(&p), + at("

    Moved to class 3 by step 05's rules

    ").expect(&p), + at(&down).expect(&p), + at(VIEWER_NOTE).expect(&p), + ); + assert!(r < h && h + 1 == d && d + 1 == v, "{p}"); + assert_eq!(p.matches("

    ").count(), 1, "{p}"); + assert!( + p.contains(&format!("

    {}

    ", e(ANNOTSV_NOTE))), + "{p}" + ); + // Red control: an unescaped gene would not match the row. + assert!(!p.contains("AChecks"), "{p}"); + assert_eq!(p.matches("

    ").count(), 1, "{p}"); + let p = page(r#"{"count": 0}"#); + assert!(!p.contains(""), "{p}"); + assert!(!p.contains("class=\"caveat\""), "{p}"); + } +} diff --git a/rust/crates/og-render/src/sections/apoe.rs b/rust/crates/og-render/src/sections/apoe.rs index de95c5197..45f9897f1 100644 --- a/rust/crates/og-render/src/sections/apoe.rs +++ b/rust/crates/og-render/src/sections/apoe.rs @@ -1 +1,53 @@ -//! The report section `apoe`. Empty until its unit of the 1.0.0 plan lands. +//! APOE, the Alzheimer's risk factor: the report keeps its lines apart and +//! last, behind a warning, because many people choose not to learn them. +//! The text report prints them after the warning; the HTML report folds +//! them away until opened. + +use pycompat::json::Value; + +use crate::model::{Cell, Layout, Line, Malformed, Table, need, py}; + +/// The warning before the APOE lines. +pub const APOE_WARNING: &str = "APOE and Alzheimer's disease: a risk factor, not a diagnosis. Many \ + people choose not to learn their APOE result; docs/42-traits.md explains it."; + +/// The text report's APOE lines, the warning first; none without APOE rows. +pub fn text_lines(rows: &[Value]) -> Result, Malformed> { + if rows.is_empty() { + return Ok(Vec::new()); + } + let mut out = vec![format!(" {APOE_WARNING}")]; + for r in rows { + let f = |k: &str| need(r, k, "sections.traits.data.apoe[]").and_then(py); + out.push(format!( + " {} ({}): {}. {}", + f("trait")?, + f("rsid")?, + f("genotype")?, + f("reading")? + )); + } + Ok(out) +} + +/// The HTML report's APOE table, folded away behind the warning: `head` is +/// the traits table's column titles, `row` builds one row as the traits +/// table does. None without APOE rows. +pub fn html_line( + rows: &[Value], + head: Vec, + row: impl Fn(&Value) -> Result, Malformed>, +) -> Result, Malformed> { + if rows.is_empty() { + return Ok(None); + } + Ok(Some(Line::Folded( + format!("{APOE_WARNING} Open to show it."), + Table { + head, + rows: rows.iter().map(row).collect::>()?, + layout: Layout::Indented, + ..Table::default() + }, + ))) +} diff --git a/rust/crates/og-render/src/sections/archaic.rs b/rust/crates/og-render/src/sections/archaic.rs index 4295ec431..42cc364a2 100644 --- a/rust/crates/og-render/src/sections/archaic.rs +++ b/rust/crates/og-render/src/sections/archaic.rs @@ -1,2 +1,89 @@ -//! The report section `archaic`. Empty until its unit of the 1.0.0 plan -//! lands. +//! Archaic ancestry (step 43, IBDmix): the share of the analysed autosomes +//! in Neanderthal and Denisovan segments, and the longest segments. + +use pycompat::fmt::fixed; +use pycompat::json::Value; + +use crate::model::{Card, Inline, Line, Malformed, State, Summary, need, py}; +use crate::notes::ARCHAIC_NOTE; +use crate::sections::ancestry::{get_text, has, items, need_text, number}; +use crate::{html, text}; + +const PATH: &str = "sections.archaic.data"; + +/// A length in base pairs as megabases (`50.0 Mb`). +fn mb(d: &Value, key: &str) -> Result { + let bp = number(need(d, key, PATH)?, &format!("{PATH}.{key}"))?; + Ok(format!("{} Mb", fixed(bp / 1e6, 1))) +} + +/// Step 43's lines, the same in both reports. +pub fn lines(d: &Value) -> Result, Malformed> { + let n = |k: &str| need_text(d, k, PATH); + let mut out = Vec::new(); + for (name, key) in [("Neanderthal", "neanderthal"), ("Denisovan", "denisovan")] { + out.push(format!( + "{name}: {}% ({} in {} segments)", + n(&format!("{key}_percent"))?, + mb(d, &format!("{key}_bp"))?, + n(&format!("{key}_segments"))? + )); + } + out.push(format!( + "Analysed: {} of the autosomes; modern frequencies: 1000 Genomes {}", + mb(d, "analysed_bp")?, + get_text(d, "population", ".", PATH)? + )); + if has(d, "longest", PATH)? { + out.push("Longest segments:".to_owned()); + for x in items(need(d, "longest", PATH)?, &format!("{PATH}.longest"))? { + out.push(format!(" {}", py(&x)?)); + } + } + Ok(out) +} + +/// The text report's block. It ends without a blank line, as the Python +/// report's did. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "archaic", Some("Archaic Ancestry (IBDmix, step 43)"))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if head.ran { + let d = s.section("archaic").data; + w.extend(lines(d)?.iter().map(|l| format!(" {l}"))); + w.push(format!(" {ARCHAIC_NOTE}")); + } + Ok(w) +} + +/// The HTML report's card, when step 43 has a result. +pub fn html(s: &Summary) -> Result { + let sec = s.section("archaic"); + if sec.state == State::Missing { + return Ok(crate::model::HtmlPart::default()); + } + let mut c = Card { + title: "Archaic Ancestry (Neanderthal, Denisovan)".to_owned(), + ..Card::default() + }; + if sec.state.ran() { + let d = sec.data; + for (label, key) in [ + ("Neanderthal", "neanderthal_percent"), + ("Denisovan", "denisovan_percent"), + ] { + c.body.push(Line::Stat( + label.to_owned(), + Inline::Text(format!("{}%", need_text(d, key, PATH)?)), + )); + } + c.body.extend(lines(d)?.into_iter().map(Line::Para)); + c.caveats.push(ARCHAIC_NOTE); + } + Ok(crate::model::HtmlPart { + cards: vec![html::card(&sec, c)?], + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/bloodtype.rs b/rust/crates/og-render/src/sections/bloodtype.rs index afc1017a2..9cc6bc980 100644 --- a/rust/crates/og-render/src/sections/bloodtype.rs +++ b/rust/crates/og-render/src/sections/bloodtype.rs @@ -1,2 +1,104 @@ -//! The report section `bloodtype`. Empty until its unit of the 1.0.0 plan -//! lands. +//! Step 41's blood groups, predicted from the genome: one row per system +//! (ABO, RhD and the others the marker table defines), with its genotype and +//! confidence. The markers' sources and licences are in the header of +//! `assets/bloodtype_markers.tsv`. + +use pycompat::fmt::ljust; +use pycompat::json::Value; + +use crate::html; +use crate::model::{ + Card, Cell, HtmlPart, Inline, Layout, Line, Malformed, Summary, Table, need, py, +}; +use crate::notes::BLOODTYPE_NOTE; +use crate::sections::paralogs::shows_body; +use crate::text; + +const PATH: &str = "sections.bloodtype.data.systems[]"; + +/// The systems, or an error when the summary holds something else. +fn systems(v: &Value) -> Result<&[Value], Malformed> { + match v { + Value::Array(a) => Ok(a), + _ => Err(Malformed( + "sections.bloodtype.data.systems is not a list".into(), + )), + } +} + +/// The text report's block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head( + s, + "bloodtype", + Some("Blood Groups (predicted from the genome)"), + )? + else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if head.ran { + let d = s.section("bloodtype"); + for x in systems(d.need("systems")?)? { + let f = |k: &str| need(x, k, PATH).and_then(py); + w.push(format!( + " {} {} {} ({})", + ljust(&f("system")?, 9), + ljust(&f("phenotype")?, 34), + ljust(&f("genotype")?, 28), + f("confidence")? + )); + } + w.push(format!(" {BLOODTYPE_NOTE}")); + w.push(String::new()); + } + Ok(w) +} + +/// The HTML report's card, when step 41 is in the summary. +pub fn html(s: &Summary) -> Result { + let d = s.section("bloodtype"); + if d.state == crate::model::State::Missing { + return Ok(HtmlPart::default()); + } + let mut body = Vec::new(); + let mut caveats = Vec::new(); + if shows_body(&d) { + let dot = |k: &str| d.get(k).and_then(|v| v.map_or(Ok(".".to_owned()), py)); + body.push(Line::Stat("ABO".to_owned(), Inline::Text(dot("abo")?))); + body.push(Line::Stat("RhD".to_owned(), Inline::Text(dot("rhd")?))); + let mut rows = Vec::new(); + for x in d.get("systems")?.map_or(Ok(&[][..]), systems)? { + let f = |k: &str| need(x, k, PATH).and_then(py).map(Cell::Text); + rows.push(vec![ + f("system")?, + f("phenotype")?, + f("genotype")?, + f("confidence")?, + ]); + } + body.push(Line::Table(Table { + head: ["System", "Predicted phenotype", "Genotype", "Confidence"] + .map(str::to_owned) + .to_vec(), + rows, + layout: Layout::OneLine, + ..Table::default() + })); + caveats.push(BLOODTYPE_NOTE); + } + let card = html::card( + &d, + Card { + title: "Blood Groups (predicted)".to_owned(), + full: true, + caveats, + body, + ..Card::default() + }, + )?; + Ok(HtmlPart { + cards: vec![card], + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/callability.rs b/rust/crates/og-render/src/sections/callability.rs index b91918804..41208ca03 100644 --- a/rust/crates/og-render/src/sections/callability.rs +++ b/rust/crates/og-render/src/sections/callability.rs @@ -1,2 +1,120 @@ -//! The report section `callability`. Empty until its unit of the 1.0.0 plan -//! lands. +//! Step 16b's gene callability: how many of the hard-to-read genes of +//! `assets/callability_genes.tsv` the reads covered, and which were not +//! read, apart from the ones a step of their own calls. + +use pycompat::fmt::fixed; + +use crate::model::{Card, Colour, HtmlPart, Inline, Line, Malformed, State, Summary}; +use crate::notes::CALLABILITY_NOTE; +use crate::sections::clinvar::D; +use crate::{html, text}; + +const TITLE: &str = "Genes This Genome Could Not Read Reliably"; + +/// `GENE (37%)`, or `GENE (not on this reference)`. +fn gene(g: &D<'_>) -> Result { + if g.at("status")?.is("not on this reference") { + return Ok(format!("{} (not on this reference)", g.s("gene")?)); + } + Ok(format!( + "{} ({}%)", + g.s("gene")?, + fixed(100.0 * g.at("share")?.num()?, 0) + )) +} + +/// The lines of the text block and the card. +pub fn lines(d: &D<'_>) -> Result, Malformed> { + let mut out = vec![format!( + "{} of the {} hard-to-read genes of assets/callability_genes.tsv were read: {}% or more of their \ + bases had {} or more reads of mapping quality {} or more.", + d.s("read")?, + d.s("catalog_genes")?, + fixed(100.0 * d.at("min_share")?.num()?, 0), + d.s("min_depth")?, + d.s("min_mapq")? + )]; + let nr = d.at("not_read")?.items()?; + let owned = d.at("not_read_owned")?.items()?; + let listed = if nr.is_empty() { + "none".to_owned() + } else { + nr.iter() + .map(gene) + .collect::, _>>()? + .join(", ") + }; + out.push(format!( + "Not read, and no other step calls them ({}): {listed}", + nr.len() + )); + if !owned.is_empty() { + let mut parts = Vec::new(); + for g in &owned { + parts.push(format!("{} by {}", gene(g)?, g.s("owner")?)); + } + out.push(format!( + "Not read here, called by a step of their own ({}): {}", + owned.len(), + parts.join(", ") + )); + } + Ok(out) +} + +/// The text report's block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "callability", Some(TITLE))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if head.ran { + let d = D::of(&s.section("callability")); + w.extend(lines(&d)?.iter().map(|l| format!(" {l}"))); + w.push(format!(" {CALLABILITY_NOTE}")); + w.push(String::new()); + } + Ok(w) +} + +/// The HTML report's card, when the step ran or failed. +pub fn html(s: &Summary) -> Result { + let sec = s.section("callability"); + if sec.state == State::Missing { + return Ok(HtmlPart::default()); + } + let d = D::of(&sec); + let mut body = Vec::new(); + if sec.state.ran() { + let not_read = d.at("not_read")?.len()?; + body.push(Line::Stat( + "Genes read".to_owned(), + Inline::Text(format!("{} of {}", d.s("read")?, d.s("catalog_genes")?)), + )); + body.push(Line::Stat( + "Not read, no other caller".to_owned(), + Inline::Badge( + not_read.to_string(), + if not_read > 0 { + Colour::Red + } else { + Colour::Green + }, + ), + )); + body.extend(lines(&d)?.into_iter().map(Line::Note)); + } + let card = html::card( + &sec, + Card { + title: TITLE.to_owned(), + caveats: vec![CALLABILITY_NOTE], + body, + ..Card::default() + }, + )?; + Ok(HtmlPart { + cards: vec![card], + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/carrier.rs b/rust/crates/og-render/src/sections/carrier.rs index 98e47f503..ffaf982a4 100644 --- a/rust/crates/og-render/src/sections/carrier.rs +++ b/rust/crates/og-render/src/sections/carrier.rs @@ -1,2 +1,107 @@ -//! The report section `carrier`. Empty until its unit of the 1.0.0 plan -//! lands. +//! The sample's own carrier list (no partner): the ClinVar P/LP hits in +//! the genes of `assets/carrier_genes.tsv`, the hypomorph and +//! pseudodeficiency alleles set aside, and the two-hit genes +//! ([`super::two_hits`]). + +use crate::model::{Card, HtmlPart, Line, Malformed, State, Summary}; +use crate::notes::CARRIER_NOTE; +use crate::sections::clinvar::D; +use crate::sections::two_hits; +use crate::{html, text}; + +const TITLE: &str = "Carrier List (this sample)"; + +fn hemi(x: &D<'_>) -> Result { + Ok(match x.opt("hemizygous")? { + Some(h) => format!(" ({})", h.text()?), + None => String::new(), + }) +} + +/// The list's lines, the same in both reports. +pub fn lines(c: &D<'_>) -> Result, Malformed> { + if c.flag("unavailable")? { + return Ok(vec![format!( + "Carrier list: not made, {}", + c.s("unavailable")? + )]); + } + let carriers = c.at("carriers")?.items()?; + let set_aside = c.at("set_aside")?; + let mut head = format!( + "Carrier list ({} genes): {} finding(s)", + c.s("genes_on_list")?, + carriers.len() + ); + if set_aside.truthy() { + head += &format!(", {} set aside", set_aside.len()?); + } + let mut out = vec![head]; + for x in &carriers { + out.push(format!( + "{} ({}; {}): {} {}{}, ClinVar {}", + x.s("gene")?, + x.s("condition")?, + x.s("inheritance")?, + x.s("variant")?, + x.s("genotype")?, + hemi(x)?, + x.s("significance")? + )); + } + for x in set_aside.items()? { + out.push(format!( + "Set aside: {} {} {}{}: {} is a {} allele, not a carrier finding on its own", + x.s("gene")?, + x.s("variant")?, + x.s("genotype")?, + hemi(&x)?, + x.s("name")?, + x.s("kind")? + )); + } + out.extend(two_hits::lines(c.get("two_hits")?.as_ref())?); + Ok(out) +} + +/// The text report's block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "carrier", Some(TITLE))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if head.ran { + let c = D::of(&s.section("carrier")); + w.extend(lines(&c)?.iter().map(|l| format!(" {l}"))); + w.push(format!(" {CARRIER_NOTE}")); + w.push(String::new()); + } + Ok(w) +} + +/// The HTML report's card, when the list was made or its step failed. +pub fn html(s: &Summary) -> Result { + let sec = s.section("carrier"); + if sec.state == State::Missing { + return Ok(HtmlPart::default()); + } + let body = if sec.state.ran() { + lines(&D::of(&sec))?.into_iter().map(Line::Para).collect() + } else { + Vec::new() + }; + let card = html::card( + &sec, + Card { + title: TITLE.to_owned(), + full: true, + caveats: vec![CARRIER_NOTE], + body, + ..Card::default() + }, + )?; + Ok(HtmlPart { + cards: vec![card], + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/cdna.rs b/rust/crates/og-render/src/sections/cdna.rs index bd3715784..727f1c34c 100644 --- a/rust/crates/og-render/src/sections/cdna.rs +++ b/rust/crates/og-render/src/sections/cdna.rs @@ -1 +1,75 @@ -//! The report section `cdna`. Empty until its unit of the 1.0.0 plan lands. +//! Step 06's cDNA marks, shown in the ClinVar block of both reports: the +//! read check of each hit (alt reads clipped at an exon edge) and the hits at a +//! known lab-construct site. The scan of every exon for reads that jump from +//! one exon to the next is `sections::cdna_scan`. + +use crate::model::Malformed; +use crate::sections::clinvar::D; + +/// Step 06's read check of one ClinVar (or CPSR) hit. A hit at a known +/// construct site always shows its read counts when they were measured. +pub fn cdna_text(h: &D<'_>) -> Result { + let c = h.at("cdna")?; + let counts = format!( + "{} of {} alt reads and {} of {} ref reads clipped at an exon edge", + c.s("alt_clipped")?, + c.s("alt_reads")?, + c.s("ref_clipped")?, + c.s("ref_reads")? + ); + let site = match c.opt("construct")? { + Some(x) => format!("known lab-construct site ({}): ", x.text()?), + None => String::new(), + }; + let check = c.at("check")?; + if check.is("flagged") { + return Ok(format!("{site}{} ({counts})", c.s("note")?)); + } + if check.is("clear") { + return Ok(if site.is_empty() { + "clear".to_owned() + } else { + format!("{site}clear ({counts})") + }); + } + let note = c.at("note")?; + let mut out = format!("{site}not checked"); + if note.truthy() { + out += &format!(": {}", note.text()?); + } + let measured = c.get("alt_reads")?.is_some_and(|x| !x.is_none()); + if !site.is_empty() && measured { + out += &format!(" ({counts})"); + } + Ok(out) +} + +/// The hits at a known lab-construct site below the first `shown`: both +/// reports print them too, so every hit the overview counts has a row. +pub fn construct_past_cap<'a>(hits: &[D<'a>], shown: usize) -> Result>, Malformed> { + let mut out = Vec::new(); + for h in hits.iter().skip(shown) { + let site = match h.opt("cdna")? { + Some(c) => c.flag("construct")?, + None => false, + }; + if site { + out.push(h.clone()); + } + } + Ok(out) +} + +/// One such hit as a line. +pub fn construct_past_cap_text(h: &D<'_>) -> Result { + Ok(format!( + "{} {}:{} {}>{} {}: {}", + h.s("gene")?, + h.s("chrom")?, + h.s("pos")?, + h.s("ref")?, + h.s("alt")?, + h.s("genotype")?, + cdna_text(h)? + )) +} diff --git a/rust/crates/og-render/src/sections/cdna_scan.rs b/rust/crates/og-render/src/sections/cdna_scan.rs index 0f0df9c0f..023cda02e 100644 --- a/rust/crates/og-render/src/sections/cdna_scan.rs +++ b/rust/crates/og-render/src/sections/cdna_scan.rs @@ -9,7 +9,9 @@ use pycompat::json::Value; -use crate::model::{Cell, Colour, Inline, Line, Malformed, Section, Table, need, py, truthy}; +use crate::model::{ + Cell, Colour, Inline, Layout, Line, Malformed, Section, Table, need, py, truthy, +}; use crate::notes::CDNA_SCAN_NOTE; const PATH: &str = "sections.clinvar.data.cdna_scan"; @@ -139,7 +141,8 @@ pub fn html_lines(d: &Section<'_>) -> Result<(Vec, Option<&'static str>), head: ["Locus", "Junctions", "Reads"].map(str::to_owned).to_vec(), rows, viewer: true, - one_line: false, + layout: Layout::Indented, })); + out.push(Line::ViewerNote); Ok((out, Some(CDNA_SCAN_NOTE))) } diff --git a/rust/crates/og-render/src/sections/clinvar.rs b/rust/crates/og-render/src/sections/clinvar.rs index 9876ec648..d8d152807 100644 --- a/rust/crates/og-render/src/sections/clinvar.rs +++ b/rust/crates/og-render/src/sections/clinvar.rs @@ -1,2 +1,676 @@ -//! The report section `clinvar`. Empty until its unit of the 1.0.0 plan -//! lands. +//! Step 06's ClinVar screen: the count of P/LP hits with the call floor and +//! the gnomAD frequency, the hits best-reviewed first (up to 50, then any +//! hit at a known lab-construct site), the cDNA read check and the cDNA +//! scan ([`super::cdna`]) and the known deep intronic sites +//! ([`super::deep_intronic`]). +//! +//! It also holds what the clinical sections share: [`D`], a dict of +//! summary.json read with Python's rules, and the Viewer cell of a findings +//! table. + +use pycompat::fmt::{general, ljust, percent}; +use pycompat::json::Value; + +use crate::model::{ + Card, Cell, Colour, HtmlPart, Inline, Layout, Line, Malformed, Section, Summary, Table, + contains, num, py, py_eq, truthy, +}; +use crate::notes::{CLINVAR_FREQ_NOTE, DEEP_INTRONIC_NOTE}; +use crate::sections::{cdna, cdna_scan, deep_intronic}; +use crate::{html, text}; + +/// The hits the report lists, and the anchor of their table (the charts +/// link a listed hit there). +pub const HITS_SHOWN: usize = 50; +pub const HITS_ANCHOR: &str = "clinvar-hits"; + +// --- a dict of summary.json, read with Python's rules --------------------------------- + +/// A value of summary.json and the path that names it in an error. +#[derive(Debug, Clone)] +pub struct D<'a> { + pub v: &'a Value, + pub path: String, +} + +impl<'a> D<'a> { + pub fn new(v: &'a Value, path: impl Into) -> D<'a> { + D { + v, + path: path.into(), + } + } + + /// The data of section `sec`. + pub fn of(sec: &Section<'a>) -> D<'a> { + D::new(sec.data, format!("sections.{}.data", sec.key)) + } + + fn child(&self, v: &'a Value, key: &str) -> D<'a> { + D::new(v, format!("{}.{key}", self.path)) + } + + /// `d[key]`. + pub fn at(&self, key: &str) -> Result, Malformed> { + crate::model::need(self.v, key, &self.path).map(|v| self.child(v, key)) + } + + /// `d.get(key)`: `None` for a missing key. + pub fn get(&self, key: &str) -> Result>, Malformed> { + Ok(crate::model::get(self.v, key, &self.path)?.map(|v| self.child(v, key))) + } + + /// `d.get(key)` when it is true, else `None`. + pub fn opt(&self, key: &str) -> Result>, Malformed> { + Ok(self.get(key)?.filter(|x| truthy(x.v))) + } + + /// `bool(d.get(key))`. + pub fn flag(&self, key: &str) -> Result { + Ok(self.opt(key)?.is_some()) + } + + /// `key in d`. + pub fn has(&self, key: &str) -> bool { + contains(self.v, key) + } + + /// `str(d[key])`. + pub fn s(&self, key: &str) -> Result { + self.at(key)?.text() + } + + /// `str(d.get(key))`: `None` for a missing key. + pub fn s_or_none(&self, key: &str) -> Result { + self.get(key)?.map_or(Ok("None".to_owned()), |x| x.text()) + } + + /// `str(d.get(key, default))`. + pub fn s_or(&self, key: &str, default: &str) -> Result { + self.get(key)?.map_or(Ok(default.to_owned()), |x| x.text()) + } + + /// `str(v)`. + pub fn text(&self) -> Result { + py(self.v) + } + + /// `v` where Python needs a string (`html.escape(v)`, `s + v`). + pub fn string(&self) -> Result<&'a str, Malformed> { + match self.v { + Value::Str(s) => Ok(s), + _ => Err(Malformed(format!("{} is not a string", self.path))), + } + } + + /// `bool(v)`. + pub fn truthy(&self) -> bool { + truthy(self.v) + } + + /// `v is None`. + pub fn is_none(&self) -> bool { + matches!(self.v, Value::Null) + } + + /// `v == text`. + pub fn is(&self, text: &str) -> bool { + matches!(self.v, Value::Str(s) if s == text) + } + + /// The items of a list. + pub fn items(&self) -> Result>, Malformed> { + match self.v { + Value::Array(a) => Ok(a + .iter() + .enumerate() + .map(|(i, v)| D::new(v, format!("{}[{i}]", self.path))) + .collect()), + _ => Err(Malformed(format!("{} is not a list", self.path))), + } + } + + /// The items of a dict, in its order. + pub fn pairs(&self) -> Result)>, Malformed> { + match self.v { + Value::Object(o) => Ok(o + .iter() + .map(|(k, v)| (k.as_str(), self.child(v, k))) + .collect()), + _ => Err(Malformed(format!("{} is not a dict", self.path))), + } + } + + /// `len(v)` of a list, dict or string. + pub fn len(&self) -> Result { + match self.v { + Value::Array(a) => Ok(a.len()), + Value::Object(o) => Ok(o.len()), + Value::Str(s) => Ok(s.chars().count()), + _ => Err(Malformed(format!("{} has no length", self.path))), + } + } + + /// `not v` of a list, dict or string. + pub fn is_empty(&self) -> Result { + Ok(self.len()? == 0) + } + + /// The number `v` is. + pub fn num(&self) -> Result { + num(Some(self.v)).ok_or_else(|| Malformed(format!("{} is not a number", self.path))) + } + + /// `v == n` for a number `n`. + pub fn is_num(&self, n: i128) -> bool { + py_eq(self.v, &Value::Int(n)) + } +} + +/// `d.get(key) or []` as a list. +pub fn list_or_empty<'a>(d: &D<'a>, key: &str) -> Result>, Malformed> { + d.opt(key)?.map_or(Ok(Vec::new()), |x| x.items()) +} + +/// `f"{v:, width: usize) -> Result { + let t = match v.v { + Value::Str(s) => s.clone(), + Value::Bool(b) => u8::from(*b).to_string(), + Value::Int(_) | Value::Float(_) => v.text()?, + _ => return Err(Malformed(format!("{} cannot be padded", v.path))), + }; + Ok(ljust(&t, width)) +} + +// --- the Viewer column ---------------------------------------------------------------- + +fn name_char(c: u8) -> bool { + c.is_ascii_alphanumeric() || c == b'_' || c == b'.' +} + +/// Every `name:start` or `name:start-end` in `text`, left to right, as the +/// Python's `([A-Za-z0-9_.]+):([0-9]+)(?:-([0-9]+))?(?![0-9])` finds them. +pub fn loci_in(text: &str) -> Vec<(&str, &str, Option<&str>)> { + let b = text.as_bytes(); + let digits = |from: usize| { + let mut j = from; + while j < b.len() && b[j].is_ascii_digit() { + j += 1; + } + j + }; + let mut out = Vec::new(); + let mut i = 0; + while i < b.len() { + if !name_char(b[i]) { + i += 1; + continue; + } + let mut j = i; + while j < b.len() && name_char(b[j]) { + j += 1; + } + // Every start inside the run ends the name where the run ends. + if j < b.len() && b[j] == b':' && j + 1 < b.len() && b[j + 1].is_ascii_digit() { + let k = digits(j + 1); + let mut end = None; + let mut next = k; + if k + 1 < b.len() && b[k] == b'-' && b[k + 1].is_ascii_digit() { + let m = digits(k + 1); + end = Some(&text[k + 1..m]); + next = m; + } + out.push((&text[i..j], &text[j + 1..k], end)); + i = next; + } else { + i = j.max(i + 1); + } + } + out +} + +/// The Viewer cell of a row whose locus is written in its text: the span of +/// every position on the first locus's chromosome. +pub fn viewer_of(text: &str) -> Result { + let found = loci_in(text); + let Some((chrom, _, _)) = found.first().copied() else { + return Ok(Cell::Viewer(None)); + }; + let pos: Vec = found + .iter() + .filter(|(c, _, _)| *c == chrom) + .flat_map(|(_, a, b)| [Some(*a), *b]) + .flatten() + .map(|x| x.parse::().unwrap_or(i128::MAX)) + .collect(); + let lo = pos.iter().min().copied().unwrap_or_default(); + let hi = pos.iter().max().copied().unwrap_or_default(); + Cell::viewer( + &Value::Str(chrom.to_owned()), + &Value::Int(lo), + Some(&Value::Int(hi)), + ) +} + +/// `str(text or "")` for [`viewer_of`]. +pub fn locus_text(v: &D<'_>) -> Result { + if v.truthy() { + v.text() + } else { + Ok(String::new()) + } +} + +// --- the hits ------------------------------------------------------------------------- + +/// A hit's combined-allele note (step 06), `""` when it has none. +pub fn combined_cell(h: &D<'_>) -> Result { + match h.get("combined_allele")? { + None => Ok(String::new()), + Some(n) if n.is(".") || n.is("") => Ok(String::new()), + Some(n) => n.text(), + } +} + +/// A hit's gnomAD allele frequency, `""` for a summary written before the +/// field. +pub fn af_text(h: &D<'_>) -> Result { + let Some(af) = h.get("af")? else { + return Ok(String::new()); + }; + if af.is_none() { + return Ok("gnomAD AF unknown".to_owned()); + } + let mut t = format!("gnomAD AF {}", general(af.num()?, 3)); + if h.flag("common")? { + t += "; common: unlikely to be a high-penetrance finding"; + } + Ok(t) +} + +/// The low-confidence mark of a ClinVar or CPSR row, `""` for a confident +/// one. +pub fn call_text(h: &D<'_>) -> Result { + if !h.flag("low_confidence")? { + return Ok(String::new()); + } + let mut vals = Vec::new(); + for (k, key) in [("GQ", "gq"), ("DP", "dp")] { + if let Some(v) = h.get(key)?.filter(|v| !v.is_none()) { + vals.push(format!("{k} {}", v.text()?)); + } + } + Ok(format!( + "low-confidence call ({}): check the reads", + vals.join(", ") + )) +} + +/// How many hits fall below step 06's call floor, or `None` for a summary +/// written before the floor. +pub fn floor_line(d: &D<'_>) -> Result, Malformed> { + if !d.has("low_confidence") { + return Ok(None); + } + let (gq, dp) = floor(d)?; + Ok(Some(format!( + "Low-confidence calls (GQ below {gq} or DP below {dp}), listed but not counted above: {}", + d.s("low_confidence")? + ))) +} + +/// The call floor's GQ and DP, `None` when the summary has none. +fn floor(d: &D<'_>) -> Result<(String, String), Malformed> { + match d.opt("call_floor")? { + Some(f) => Ok((f.s_or_none("GQ")?, f.s_or_none("DP")?)), + None => Ok(("None".to_owned(), "None".to_owned())), + } +} + +/// Where the hits' gnomAD frequencies came from, or why there are none. +pub fn freq_line(d: &D<'_>) -> Result, Malformed> { + let Some(fr) = d.opt("frequency")? else { + return Ok(None); + }; + if let Some(note) = fr.opt("note")? { + return Ok(Some(format!( + "gnomAD AF: unknown for every hit ({})", + note.text()? + ))); + } + let mut common = 0; + if let Some(hits) = d.get("hits")? { + for h in hits.items()? { + if h.flag("common")? { + common += 1; + } + } + } + Ok(Some(format!( + "gnomAD AF from {}: {common} hit(s) above {}", + fr.s("source")?, + percent(fr.at("common_af")?.num()?, 0) + ))) +} + +// --- the text report ------------------------------------------------------------------ + +/// The text report's block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "clinvar", Some("ClinVar Pathogenic Screen"))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if !head.ran { + return Ok(w); + } + let d = D::of(&s.section("clinvar")); + w.push(format!( + " Pathogenic/Likely Pathogenic hits: {}", + d.s("count")? + )); + if let Some(low) = floor_line(&d)? { + w.push(format!(" {low}")); + } + w.push(format!(" ClinVar file date: {}", clinvar_release(s)?)); + let hits = d.at("hits")?; + if hits.truthy() { + let hits = hits.items()?; + let by = d.at("by_stars")?; + let mut parts = Vec::new(); + for k in ["4", "3", "2", "1", "0"] { + let plural = if k == "1" { "" } else { "s" }; + parts.push(format!("{k} star{plural} {}", by.s(k)?)); + } + w.push(format!(" By review status: {}", parts.join(", "))); + if d.has("cdna_flagged") { + w.push(format!( + " cDNA read check (alt reads clipped at exon edges): {} flagged", + d.s("cdna_flagged")? + )); + } + if let Some(freq) = freq_line(&d)? { + w.push(format!(" {freq}")); + } + w.push(String::new()); + w.push( + " Hits (best-reviewed first, up to 50): gene, position, genotype, significance [review status] stars" + .to_owned(), + ); + for h in hits.iter().take(HITS_SHOWN) { + let note = combined_cell(h)?; + let mut line = format!( + " {} {}:{} {}>{} {} {} [{}] {}*", + lj(&h.at("gene")?, 10)?, + h.s("chrom")?, + h.s("pos")?, + h.s("ref")?, + h.s("alt")?, + h.s("genotype")?, + h.s("significance")?, + h.s("review_status")?, + h.s("stars")? + ); + if !note.is_empty() { + line += &format!(" ({note})"); + } + w.push(line); + let af = af_text(h)?; + if !af.is_empty() { + w.push(format!(" frequency: {af}")); + } + let call = call_text(h)?; + if !call.is_empty() { + w.push(format!(" call: {call}")); + } + if h.has("cdna") { + w.push(format!(" reads: {}", cdna::cdna_text(h)?)); + } + } + let extra = cdna::construct_past_cap(&hits, HITS_SHOWN)?; + if !extra.is_empty() { + w.push(" Hits at a known lab-construct site past the first 50:".to_owned()); + for h in &extra { + w.push(format!(" {}", cdna::construct_past_cap_text(h)?)); + } + } + } + w.extend(cdna_scan::text_lines(&s.section("clinvar"))?); + if d.has("deep_intronic") { + w.push(" Known deep intronic sites (always reported): gene, variant, call".to_owned()); + for x in d.at("deep_intronic")?.items()? { + w.push(format!( + " {} {} {}:{} {}>{} {}", + lj(&x.at("gene")?, 10)?, + x.s("hgvs")?, + x.s("chrom")?, + x.s("pos")?, + x.s("ref")?, + x.s("alt")?, + deep_intronic::call(&x)? + )); + } + w.push(format!(" {DEEP_INTRONIC_NOTE}")); + } + w.push(format!(" {CLINVAR_FREQ_NOTE}")); + w.push(String::new()); + Ok(w) +} + +/// `s['databases'][key] or 'unknown'`. +pub fn database(s: &Summary, key: &str) -> Result { + let v = crate::model::need(s.need("databases")?, key, "summary.databases")?; + if truthy(v) { + py(v) + } else { + Ok("unknown".to_owned()) + } +} + +fn clinvar_release(s: &Summary) -> Result { + database(s, "clinvar_release") +} + +// --- the HTML report ------------------------------------------------------------------ + +fn badge(n: &D<'_>, on: Colour) -> Result { + let colour = if n.truthy() { on } else { Colour::Gray }; + Ok(Inline::Badge(n.text()?, colour)) +} + +/// The screen's card: the counts, never green (no hit is not an +/// all-clear). +fn screen_card(s: &Summary, d: &D<'_>) -> Result { + let count = d.get("count")?; + let n = count.clone().map_or(Ok("0".to_owned()), |x| x.text())?; + let n_on = count.as_ref().is_some_and(D::truthy); + let mut stars = Vec::new(); + let by = d.get("by_stars")?; + for k in ["4", "3", "2", "1", "0"] { + stars.push(match &by { + Some(by) => by.s_or(k, "0")?, + None => "0".to_owned(), + }); + } + let mut body = vec![ + Line::Stat( + "ClinVar matches".to_owned(), + Inline::Badge(n, if n_on { Colour::Yellow } else { Colour::Gray }), + ), + Line::Stat( + "By stars (4/3/2/1/0)".to_owned(), + Inline::Text(stars.join(" / ")), + ), + Line::Stat( + "ClinVar file date".to_owned(), + Inline::Text(clinvar_release(s)?), + ), + ]; + if d.has("cdna_flagged") { + body.insert( + 2, + Line::Stat( + "Flagged by the cDNA read check".to_owned(), + badge(&d.at("cdna_flagged")?, Colour::Red)?, + ), + ); + } + if let Some(sites) = d.opt("construct_sites")? { + body.insert( + 2, + Line::Stat( + "Hits at a known lab-construct site (check the reads)".to_owned(), + Inline::Badge(sites.text()?, Colour::Yellow), + ), + ); + } + if let Some(stat) = cdna_scan::stat(&s.section("clinvar"))? { + body.insert(2, stat); + } + if d.has("low_confidence") { + let (gq, dp) = floor(d)?; + body.insert( + 1, + Line::Stat( + format!("Low-confidence calls (GQ below {gq} or DP below {dp}), not counted"), + badge(&d.at("low_confidence")?, Colour::Yellow)?, + ), + ); + } + if let Some(freq) = freq_line(d)? { + body.push(Line::Note(freq)); + } + Ok(Card { + title: "ClinVar Screening".to_owned(), + caveats: vec![CLINVAR_FREQ_NOTE], + body, + ..Card::default() + }) +} + +/// The table of the first 50 hits, with the columns the summary carries. +fn hits_card(d: &D<'_>) -> Result { + let reads = d.has("cdna_flagged"); + let freq = d.has("frequency"); + let call = d.has("low_confidence"); + let hits = d.at("hits")?.items()?; + let nh = hits.len(); + let mut head: Vec = [ + "Chr", + "Position", + "Ref", + "Alt", + "Genotype", + "Gene", + "Significance", + "Review status", + "Stars", + "Combined allele", + ] + .map(str::to_owned) + .to_vec(); + if freq { + head.push("gnomAD frequency".to_owned()); + } + if call { + head.push("Call quality".to_owned()); + } + if reads { + head.push("cDNA read check".to_owned()); + } + let mut rows = Vec::new(); + for h in hits.iter().take(HITS_SHOWN) { + let st = + |k: &str| -> Result { Ok(Cell::Text(h.at(k)?.string()?.to_owned())) }; + let mut row = vec![ + st("chrom")?, + Cell::Text(h.s("pos")?), + st("ref")?, + st("alt")?, + st("genotype")?, + st("gene")?, + st("significance")?, + st("review_status")?, + Cell::Text(h.s("stars")?), + Cell::Text(combined_cell(h)?), + ]; + if freq { + row.push(Cell::Text(af_text(h)?)); + } + if call { + row.push(Cell::Text(call_text(h)?)); + } + if reads { + row.push(Cell::Text(cdna::cdna_text(h)?)); + } + row.push(Cell::viewer(h.at("chrom")?.v, h.at("pos")?.v, None)?); + rows.push(row); + } + let mut body = vec![ + Line::Raw(format!(" ")), + Line::Table(Table { + head, + rows, + viewer: true, + layout: Layout::Indented, + }), + ]; + for h in cdna::construct_past_cap(&hits, HITS_SHOWN)? { + body.push(Line::Note(format!( + "Hit at a known lab-construct site past the first 50: {}", + cdna::construct_past_cap_text(&h)? + ))); + } + if nh > 0 { + body.push(Line::ViewerNote); + } + Ok(Card { + title: format!( + "ClinVar Hits (best-reviewed first, {} of {nh})", + nh.min(HITS_SHOWN) + ), + full: true, + body, + ..Card::default() + }) +} + +/// The known deep intronic sites and the cDNA scan's loci. +fn sites_card(sec: &Section<'_>, d: &D<'_>) -> Result { + let mut body = Vec::new(); + let mut caveats = Vec::new(); + if d.has("deep_intronic") { + body.extend(deep_intronic::table(&d.at("deep_intronic")?)?); + caveats.push(DEEP_INTRONIC_NOTE); + } + let (scan, caveat) = cdna_scan::html_lines(sec)?; + body.extend(scan); + caveats.extend(caveat); + Ok(Card { + title: "Known Deep Intronic Sites and the cDNA Scan".to_owned(), + full: true, + caveats, + body, + ..Card::default() + }) +} + +/// The HTML report's cards: the screen always, the hits table when there +/// are hits, the deep intronic sites and the cDNA scan when step 06 wrote +/// them. +pub fn html(s: &Summary) -> Result { + let sec = s.section("clinvar"); + let d = D::of(&sec); + let mut cards = vec![html::card(&sec, screen_card(s, &d)?)?]; + if sec.state.ran() && d.opt("hits")?.is_some() { + cards.push(html::card(&sec, hits_card(&d)?)?); + } + if sec.state.ran() && (d.has("deep_intronic") || d.opt("cdna_scan")?.is_some()) { + cards.push(html::card(&sec, sites_card(&sec, &d)?)?); + } + Ok(HtmlPart { + cards, + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/cn_intervals.rs b/rust/crates/og-render/src/sections/cn_intervals.rs index 11bb963e1..8061cf330 100644 --- a/rust/crates/og-render/src/sections/cn_intervals.rs +++ b/rust/crates/og-render/src/sections/cn_intervals.rs @@ -1,2 +1,151 @@ -//! The report section `cn_intervals`. Empty until its unit of the 1.0.0 -//! plan lands. +//! Step 35's copy-number intervals from read depth: ClinGen's recurrent CNV +//! regions, carrier deletion alleles and the AZF regions. One summary line, +//! then the rows with a deletion or a duplication. + +use pycompat::json::Value; + +use crate::model::{Cell, Inline, Layout, Line, Malformed, Table, get, need, py, truthy}; + +/// The label of the summary line. +pub const LABEL: &str = + "Copy-number intervals (ClinGen recurrent regions, carrier deletions, AZF; from depth)"; + +const PATH: &str = "sections.parascopy.data.cn_intervals"; + +/// `(iv.get(key) or default)` for a dict or a list. +fn or_empty<'a>(iv: &'a Value, key: &str) -> Result, Malformed> { + Ok(get(iv, key, PATH)?.filter(|v| truthy(v))) +} + +/// The rows with a change. +fn changes(iv: Option<&Value>) -> Result<&[Value], Malformed> { + let Some(iv) = iv.filter(|v| truthy(v)) else { + return Ok(&[]); + }; + match or_empty(iv, "changes")? { + None => Ok(&[]), + Some(Value::Array(a)) => Ok(a), + Some(_) => Err(Malformed(format!("{PATH}.changes is not a list"))), + } +} + +/// The interval table in one line: rows by status and how many changed. +pub fn summary(iv: Option<&Value>) -> Result { + let Some(iv) = iv.filter(|v| !matches!(v, Value::Null)) else { + return Ok(format!( + "{LABEL}: not assessed (step 35's _cn_intervals.tsv is not in this run's output)" + )); + }; + let by = or_empty(iv, "by_status")?; + let mut counts: Vec<(String, String)> = Vec::new(); + if let Some(by) = by { + let Value::Object(o) = by else { + return Err(Malformed(format!("{PATH}.by_status is not an object"))); + }; + for (k, n) in o { + counts.push((k.clone(), py(n)?)); + } + } + counts.sort(); + let mut counts: String = counts + .iter() + .map(|(k, n)| format!("{n} {k}")) + .collect::>() + .join(", "); + if counts.is_empty() { + counts = "no row".to_owned(); + } + let n = changes(Some(iv))?.len(); + let called = match by { + Some(by) => get(by, "called", PATH)?, + None => None, + }; + let end = if n > 0 { + format!("{n} with a deletion or a duplication") + } else if let Some(k) = called.filter(|k| truthy(k)) { + format!( + "no deletion or duplication among the {} called rows", + py(k)? + ) + } else { + "none called".to_owned() + }; + let rows = get(iv, "rows", PATH)?.map_or(Ok("0".to_owned()), py)?; + Ok(format!("{LABEL}: {rows} rows ({counts}); {end}")) +} + +/// `chrom:start-end` (1-based) of an interval row of the table (BED). +pub fn place(r: &Value) -> Result { + let at = |k: &str| get(r, k, "an interval row"); + let dot = |k: &str| at(k).and_then(|v| v.map_or(Ok(".".to_owned()), py)); + let b = dot("start")?; + let start = if pycompat::text::isdigit(&b) { + let n = pycompat::num::parse_int(&b) + .map_err(|_| Malformed(format!("an interval row's start {b} is not a number")))?; + (n + 1).to_string() + } else { + b + }; + Ok(format!("{}:{start}-{}", dot("chrom")?, dot("end")?)) +} + +/// `str.capitalize()`: the first character upper case, the rest lower. +fn capitalize(s: &str) -> String { + let mut c = s.chars(); + match c.next() { + Some(first) => { + first.to_uppercase().collect::() + &pycompat::text::lower(c.as_str()) + } + None => String::new(), + } +} + +/// The text report's lines: the summary, then one line per row with a +/// change. +pub fn lines(iv: Option<&Value>) -> Result, Malformed> { + let mut out = vec![summary(iv)?]; + for r in changes(iv)? { + let f = |k: &str| need(r, k, "an interval row").and_then(py); + out.push(format!( + " {}: {} ({}, {}), {} of {} copies", + capitalize(&f("change")?), + f("name")?, + f("locus")?, + place(r)?, + f("copies")?, + f("expected")? + )); + } + Ok(out) +} + +/// The HTML card's lines: the summary as a stat, then a table of the rows +/// with a change. +pub fn html(iv: Option<&Value>) -> Result, Malformed> { + let s = summary(iv)?; + let (label, value) = s.split_once(": ").unwrap_or((&s, "")); + let mut out = vec![Line::Stat(label.to_owned(), Inline::Text(value.to_owned()))]; + let rows = changes(iv)?; + if !rows.is_empty() { + let mut cells = Vec::new(); + for r in rows { + let f = |k: &str| need(r, k, "an interval row").and_then(py); + cells.push(vec![ + Cell::Text(f("name")?), + Cell::Text(f("locus")?), + Cell::Text(place(r)?), + Cell::Text(format!("{} of {}", f("copies")?, f("expected")?)), + Cell::Text(f("change")?), + ]); + } + out.push(Line::Table(Table { + head: ["Region", "Row", "Place", "Copies", "Change"] + .map(str::to_owned) + .to_vec(), + rows: cells, + layout: Layout::OneLine, + ..Table::default() + })); + } + Ok(out) +} diff --git a/rust/crates/og-render/src/sections/comphet.rs b/rust/crates/og-render/src/sections/comphet.rs index df853c692..8c1db9ac0 100644 --- a/rust/crates/og-render/src/sections/comphet.rs +++ b/rust/crates/og-render/src/sections/comphet.rs @@ -1,2 +1,179 @@ -//! The report section `comphet`. Empty until its unit of the 1.0.0 plan -//! lands. +//! Step 31's variant prioritization (slivar): the prioritized and +//! compound-het counts, and the phase of each compound-het pair read from +//! the reads (trans, cis or unknown), with the pairs in trans. + +use pycompat::fmt::ljust; +use pycompat::json::Value; + +use crate::model::{Card, Cell, HtmlPart, Inline, Layout, Line, Malformed, Summary, Table}; +use crate::notes::COMPHET_NOTE; +use crate::sections::clinvar::{D, list_or_empty, locus_text, viewer_of}; +use crate::{html, text}; + +const TITLE: &str = "Variant Prioritization (slivar)"; + +/// `ph.get(key, 0)` as text. +fn count(ph: &D<'_>, key: &str) -> Result { + ph.s_or(key, "0") +} + +/// `ph.get("trans", 0) - len(trans)`, an int or a float as Python gives it. +fn more_in_trans(ph: &D<'_>, shown: usize) -> Result, Malformed> { + let Some(t) = ph.get("trans")? else { + return Ok(None); + }; + let n = t.num()?; + if n <= shown as f64 { + return Ok(None); + } + Ok(Some(match t.v { + Value::Int(i) => (i - shown as i128).to_string(), + Value::Bool(_) => (n as i128 - shown as i128).to_string(), + _ => pycompat::fmt::repr(n - shown as f64), + })) +} + +/// The phase lines of the text block. +pub fn phase_text(d: &D<'_>) -> Result, Malformed> { + let ph = d.at("compound_het_phase")?; + let mut out = vec![format!( + " Phase from the reads: {} trans, {} cis, {} unknown of {} pairs", + count(&ph, "trans")?, + count(&ph, "cis")?, + count(&ph, "unknown")?, + d.s_or("compound_het_pairs", "0")? + )]; + let trans = list_or_empty(d, "compound_het_trans")?; + for x in &trans { + out.push(format!( + " {} {} in trans on {} reads or read pairs", + ljust(&x.s("gene")?, 8), + x.s("variants")?, + x.s("reads")? + )); + } + if let Some(more) = more_in_trans(&ph, trans.len())? { + // The Python takes len() of the list here, which a missing or + // null list cannot give. + d.at("compound_het_trans")?.len()?; + out.push(format!(" ... {more} more in the pairs table")); + } + out.push(format!(" ({COMPHET_NOTE})")); + Ok(out) +} + +/// The text report's block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "slivar", Some(TITLE))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if !head.ran { + return Ok(w); + } + let d = D::of(&s.section("slivar")); + w.push(format!(" Prioritized variants: {}", d.s("prioritized")?)); + if d.has("frequency_unknown") { + w.push(format!( + " Frequency unknown (prioritized, not counted as rare): {}", + d.s("frequency_unknown")? + )); + } + if d.has("compound_het_variants") { + w.push(format!( + " Compound het candidates: {} variants across {} genes", + d.s("compound_het_variants")?, + d.s("compound_het_genes")? + )); + } + if d.has("compound_het_phase") { + w.extend(phase_text(&d)?); + } else if d.has("compound_het_variants") { + w.push( + " (Candidates only: unphased single-sample data. See docs/31-slivar.md)".to_owned(), + ); + } + w.push(String::new()); + Ok(w) +} + +/// The phase lines of the card: the counts, then the pairs in trans. +fn phase_html(d: &D<'_>) -> Result, Malformed> { + let ph = d.at("compound_het_phase")?; + let mut out = vec![Line::Stat( + "Pairs in trans / cis / unknown".to_owned(), + Inline::Text(format!( + "{} / {} / {}", + count(&ph, "trans")?, + count(&ph, "cis")?, + count(&ph, "unknown")? + )), + )]; + let trans = list_or_empty(d, "compound_het_trans")?; + if !trans.is_empty() { + let mut rows = Vec::new(); + for x in &trans { + let variants = x.at("variants")?; + rows.push(vec![ + Cell::Text(x.s("gene")?), + Cell::Text(variants.text()?), + Cell::Text(x.s("reads")?), + viewer_of(&locus_text(&variants)?)?, + ]); + } + out.push(Line::Table(Table { + head: ["Gene", "Pair in trans", "Reads"] + .map(str::to_owned) + .to_vec(), + rows, + viewer: true, + layout: Layout::Indented, + })); + out.push(Line::ViewerNote); + } + Ok(out) +} + +/// The HTML report's card, always there. +pub fn html(s: &Summary) -> Result { + let sec = s.section("slivar"); + let d = D::of(&sec); + let mut body = vec![Line::Stat( + "Prioritized variants".to_owned(), + Inline::Text(d.s_or_none("prioritized")?), + )]; + if d.has("frequency_unknown") { + body.push(Line::Stat( + "Frequency unknown (not rare)".to_owned(), + Inline::Text(d.s("frequency_unknown")?), + )); + } + if d.has("compound_het_variants") { + body.push(Line::Stat( + "Compound het candidates".to_owned(), + Inline::Text(format!( + "{} variants, {} genes", + d.s("compound_het_variants")?, + d.s("compound_het_genes")? + )), + )); + } + let mut caveats = Vec::new(); + if d.has("compound_het_phase") { + body.extend(phase_html(&d)?); + caveats.push(COMPHET_NOTE); + } + let card = html::card( + &sec, + Card { + title: TITLE.to_owned(), + caveats, + body, + ..Card::default() + }, + )?; + Ok(HtmlPart { + cards: vec![card], + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/couple.rs b/rust/crates/og-render/src/sections/couple.rs index cf30455af..66a32335e 100644 --- a/rust/crates/og-render/src/sections/couple.rs +++ b/rust/crates/og-render/src/sections/couple.rs @@ -1,2 +1,76 @@ -//! The report section `couple`. Empty until its unit of the 1.0.0 plan -//! lands. +//! Step 39's couple carrier screen: the counts, then each shared and +//! X-linked gene with the chance for a child and both partners' findings. + +use crate::model::{Card, HtmlPart, Line, Malformed, State, Summary}; +use crate::notes::COUPLE_NOTE; +use crate::sections::clinvar::D; +use crate::{html, text}; + +/// The lines, the same in both reports; the last is [`COUPLE_NOTE`]. +pub fn lines(d: &D<'_>) -> Result, Malformed> { + let partner = match d.opt("partner")? { + Some(p) => p.text()?, + None => "the partner".to_owned(), + }; + let count = |k: &str| -> Result { d.get(k)?.map_or(Ok(0), |v| v.len()) }; + let mut out = vec![format!( + "Partner: {partner}. Genes where both carry a change: {}; X-linked findings: {}; genes where only \ + one partner carries a change: {}", + count("shared")?, + count("x_linked")?, + d.s_or("one_partner", "0")? + )]; + for (label, key) in [("Shared", "shared"), ("X-linked", "x_linked")] { + let Some(list) = d.get(key)? else { continue }; + for x in list.items()? { + out.push(format!( + "{label}: {} ({}): {}. This sample: {}. {partner}: {}", + x.s("gene")?, + x.s("condition")?, + x.s("risk")?, + x.s("own")?, + x.s("partner_findings")? + )); + } + } + out.push(COUPLE_NOTE.to_owned()); + Ok(out) +} + +/// The text report's block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "couple", Some("Couple Carrier Screen (step 39)"))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if head.ran { + let d = D::of(&s.section("couple")); + w.extend(lines(&d)?.iter().map(|l| format!(" {l}"))); + w.push(String::new()); + } + Ok(w) +} + +/// The HTML report's card, when the screen ran or its step failed. +pub fn html(s: &Summary) -> Result { + let sec = s.section("couple"); + if sec.state == State::Missing { + return Ok(HtmlPart::default()); + } + let mut body = lines(&D::of(&sec))?; + body.pop(); + let card = html::card( + &sec, + Card { + title: "Couple Carrier Screen".to_owned(), + full: true, + caveats: vec![COUPLE_NOTE], + body: body.into_iter().map(Line::Para).collect(), + ..Card::default() + }, + )?; + Ok(HtmlPart { + cards: vec![card], + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/cpsr.rs b/rust/crates/og-render/src/sections/cpsr.rs index 4db6327f4..c2ae094a0 100644 --- a/rust/crates/og-render/src/sections/cpsr.rs +++ b/rust/crates/og-render/src/sections/cpsr.rs @@ -1 +1,203 @@ -//! The report section `cpsr`. Empty until its unit of the 1.0.0 plan lands. +//! Step 17's cancer predisposition screen (CPSR): its classification +//! breakdown, the two ClinVar releases, CPSR's own benign calls among the +//! final P/LP rows, the rows where CPSR and ClinVar disagree, the rows below +//! the depth floor and the cDNA read check. + +use crate::model::{Card, Colour, HtmlPart, Inline, Line, Malformed, Summary}; +use crate::notes::CPSR_NOTE; +use crate::sections::cdna::cdna_text; +use crate::sections::clinvar::{D, call_text}; +use crate::{html, text}; + +/// One P/LP or disagreement row: variant, gene, the classes and codes. +pub fn row_text(x: &D<'_>) -> Result { + let mut first = format!("final {}", x.s("final")?); + if let Some(a) = x.opt("authority")? { + first += &format!(" ({})", a.text()?); + } + let mut parts = vec![first]; + if let Some(c) = x.opt("clinvar")? { + parts.push(format!("ClinVar {}", c.text()?)); + } + if let Some(c) = x.opt("cpsr")? { + let mut p = format!("CPSR's own {}", c.text()?); + if let Some(codes) = x.opt("acmg_codes")? { + p += &format!(" ({})", codes.text()?); + } + parts.push(p); + } + Ok(format!( + "{} {}: {}", + x.s("gene")?, + x.s("variant")?, + parts.join(", ") + )) +} + +/// `dbs.get(key) or 'unknown'`. +fn release(dbs: &D<'_>, key: &str) -> Result { + match dbs.opt(key)? { + Some(v) => v.text(), + None => Ok("unknown".to_owned()), + } +} + +/// The lines both reports share beyond the breakdown. +pub fn lines(d: &D<'_>, dbs: &D<'_>) -> Result, Malformed> { + let mut out = Vec::new(); + if !d.has("plp") { + return Ok(out); + } + out.push(format!( + "ClinVar release in CPSR's bundle: {}; step 6 used the ClinVar file of {}", + release(dbs, "cpsr_clinvar_release")?, + release(dbs, "clinvar_release")? + )); + if d.flag("own_class_column")? { + out.push(format!( + "Final P/LP rows that CPSR's own rules class benign or likely benign: {}", + d.s("final_plp_own_benign")? + )); + let dis = d.at("disagreements")?; + out.push(format!( + "Rows where ClinVar and CPSR's own class disagree on P/LP: {}", + dis.len()? + )); + for x in dis.items()? { + out.push(format!(" {}", row_text(&x)?)); + } + } else { + out.push( + "The classification table has no CPSR_CLASSIFICATION column, so no disagreement can be shown." + .to_owned(), + ); + } + out.push(format!( + "P/LP rows below the depth floor (DP below {}): {}", + d.s("call_floor_dp")?, + d.s("low_confidence")? + )); + let plp = d.at("plp")?.items()?; + for x in &plp { + if x.flag("low_confidence")? { + out.push(format!(" {}; {}", row_text(x)?, call_text(x)?)); + } + } + if d.has("cdna_flagged") { + out.push(format!( + "cDNA read check of the P/LP rows (alt reads clipped at exon edges): {} flagged", + d.s("cdna_flagged")? + )); + for x in &plp { + if x.at("cdna")?.at("check")?.is("flagged") { + out.push(format!(" {}; reads: {}", row_text(x)?, cdna_text(x)?)); + } + } + } + Ok(out) +} + +fn databases(s: &Summary) -> Result, Malformed> { + Ok(D::new(s.need("databases")?, "summary.databases")) +} + +/// The text report's block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "cpsr", Some("Cancer Predisposition Screening (CPSR)"))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if !head.ran { + return Ok(w); + } + let d = D::of(&s.section("cpsr")); + if d.flag("html_report")? { + w.push(" HTML report in cpsr/ (open in a browser)".to_owned()); + } + if let Some(cls) = d.opt("classification")? { + w.push(format!( + " Classification breakdown ({}):", + d.s("classification_column")? + )); + for (k, v) in cls.pairs()? { + w.push(format!(" {k}: {}", v.text()?)); + } + } else if d.has("classification_column") { + w.push(" Classification table has no CLASSIFICATION column.".to_owned()); + } + for line in lines(&d, &databases(s)?)? { + w.push(format!(" {line}")); + } + w.push(format!(" {CPSR_NOTE}")); + w.push(String::new()); + Ok(w) +} + +/// Python's `str.strip()`: whitespace as `str.isspace` counts it. +fn strip(s: &str) -> &str { + pycompat::text::strip(s, None) +} + +/// `os.path.dirname(p)`. +fn dirname(p: &str) -> &str { + let Some(i) = p.rfind('/') else { + return ""; + }; + let head = &p[..=i]; + if head.chars().all(|c| c == '/') { + head + } else { + head.trim_end_matches('/') + } +} + +/// The HTML report's card, always there, with a link to CPSR's own report +/// beside this one. +pub fn html(s: &Summary) -> Result { + let sec = s.section("cpsr"); + let d = D::of(&sec); + let report = if d.flag("html_report")? { + Inline::Badge("Complete".to_owned(), Colour::State) + } else { + Inline::Badge("No HTML".to_owned(), Colour::Gray) + }; + let mut body = vec![Line::Stat("CPSR report".to_owned(), report)]; + if let Some(cls) = d.opt("classification")? { + for (k, v) in cls.pairs()? { + body.push(Line::Stat(k.to_owned(), Inline::Text(v.text()?))); + } + } + for line in lines(&d, &databases(s)?)? { + body.push(Line::Note(strip(&line).to_owned())); + } + let card = html::card( + &sec, + Card { + title: "Cancer Predisposition".to_owned(), + caveats: vec![CPSR_NOTE], + body, + ..Card::default() + }, + )?; + let mut files = Vec::new(); + let src = sec.source.filter(|v| crate::model::truthy(v)); + if sec.state.ran() + && let Some(src) = src + && d.flag("html_report")? + { + let src = D::new(src, "sections.cpsr.source"); + let src = src.string()?; + files.push(if src.ends_with(".html") { + (src.to_owned(), "CPSR HTML report".to_owned()) + } else { + ( + format!("{}/", dirname(src)), + "the cpsr folder, which holds the CPSR HTML report".to_owned(), + ) + }); + } + Ok(HtmlPart { + cards: vec![card], + files, + }) +} diff --git a/rust/crates/og-render/src/sections/cyp21a2.rs b/rust/crates/og-render/src/sections/cyp21a2.rs index 3699dbddb..27456d9ca 100644 --- a/rust/crates/og-render/src/sections/cyp21a2.rs +++ b/rust/crates/og-render/src/sections/cyp21a2.rs @@ -1,2 +1,46 @@ -//! The report section `cyp21a2`. Empty until its unit of the 1.0.0 plan -//! lands. +//! Step 35's CYP21A2 call (congenital adrenal hyperplasia carriers): the +//! copies from depth and the six sites where CYP21A2 differs from its +//! pseudogene CYP21A1P, one line in both reports. + +use pycompat::json::Value; + +use crate::model::{Malformed, get, is_str, py, truthy}; + +const LABEL: &str = "CYP21A2 (RCCX module, from depth and six PSV sites)"; + +/// The CYP21A2 line; not assessed only when the table says so or is not +/// there. +pub fn cyp21a2_line(c: Option<&Value>) -> Result { + let Some(c) = c.filter(|c| truthy(c)) else { + return Ok(format!( + "{LABEL}: not assessed (step 35's _cyp21a2_psv.tsv is not in this run's output)" + )); + }; + let at = |k: &str| get(c, k, "sections.parascopy.data.cyp21a2"); + let or = |k: &str| at(k).and_then(|v| v.map_or(Ok(".".to_owned()), py)); + if !is_str(at("status")?, "called") { + let why = match at("message")? + .filter(|v| truthy(v)) + .or(at("reason")?.filter(|v| truthy(v))) + { + Some(v) => py(v)?, + None => ".".to_owned(), + }; + let why = why.strip_prefix("not assessed: ").unwrap_or(&why); + return Ok(format!("{LABEL}: not assessed ({why})")); + } + let mut value = format!( + "{} CYP21A2 copies, module total {} of {}, finding {}", + or("cyp21a2_copies")?, + or("module_total")?, + or("expected_total")?, + or("finding")? + ); + if let Some(v) = at("conversion_sites")?.filter(|v| !is_str(Some(v), ".")) { + value += &format!(" at {}", py(v)?); + } + if let Some(v) = at("reading")?.filter(|v| !is_str(Some(v), ".")) { + value += &format!(". Reading: {}", py(v)?); + } + Ok(format!("{LABEL}: {value}")) +} diff --git a/rust/crates/og-render/src/sections/deep_intronic.rs b/rust/crates/og-render/src/sections/deep_intronic.rs index 8bda20f8a..43d5c09c5 100644 --- a/rust/crates/og-render/src/sections/deep_intronic.rs +++ b/rust/crates/og-render/src/sections/deep_intronic.rs @@ -1,2 +1,58 @@ -//! The report section `deep_intronic`. Empty until its unit of the 1.0.0 -//! plan lands. +//! Step 06's known deep intronic sites, reported on every run: they lie far +//! from any exon, so most filters miss them. + +use crate::model::{Cell, Layout, Line, Malformed, Table}; +use crate::sections::clinvar::D; + +/// One site's call: `het, GQ 40, DP 30`, `ref, GQ 50 (gVCF block)`, +/// `no call: why`. +pub fn call(x: &D<'_>) -> Result { + let c = x.at("call")?; + let detail = x.at("detail")?; + if c.is("no call") { + let mut out = "no call".to_owned(); + if detail.truthy() { + out += &format!(": {}", detail.text()?); + } + return Ok(out); + } + let mut out = c.string()?.to_owned(); + if detail.truthy() { + out += &format!(", {}", detail.text()?); + } + Ok(out) +} + +/// The table of the sites, with their Viewer links, and the Viewer note. +pub fn table(sites: &D<'_>) -> Result, Malformed> { + let st = + |x: &D<'_>, k: &str| -> Result { Ok(x.at(k)?.string()?.to_owned()) }; + let mut rows = Vec::new(); + for x in sites.items()? { + let (chrom, pos) = (x.at("chrom")?, x.at("pos")?); + rows.push(vec![ + Cell::Text(st(&x, "gene")?), + Cell::Text(st(&x, "hgvs")?), + Cell::Text(format!( + "{}:{} {}>{}", + chrom.string()?, + pos.text()?, + st(&x, "ref")?, + st(&x, "alt")? + )), + Cell::Text(call(&x)?), + Cell::viewer(chrom.v, pos.v, None)?, + ]); + } + Ok(vec![ + Line::Table(Table { + head: ["Gene", "Variant", "Position", "Call"] + .map(str::to_owned) + .to_vec(), + rows, + viewer: true, + layout: Layout::Indented, + }), + Line::ViewerNote, + ]) +} diff --git a/rust/crates/og-render/src/sections/dosage.rs b/rust/crates/og-render/src/sections/dosage.rs index 49d9dbfa1..bb51531e2 100644 --- a/rust/crates/og-render/src/sections/dosage.rs +++ b/rust/crates/og-render/src/sections/dosage.rs @@ -1,2 +1,59 @@ -//! The report section `dosage`. Empty until its unit of the 1.0.0 plan -//! lands. +//! Step 35's gene-dosage loci from read depth (HP, LCE3B/LCE3C, +//! CCL3L1/CCL3L3), one line each in both reports. + +use pycompat::json::Value; + +use crate::model::{Malformed, is_str, need, py, truthy}; + +/// The display name of a dosage locus. +fn name(locus: &str) -> &str { + match locus { + "HP" => "HP (haptoglobin)", + "LCE3BC" => "LCE3B/LCE3C", + "CCL3L1" => "CCL3L1/CCL3L3", + other => other, + } +} + +/// One line per dosage row, or one line saying why there is none. +pub fn dosage_lines(rows: Option<&Value>) -> Result, Malformed> { + let rows = match rows { + None | Some(Value::Null) => { + return Ok(vec![ + "Gene-dosage loci (HP, LCE3B/LCE3C, CCL3L1): not assessed (step 35's \ + _cn_dosage.tsv is not in this run's output)" + .to_owned(), + ]); + } + Some(r) if !truthy(r) => { + return Ok(vec![ + "Gene-dosage loci (HP, LCE3B/LCE3C, CCL3L1): not assessed (step 35's dosage \ + table has no row)" + .to_owned(), + ]); + } + Some(Value::Array(r)) => r, + Some(_) => { + return Err(Malformed( + "sections.parascopy.data.dosage is not a list".into(), + )); + } + }; + let path = "sections.parascopy.data.dosage[]"; + let mut out = Vec::new(); + for r in rows { + let f = |k: &str| need(r, k, path); + let label = format!("{} copies (from depth)", name(&py(f("locus")?)?)); + let value = if is_str(Some(f("status")?), "called") { + let mut v = format!("{} (estimate {})", py(f("copies")?)?, py(f("estimate")?)?); + if !is_str(Some(f("reading")?), ".") { + v += &format!(". Reading: {}", py(f("reading")?)?); + } + v + } else { + format!("{} ({})", py(f("status")?)?, py(f("reason")?)?) + }; + out.push(format!("{label}: {value}")); + } + Ok(out) +} diff --git a/rust/crates/og-render/src/sections/exomiser.rs b/rust/crates/og-render/src/sections/exomiser.rs index 1285f6361..55fb6ff58 100644 --- a/rust/crates/og-render/src/sections/exomiser.rs +++ b/rust/crates/og-render/src/sections/exomiser.rs @@ -4,8 +4,8 @@ use pycompat::json::Value; use crate::model::{ - Card, Cell, HtmlPart, Inline, Line, Malformed, Section, State, Summary, Table, get, is_str, - need, py, truthy, + Card, Cell, HtmlPart, Inline, Layout, Line, Malformed, Section, State, Summary, Table, get, + is_str, need, py, truthy, }; use crate::notes::EXOMISER_NOTE; use crate::{html, text}; @@ -227,8 +227,9 @@ pub fn html(s: &Summary) -> Result { .to_vec(), rows, viewer: true, - one_line: false, + layout: Layout::Indented, })); + card.body.push(Line::ViewerNote); } card.caveats.push(EXOMISER_NOTE); card.body.push(Line::Note(format!( diff --git a/rust/crates/og-render/src/sections/g2.rs b/rust/crates/og-render/src/sections/g2.rs index 6f24ec5e0..a14fdedec 100644 --- a/rust/crates/og-render/src/sections/g2.rs +++ b/rust/crates/og-render/src/sections/g2.rs @@ -1 +1,89 @@ -//! The report section `g2`. Empty until its unit of the 1.0.0 plan lands. +//! A genotype rule's result (several sites read together), and APOL1's G2 +//! beside it: G2 (rs71785313) is read by its own helper, so the rule's +//! G0/G0 or G0/G1 says nothing of it, and the report prints the combined +//! answer of `summary.json` (`traits.rules[APOL1].g2`) under the rule. + +use pycompat::json::Value; + +use crate::model::{Inline, Malformed, get, is_str, need, py, truthy}; + +const PATH: &str = "sections.traits.data.rules[]"; + +/// How a rule's sites were phased, in words; `None` for a phase with no +/// words. +fn phase_text(phase: &Value) -> Option<&'static str> { + let Value::Str(p) = phase else { return None }; + Some(match p.as_str() { + "read_backed" => "phased from the reads", + "not_needed" => "no phase needed", + "not_used" => "phase not used", + "unphased" => "not phased", + _ => return None, + }) +} + +/// A rule's result: its label when called, else its status; and how it was +/// phased. +pub fn rule_result(r: &Value) -> Result { + let st = need(r, "status", PATH)?; + let res = if is_str(Some(st), "called") { + py(need(r, "result", PATH)?)? + } else { + py(st)?.replace('_', " ") + }; + Ok(match phase_text(need(r, "phase", PATH)?) { + Some(ph) => format!("{res} ({ph})"), + None => res, + }) +} + +/// The rule's G2 object, when it has one (APOL1 only). +fn g2(r: &Value) -> Result, Malformed> { + Ok(get(r, "g2", PATH)?.filter(|v| truthy(v))) +} + +/// The APOL1 rule's result with G2, or `None` for any other rule. +pub fn g2_result(r: &Value) -> Result, Malformed> { + let Some(g) = g2(r)? else { return Ok(None) }; + let res = need(g, "result", "the rule's g2")?; + Ok(Some(if is_str(Some(res), ".") { + "not read".to_owned() + } else { + py(res)? + })) +} + +/// The text report's G2 line under the APOL1 rule. +pub fn g2_text(r: &Value) -> Result, Malformed> { + let Some(g) = g2(r)? else { return Ok(None) }; + let f = |k: &str| need(g, k, "the rule's g2").and_then(py); + let res = g2_result(r)?.unwrap_or_default(); + Ok(Some(format!( + " With G2 (rs71785313): {res}. {} {}", + f("text")?, + f("reading")? + ))) +} + +/// The G2 result under the rule's result in the HTML table: nothing +/// without one. +pub fn g2_result_html(r: &Value) -> Result, Malformed> { + Ok(match g2_result(r)?.filter(|x| !x.is_empty()) { + Some(res) => vec![Inline::Break, Inline::Text(format!("With G2: {res}"))], + None => Vec::new(), + }) +} + +/// The G2 reading and its explanation under the rule's reading in the HTML +/// table: nothing without G2. +pub fn g2_reading_html(r: &Value) -> Result, Malformed> { + let Some(g) = g2(r)? else { + return Ok(Vec::new()); + }; + let f = |k: &str| need(g, k, "the rule's g2").and_then(py); + Ok(vec![ + Inline::Break, + Inline::Text(format!("With G2: {}", f("reading")?)), + Inline::Small(f("text")?), + ]) +} diff --git a/rust/crates/og-render/src/sections/haplogroups.rs b/rust/crates/og-render/src/sections/haplogroups.rs index 8f28f8f32..94fdad73e 100644 --- a/rust/crates/og-render/src/sections/haplogroups.rs +++ b/rust/crates/og-render/src/sections/haplogroups.rs @@ -1,2 +1,199 @@ -//! The report section `haplogroups`. Empty until its unit of the 1.0.0 plan -//! lands. +//! The haplogroups: the mitochondrial one with haplocheck's contamination +//! verdict (step 12), and the Y chromosome section, which shows Yleaf's +//! haplogroup (step 37), the second placement on the current public tree +//! ([`crate::sections::y_tree`]) and the Y-STR markers +//! ([`crate::sections::ystr`]). Each haplogroup carries the links where it +//! can be explored further. + +use pycompat::json::Value; + +use crate::model::{Card, Colour, HtmlPart, Inline, Line, Malformed, Section, State, Summary, py}; +use crate::notes::EXPLORE_NOTE; +use crate::sections::ancestry::{explore_html, explore_lines, get_text, need_text}; +use crate::sections::{y_tree, ystr}; +use crate::{html, text}; + +/// The Status badge of a section that ran: Stale for an earlier run's +/// result, else Complete. +pub fn done_badge(sec: &Section<'_>) -> Inline { + if sec.state == State::Stale { + Inline::Badge("Stale".to_owned(), Colour::Yellow) + } else { + Inline::Badge("Complete".to_owned(), Colour::Green) + } +} + +/// Whether a card's lines are shown: the card says Not run, Failed or +/// Unreadable in their place otherwise. +pub fn shows_body(sec: &Section<'_>) -> bool { + !matches!( + sec.state, + State::Missing | State::Failed | State::Unreadable + ) +} + +fn path(sec: &Section<'_>) -> String { + format!("sections.{}.data", sec.key) +} + +/// haplocheck's verdict: whether the reads hold two mtDNA haplogroups. +pub fn contamination_text(sec: &Section<'_>) -> Result { + let Some(status) = sec.get("contamination_status")? else { + return Ok("not checked (needs step 20's Mutect2 chrM calls)".to_owned()); + }; + let word = match status { + Value::Str(s) => match s.as_str() { + "YES" => "YES, two mtDNA haplogroups in the reads (another person's DNA?)".to_owned(), + "NO" => "no".to_owned(), + "ND" => "not determined".to_owned(), + _ => s.clone(), + }, + Value::Array(_) | Value::Object(_) => { + return Err(Malformed(format!( + "{}.contamination_status is not a value", + path(sec) + ))); + } + v => py(v)?, + }; + let level = get_text(sec.data, "contamination_level", ".", &path(sec))?; + Ok(format!("{word} (level {level})")) +} + +/// The text report's mitochondrial block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "haplogroup", Some("Mitochondrial Haplogroup"))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if head.ran { + let sec = s.section("haplogroup"); + let p = path(&sec); + w.push(format!( + " Haplogroup: {}", + need_text(sec.data, "haplogroup", &p)? + )); + w.push(format!( + " Contamination (haplocheck): {}", + contamination_text(&sec)? + )); + w.extend( + explore_lines(sec.data, &p)? + .iter() + .map(|l| format!(" {l}")), + ); + w.push(String::new()); + } + Ok(w) +} + +/// The HTML report's mitochondrial card. It is always there, so a run +/// without step 12 says so. +pub fn html(s: &Summary) -> Result { + let sec = s.section("haplogroup"); + let mut c = Card { + title: "Mitochondrial Haplogroup".to_owned(), + ..Card::default() + }; + if shows_body(&sec) { + let p = path(&sec); + c.body = vec![ + Line::Stat("Status".to_owned(), done_badge(&sec)), + Line::Stat( + "Haplogroup".to_owned(), + Inline::Text(get_text(sec.data, "haplogroup", ".", &p)?), + ), + Line::Stat( + "Contamination (haplocheck)".to_owned(), + Inline::Text(contamination_text(&sec)?), + ), + ]; + let (links, note) = explore_html(sec.data, &p)?; + c.body.extend(links); + if note { + c.caveats.push(EXPLORE_NOTE); + } + } + Ok(HtmlPart { + cards: vec![html::card(&sec, c)?], + files: Vec::new(), + }) +} + +/// The text report's block of Yleaf's haplogroup. +fn y_text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "y_haplogroup", Some("Y-Chromosome Haplogroup (Yleaf)"))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if head.ran { + let sec = s.section("y_haplogroup"); + let p = path(&sec); + let n = |k: &str| need_text(sec.data, k, &p); + w.push(format!( + " Haplogroup: {} ({} markers, QC-score {})", + n("haplogroup")?, + n("valid_markers")?, + n("qc_score")? + )); + w.extend( + explore_lines(sec.data, &p)? + .iter() + .map(|l| format!(" {l}")), + ); + w.push(String::new()); + } + Ok(w) +} + +/// The HTML report's card of Yleaf's haplogroup, when step 37 has a result. +fn y_card(s: &Summary) -> Result, Malformed> { + let sec = s.section("y_haplogroup"); + if sec.state == State::Missing { + return Ok(None); + } + let mut c = Card { + title: "Y-Chromosome Haplogroup".to_owned(), + ..Card::default() + }; + if shows_body(&sec) { + let p = path(&sec); + for (label, key) in [ + ("Haplogroup", "haplogroup"), + ("Markers", "valid_markers"), + ("QC-score", "qc_score"), + ] { + c.body.push(Line::Stat( + label.to_owned(), + Inline::Text(get_text(sec.data, key, ".", &p)?), + )); + } + let (links, note) = explore_html(sec.data, &p)?; + c.body.extend(links); + if note { + c.caveats.push(EXPLORE_NOTE); + } + } + Ok(Some(html::card(&sec, c)?)) +} + +/// The text report's Y chromosome blocks: Yleaf, the second placement, +/// the Y-STR markers. +pub fn y_chromosome_text(s: &Summary) -> Result, Malformed> { + let mut w = y_text(s)?; + w.extend(y_tree::text(s)?); + w.extend(ystr::text(s)?); + Ok(w) +} + +/// The HTML report's Y chromosome cards, each when its step has a result. +pub fn y_chromosome_html(s: &Summary) -> Result { + let cards = [y_card(s)?, y_tree::card(s)?, ystr::card(s)?] + .into_iter() + .flatten() + .collect(); + Ok(HtmlPart { + cards, + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/hba.rs b/rust/crates/og-render/src/sections/hba.rs index d3f347f6a..fdcd0f9ee 100644 --- a/rust/crates/og-render/src/sections/hba.rs +++ b/rust/crates/og-render/src/sections/hba.rs @@ -1 +1,25 @@ -//! The report section `hba`. Empty until its unit of the 1.0.0 plan lands. +//! Step 35's HBA1 and HBA2 count from read depth (alpha-thalassaemia), one +//! line in both reports. + +use pycompat::json::Value; + +use crate::model::{Malformed, get, is_str, py}; + +/// `HBA1 and HBA2 together (alpha-globin genes, from depth): ...`. +pub fn hba_line(h: &Value) -> Result { + let at = |k: &str| get(h, k, "sections.parascopy.data.hba"); + let or = |k: &str, d: &str| at(k).and_then(|v| v.map_or(Ok(d.to_owned()), py)); + let value = if is_str(at("status")?, "called") { + format!( + "{} ({}; estimate {})", + or("alpha_globin_genes", ".")?, + or("message", ".")?, + or("estimate", ".")? + ) + } else { + or("message", "not assessed")? + }; + Ok(format!( + "HBA1 and HBA2 together (alpha-globin genes, from depth): {value}" + )) +} diff --git a/rust/crates/og-render/src/sections/hla.rs b/rust/crates/og-render/src/sections/hla.rs index 79b3352a3..548422b68 100644 --- a/rust/crates/og-render/src/sections/hla.rs +++ b/rust/crates/og-render/src/sections/hla.rs @@ -1 +1,344 @@ -//! The report section `hla`. Empty until its unit of the 1.0.0 plan lands. +//! HLA typing (step 08): T1K's alleles per locus with their quality, the +//! lines derived from them (the coeliac DQ types, the DQB1*02 dose and the +//! allele table of bin/hla_alleles.tsv), OptiType's second opinion on the +//! class I genes, and the KIR genes. + +use pycompat::json::Value; + +use crate::html; +use crate::model::{Card, Inline, Line, Malformed, Summary, get, is_str, need, num, py, truthy}; +use crate::notes::{HLA_DERIVED_NOTE, HLA_NOTE, KIR_NOTE}; +use crate::sections::pgx::{items, left, or_text, text_of}; +use crate::text; + +/// A quality-0 allele with more candidates than this reads "no confident +/// call" (og summary collect keeps this many of them; its own copy is +/// private, so tests/port_report_pgx.rs checks the two caps agree). +pub const MAX_CANDIDATES: f64 = 3.0; + +/// T1K's quality as a number; one that is not a number counts as -1, as in +/// step 36. +fn quality(q: &Value) -> Result { + match q { + Value::Str(s) => Ok(pycompat::num::parse_float(s).unwrap_or(-1.0)), + _ => num(Some(q)).ok_or_else(|| Malformed("an HLA quality is not a number".into())), + } +} + +/// One HLA locus: each allele with its T1K quality, and the low-confidence +/// mark. A quality-0 allele with more than [`MAX_CANDIDATES`] candidates +/// reads "no confident call"; the HFE row carries its label. +pub fn hla_text(locus: &Value) -> Result { + let p = "sections.hla.data.loci[]"; + let q = items(get(locus, "quality", p)?.filter(|v| truthy(v)), p)?; + let n = items(get(locus, "candidates", p)?.filter(|v| truthy(v)), p)?; + let mut parts = Vec::new(); + for (i, a) in items(Some(need(locus, "alleles", p)?), p)? + .iter() + .enumerate() + { + let blank = Value::Str(String::new()); + let qi = q.get(i).unwrap_or(&blank); + let has_q = !is_str(Some(qi), ""); + let many = match n.get(i) { + Some(c) => { + num(Some(c)) + .ok_or_else(|| Malformed("an HLA candidate count is not a number".into()))? + > MAX_CANDIDATES + } + None => false, + }; + if many && has_q && quality(qi)? <= 0.0 { + parts.push(format!( + "no confident call ({} candidates, quality {})", + py(&n[i])?, + py(qi)? + )); + } else { + let mut t = text_of(a, "an HLA allele")?; + if has_q { + t += &format!(" (quality {})", py(qi)?); + } + parts.push(t); + } + } + let mut txt = parts.join(" / "); + if txt.is_empty() { + txt = "no call".to_owned(); + } + if get(locus, "low_confidence", p)?.is_some_and(truthy) { + txt += "; low confidence (T1K quality 0 or below)"; + if get(locus, "withheld_from_pharmcat", p)?.is_some_and(truthy) { + txt += ", not passed to PharmCAT"; + } + } + if let Some(l) = get(locus, "label", p)?.filter(|v| truthy(v)) { + txt += &format!(" [{}]", py(l)?); + } + Ok(txt) +} + +/// The order of an allele's state in the table: present ones first. +fn rank(state: &Value) -> u8 { + match state { + Value::Str(s) => match s.as_str() { + "present" => 0, + "uncertain" => 1, + "absent" => 2, + "not_determined" => 3, + _ => 9, + }, + _ => 9, + } +} + +/// The derived HLA lines both reports print: the coeliac DQ line, the +/// DQB1*02 dose and the allele table, present ones first. +pub fn hla_derived_lines(d: Option<&Value>) -> Result, Malformed> { + let Some(d) = d.filter(|v| truthy(v)) else { + return Ok(Vec::new()); + }; + let p = "sections.hla.data.derived"; + let empty = Value::Object(Default::default()); + let c = get(d, "coeliac", p)? + .filter(|v| truthy(v)) + .unwrap_or(&empty); + let cp = "sections.hla.data.derived.coeliac"; + let mut out = Vec::new(); + if is_str(get(c, "status", cp)?, "derived") { + let mut dq25 = text_of(need(c, "dq2_5", cp)?, "the DQ2.5 state")?; + if let Some(ph) = get(c, "dq2_5_phase", cp)?.filter(|v| truthy(v)) { + dq25 += &format!(", {}", py(ph)?); + } + out.push(format!( + "Coeliac DQ: DQ2.5 {dq25}; DQ2.2 {}; DQ8 {}", + py(need(c, "dq2_2", cp)?)?, + py(need(c, "dq8", cp)?)? + )); + } else { + out.push(format!( + "Coeliac DQ: not determined ({})", + or_text(get(c, "reason", cp)?, "no DQA1 and DQB1 call")? + )); + } + let dose = match get(c, "dqb1_02_dose", cp)? { + None | Some(Value::Null) => "not determined".to_owned(), + Some(v) => py(v)?, + }; + out.push(format!("DQB1*02 copies: {dose}")); + let mut alleles: Vec<&Value> = items( + get(d, "alleles", p)?.filter(|v| truthy(v)), + "sections.hla.data.derived.alleles", + )? + .iter() + .collect(); + let ap = "sections.hla.data.derived.alleles[]"; + for a in &alleles { + need(a, "state", ap)?; + } + alleles.sort_by_key(|a| rank(need(a, "state", ap).expect("checked above"))); + for a in alleles { + let mut state = text_of(need(a, "state", ap)?, "an allele state")?.replace('_', " "); + if let Some(r) = get(a, "reason", ap)?.filter(|v| truthy(v)) { + state += &format!(" ({})", py(r)?); + } + out.push(format!( + "{} {state}: {} ({})", + py(need(a, "allele", ap)?)?, + py(need(a, "association", ap)?)?, + py(need(a, "citation", ap)?)? + )); + } + Ok(out) +} + +/// Step 08's OptiType check: (gene, line) per class I gene, or one +/// (`""`, line) with the reason the callers were not compared. +pub fn hla_optitype_lines(ot: &Value) -> Result, Malformed> { + let p = "sections.hla.data.optitype"; + let genes = need(ot, "genes", p)?; + if !truthy(genes) { + return Ok(vec![( + String::new(), + format!("not compared: {}", py(need(ot, "reason", p)?)?), + )]); + } + let gp = "sections.hla.data.optitype.genes[]"; + let mut out = Vec::new(); + for g in items(Some(genes), p)? { + let agree = need(g, "agree", gp)?; + let word = match agree { + Value::Str(s) if s == "yes" => "agree".to_owned(), + Value::Str(s) if s == "no" => "disagree".to_owned(), + _ => or_text(Some(agree), "not compared")?, + }; + let mut line = format!( + "{word}: T1K {}, OptiType {}", + py(need(g, "t1k", gp)?)?, + py(need(g, "optitype", gp)?)? + ); + let note = need(g, "note", gp)?; + if !(is_str(Some(note), "") || is_str(Some(note), "-")) { + line += &format!(" ({})", py(note)?); + } + out.push((py(need(g, "gene", gp)?)?, line)); + } + Ok(out) +} + +/// Step 08's KIR genes: (gene, alleles) per typed gene, then (`""`, line) +/// for the genes with no allele, or the reason nothing was typed. +pub fn kir_lines(kir: &Value) -> Result, Malformed> { + let p = "sections.hla.data.kir"; + let reason = need(kir, "reason", p)?; + if truthy(reason) { + return Ok(vec![(String::new(), py(reason)?)]); + } + let gp = "sections.hla.data.kir.typed[]"; + let mut out = Vec::new(); + for g in items(Some(need(kir, "typed", p)?), p)? { + let alleles = items(Some(need(g, "alleles", gp)?), gp)?; + let quals = items(Some(need(g, "quality", gp)?), gp)?; + let mut parts = Vec::new(); + for (a, q) in alleles.iter().zip(quals) { + let mut t = text_of(a, "a KIR allele")?; + if !is_str(Some(q), "") { + t += &format!(" (quality {})", py(q)?); + } + parts.push(t); + } + out.push((py(need(g, "gene", gp)?)?, parts.join(" / "))); + } + let not_called = need(kir, "not_called", p)?; + if truthy(not_called) { + let names: Result, Malformed> = items(Some(not_called), p)? + .iter() + .map(|x| text_of(x, "a KIR gene")) + .collect(); + out.push(( + String::new(), + format!( + "No allele called (absent, or too few reads): {}", + names?.join(", ") + ), + )); + } + Ok(out) +} + +/// ` GENE line`, or the line alone without a gene. +fn gene_line(g: &str, line: &str) -> String { + if g.is_empty() { + format!(" {line}") + } else { + format!(" {} {line}", pycompat::fmt::ljust(g, 8)) + } +} + +fn database(s: &Summary) -> Result { + or_text(Some(s.need2("databases", "hla_database")?), "unknown") +} + +/// The text report's block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "hla", Some("HLA Typing (T1K)"))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if head.ran { + let sec = s.section("hla"); + for l in items(Some(sec.need("loci")?), "sections.hla.data.loci")? { + w.push(format!( + " {} {}", + left( + need(l, "gene", "sections.hla.data.loci[]")?, + 8, + "an HLA gene" + )?, + hla_text(l)? + )); + } + let derived = sec.get("derived")?; + w.extend( + hla_derived_lines(derived)? + .into_iter() + .map(|l| format!(" {l}")), + ); + if derived.is_some_and(truthy) { + w.push(format!(" {HLA_DERIVED_NOTE}")); + } + w.push(format!(" HLA database: {}", database(s)?)); + w.push(format!(" {HLA_NOTE}")); + if let Some(ot) = sec.get("optitype")?.filter(|v| **v != Value::Null) { + w.push(" OptiType check (HLA_QC=true, two fields):".to_owned()); + for (g, line) in hla_optitype_lines(ot)? { + w.push(gene_line(&g, &line)); + } + } + if let Some(kir) = sec.get("kir")?.filter(|v| **v != Value::Null) { + w.push(" KIR genes (T1K, KIR=true):".to_owned()); + for (g, line) in kir_lines(kir)? { + w.push(gene_line(&g, &line)); + } + w.push(format!(" {KIR_NOTE}")); + } + w.push(String::new()); + } + Ok(w) +} + +fn stat(label: &str, value: String) -> Line { + Line::Stat(label.to_owned(), Inline::Text(value)) +} + +/// The HTML report's card. +pub fn html(s: &Summary) -> Result { + let sec = s.section("hla"); + let mut body = Vec::new(); + let mut caveats = Vec::new(); + for l in items(sec.get("loci")?, "sections.hla.data.loci")? { + body.push(stat( + &py(need(l, "gene", "sections.hla.data.loci[]")?)?, + hla_text(l)?, + )); + } + let derived = sec.get("derived")?.filter(|v| truthy(v)); + if derived.is_some() { + body.extend(hla_derived_lines(derived)?.into_iter().map(Line::Note)); + caveats.push(HLA_DERIVED_NOTE); + } + body.push(stat("HLA database", database(s)?)); + caveats.push(HLA_NOTE); + if let Some(ot) = sec.get("optitype")?.filter(|v| **v != Value::Null) { + for (g, line) in hla_optitype_lines(ot)? { + body.push(if g.is_empty() { + stat("OptiType check", line) + } else { + stat(&format!("{g}: T1K vs OptiType"), line) + }); + } + } + if let Some(kir) = sec.get("kir")?.filter(|v| **v != Value::Null) { + for (g, line) in kir_lines(kir)? { + body.push(if g.is_empty() { + Line::Note(line) + } else { + stat(&g, line) + }); + } + caveats.push(KIR_NOTE); + } + let card = html::card( + &sec, + Card { + title: "HLA Typing".into(), + caveats, + body, + ..Card::default() + }, + )?; + Ok(crate::model::HtmlPart { + cards: vec![card], + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/locityper.rs b/rust/crates/og-render/src/sections/locityper.rs index 16844be6e..4db29159e 100644 --- a/rust/crates/og-render/src/sections/locityper.rs +++ b/rust/crates/og-render/src/sections/locityper.rs @@ -5,8 +5,8 @@ use pycompat::json::Value; use crate::model::{ - Card, Cell, Colour, HtmlPart, Inline, Line, Malformed, Section, State, Summary, Table, get, - is_str, need, py, + Card, Cell, Colour, HtmlPart, Inline, Layout, Line, Malformed, Section, State, Summary, Table, + get, is_str, need, py, }; use crate::notes::LOCITYPER_NOTE; use crate::{html, text}; @@ -146,7 +146,7 @@ pub fn html(s: &Summary) -> Result { .to_vec(), rows, viewer: false, - one_line: true, + layout: Layout::OneLine, }), ]; } diff --git a/rust/crates/og-render/src/sections/lpa.rs b/rust/crates/og-render/src/sections/lpa.rs index 087b3e19b..cef768bba 100644 --- a/rust/crates/og-render/src/sections/lpa.rs +++ b/rust/crates/og-render/src/sections/lpa.rs @@ -1 +1,139 @@ -//! The report section `lpa`. Empty until its unit of the 1.0.0 plan lands. +//! Step 46's LPA KIV-2 repeat count with its 95% interval, the KIV-2 +//! 4925G>A null allele, step 42's two LPA risk variants, the flag and the +//! advice: the same lines in both reports. + +use pycompat::json::Value; + +use crate::html; +use crate::model::{ + Card, Colour, HtmlPart, Inline, Line, Malformed, Section, State, Summary, get, is_str, need, + py, py_eq, truthy, +}; +use crate::notes::LPA_NOTE; +use crate::sections::paralogs::shows_body; +use crate::sections::smn1::marker_state; +use crate::text; + +/// What a null allele means for the level, and how to measure it. +pub const LPA_NULL_CAVEAT: &str = "A null allele lowers the Lp(a) made from the copies it sits on; \ + the Lp(a) blood test is the measure."; + +/// The lines of the text section and of the card. +pub fn lines(d: &Section<'_>) -> Result, Malformed> { + let dot = |k: &str| d.get(k).and_then(|v| v.map_or(Ok(".".to_owned()), py)); + let need = |k: &str| d.need(k).and_then(py); + let mut out = vec![if is_str(d.get("status")?, "estimated") { + format!( + "KIV-2 repeats (both chromosomes): {} (95% interval {} to {}), {}", + need("kiv2_copies")?, + need("ci95_low")?, + need("ci95_high")?, + need("class")? + ) + } else { + format!("KIV-2 repeats: {}", dot("status")?) + }]; + out.push(format!(" {}", dot("message")?)); + if let Some(na) = d.get("null_allele")?.filter(|v| truthy(v)) { + let path = "sections.lpa.data.null_allele"; + let rsid = match get(na, "rsid", path)?.filter(|v| truthy(v)) { + Some(v) => py(v)?, + None => ".".to_owned(), + }; + let rsid = rsid.split(' ').next().unwrap_or_default(); + out.push(format!( + "KIV-2 4925G>A null allele ({rsid}): {}", + marker_state(na)? + )); + out.push(format!( + " {}", + get(na, "message", path)?.map_or(Ok(".".to_owned()), py)? + )); + out.push(format!(" {LPA_NULL_CAVEAT}")); + } + let variants: &[Value] = match d.get("variants")? { + None => &[], + Some(Value::Array(a)) => a, + Some(_) => return Err(Malformed("sections.lpa.data.variants is not a list".into())), + }; + for v in variants { + let path = "sections.lpa.data.variants[]"; + let n = need_v(v, "risk_alleles", path)?; + let count = if matches!(n, Value::Null) { + "not read".to_owned() + } else { + let s = if py_eq(n, &Value::Int(1)) { "" } else { "s" }; + format!("{} risk allele{s}", py(n)?) + }; + out.push(format!( + "{}: {} ({count})", + py(need_v(v, "rsid", path)?)?, + py(need_v(v, "genotype", path)?)? + )); + } + if !d.get("variants")?.is_some_and(truthy) { + out.push( + "rs10455872 and rs3798220: not read (step 42, the traits panel, did not run)" + .to_owned(), + ); + } + out.push(dot("flag_text")?); + out.push(dot("advice")?); + Ok(out) +} + +fn need_v<'a>(v: &'a Value, k: &str, path: &str) -> Result<&'a Value, Malformed> { + need(v, k, path) +} + +/// The text report's block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head( + s, + "lpa", + Some("Lp(a): LPA KIV-2 Repeat Count and Risk Variants"), + )? + else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if head.ran { + w.extend(lines(&s.section("lpa"))?.iter().map(|l| format!(" {l}"))); + w.push(format!(" {LPA_NOTE}")); + w.push(String::new()); + } + Ok(w) +} + +/// The HTML report's card, when step 46 is in the summary. +pub fn html(s: &Summary) -> Result { + let d = s.section("lpa"); + if d.state == State::Missing { + return Ok(HtmlPart::default()); + } + let mut body = Vec::new(); + let mut caveats = Vec::new(); + if shows_body(&d) { + if is_str(d.get("flag")?, "high") { + body.push(Line::Stat( + "Genetic sign of high Lp(a)".to_owned(), + Inline::Badge("Yes".to_owned(), Colour::Yellow), + )); + } + body.extend(lines(&d)?.into_iter().map(Line::Para)); + caveats.push(LPA_NOTE); + } + let card = html::card( + &d, + Card { + title: "Lp(a): LPA KIV-2 Repeat Count".to_owned(), + caveats, + body, + ..Card::default() + }, + )?; + Ok(HtmlPart { + cards: vec![card], + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/mei.rs b/rust/crates/og-render/src/sections/mei.rs index 6389f0403..32a44fd1d 100644 --- a/rust/crates/og-render/src/sections/mei.rs +++ b/rust/crates/og-render/src/sections/mei.rs @@ -4,8 +4,8 @@ use pycompat::json::Value; use crate::model::{ - Card, Cell, HtmlPart, Inline, Line, Malformed, Section, State, Summary, Table, is_str, need, - py, truthy, + Card, Cell, HtmlPart, Inline, Layout, Line, Malformed, Section, State, Summary, Table, is_str, + need, py, truthy, }; use crate::notes::MEI_NOTE; use crate::{html, text}; @@ -130,7 +130,7 @@ pub fn html(s: &Summary) -> Result { .to_vec(), rows, viewer: false, - one_line: true, + layout: Layout::OneLine, }), ]; } diff --git a/rust/crates/og-render/src/sections/mito.rs b/rust/crates/og-render/src/sections/mito.rs index e6f32d585..daf57928b 100644 --- a/rust/crates/og-render/src/sections/mito.rs +++ b/rust/crates/og-render/src/sections/mito.rs @@ -1 +1,479 @@ -//! The report section `mito`. Empty until its unit of the 1.0.0 plan lands. +//! Mitochondrial variants (step 20): the Mutect2 chrM calls with the D-loop +//! poly-C tracts, ClinVar and gnomAD on each call, the confirmed disease +//! variants of assets/mito_confirmed.tsv, eKLIPse's large deletions and the +//! mtDNA copies per cell. + +use pycompat::json::Value; + +use crate::html; +use crate::model::{ + Card, Cell, Colour, HtmlPart, Inline, Layout, Line, Malformed, Section, Summary, Table, get, + is_str, need, num, py, truthy, +}; +use crate::notes::{MITO_COPIES_NOTE, MITO_DELETION_NOTE, MITO_NOTE, MITO_POLYC_NOTE}; +use crate::sections::pgx::{items, or_none, or_text, text_of}; +use crate::text; + +/// `"."` for a value that is missing, empty or `.`, else its text. +fn val(x: Option<&Value>) -> Result { + match x { + None | Some(Value::Null) => Ok(".".into()), + Some(Value::Str(s)) if s.is_empty() || s == "." => Ok(".".into()), + Some(v) => py(v), + } +} + +/// One annotated chrM call as a line. +pub fn mito_variant_text(v: &Value) -> Result { + let p = "sections.mito.data.annotation.variants[]"; + let g = |k: &str| get(v, k, p); + let cv = val(g("clinvar_significance")?)?; + let cv = if cv == "." { + "not in ClinVar".to_owned() + } else { + format!( + "ClinVar {cv} ({}; {})", + val(g("clinvar_review")?)?, + val(g("clinvar_condition")?)? + ) + }; + let gn = if val(g("gnomad_af_hom")?)? == "." { + "not in gnomAD".to_owned() + } else { + format!( + "gnomAD AF hom {}, het {}", + py(need(v, "gnomad_af_hom", p)?)?, + val(g("gnomad_af_het")?)? + ) + }; + let polyc = if g("polyc")?.is_some_and(truthy) { + " poly-C tract" + } else { + "" + }; + Ok(format!( + "chrM:{} {}>{} AF {} {cv} {gn}{polyc}", + py(need(v, "pos", p)?)?, + py(need(v, "ref", p)?)?, + py(need(v, "alt", p)?)?, + val(g("af")?)? + )) +} + +/// The annotation block, `{}` when the summary has none. +fn annotation<'a>(d: &Section<'a>) -> Result, Malformed> { + Ok(d.get("annotation")?.filter(|v| truthy(v))) +} + +/// Step 20's lines under the counts, the same in both reports. +pub fn mito_lines(d: &Section<'_>) -> Result, Malformed> { + let mut lines = Vec::new(); + let pc = items( + d.get("polyc_heteroplasmic")?.filter(|v| truthy(v)), + "sections.mito.data.polyc_heteroplasmic", + )?; + if d.get("polyc_heteroplasmic")?.is_some() { + let tracts: Result, Malformed> = items( + d.get("polyc_tracts")?.filter(|v| truthy(v)), + "sections.mito.data.polyc_tracts", + )? + .iter() + .map(|t| text_of(t, "a poly-C tract")) + .collect(); + lines.push(format!( + "Length heteroplasmy in the D-loop poly-C tracts ({}; common, usually technical): {}", + tracts?.join(", "), + pc.len() + )); + for x in pc { + lines.push(format!(" {}", polyc_text(x)?)); + } + } + if let Some(a) = annotation(d)? { + let ap = "sections.mito.data.annotation"; + if let Some(src) = get(a, "source", ap)?.filter(|v| truthy(v)) { + lines.push(format!( + "Annotation ({}): ClinVar {}; gnomAD {}", + py(src)?, + or_text(get(a, "clinvar", ap)?, "unknown")?, + or_text(get(a, "gnomad", ap)?, "unknown")? + )); + lines.push(format!( + "ClinVar pathogenic or likely pathogenic: {}", + get(a, "clinvar_plp", ap)?.map_or(Ok("0".into()), py)? + )); + let vs = get(a, "variants", ap)?.filter(|v| truthy(v)); + if vs.is_some() { + lines.push( + "Heteroplasmic, or in ClinVar with a class other than benign:".to_owned(), + ); + for v in items(vs, ap)? { + lines.push(format!(" {}", mito_variant_text(v)?)); + } + } + } else if let Some(n) = get(a, "note", ap)?.filter(|v| truthy(v)) { + lines.push(format!("ClinVar and gnomAD: not annotated ({})", py(n)?)); + } + } + lines.extend(mito_extended_lines(d)?); + Ok(lines) +} + +/// One poly-C length heteroplasmy: `variant AF af`. +fn polyc_text(x: &Value) -> Result { + let p = "sections.mito.data.polyc_heteroplasmic[]"; + Ok(format!( + "{} AF {}", + py(need(x, "variant", p)?)?, + py(need(x, "af", p)?)? + )) +} + +/// One confirmed disease variant. +pub fn mito_confirmed_text(v: &Value) -> Result { + let p = "sections.mito.data.confirmed[]"; + let pmids = text_of(need(v, "pmids", p)?, "a confirmed variant's PubMed ids")?; + Ok(format!( + "{} ({}) AF {} filter {} {}; ClinVar {}; PMID {}. {}", + py(need(v, "hgvs", p)?)?, + py(need(v, "gene", p)?)?, + or_text(get(v, "af", p)?, ".")?, + or_text(get(v, "filter", p)?, ".")?, + py(need(v, "disease", p)?)?, + py(need(v, "clinvar_id", p)?)?, + pmids.replace(';', ", "), + py(need(v, "note", p)?)? + )) +} + +/// `x + d` as Python adds a number to an int or a float. +fn plus(x: &Value, d: i128) -> Result { + match x { + Value::Int(i) => Ok((i + d).to_string()), + Value::Bool(b) => Ok((i128::from(*b) + d).to_string()), + Value::Float(f) => py(&Value::Float(f + d as f64)), + _ => Err(Malformed("a deletion breakpoint is not a number".into())), + } +} + +/// One large deletion: its span (across the origin when the 3' breakpoint +/// comes first), its level, and the confirmed deletion it matches. +pub fn mito_deletion_text(r: &Value) -> Result { + let p = "sections.mito.data.deletions.listed[]"; + let (b5, b3) = (need(r, "bkp5", p)?, need(r, "bkp3", p)?); + let (n5, n3) = match (num(Some(b5)), num(Some(b3))) { + (Some(a), Some(b)) => (a, b), + _ => return Err(Malformed("a deletion breakpoint is not a number".into())), + }; + let span = if n3 > n5 { + format!( + "m.{}_{}del ({} bp)", + plus(b5, 1)?, + plus(b3, -1)?, + py(need(r, "length", p)?)? + ) + } else { + format!( + "m.{} to m.{} across the origin ({} bp)", + plus(b5, 1)?, + plus(b3, -1)?, + or_none(get(r, "length", p)?)? + ) + }; + let mut out = format!("{span} level {}%", py(need(r, "level_pct", p)?)?); + if let Some(k) = get(r, "known", p)?.filter(|v| truthy(v)) { + let kp = "sections.mito.data.deletions.listed[].known"; + let pmids = text_of(need(k, "pmids", kp)?, "a deletion's PubMed ids")?; + out += &format!( + " {}; ClinVar {} ({}); PMID {}", + py(need(k, "disease", kp)?)?, + py(need(k, "clinvar_id", kp)?)?, + py(need(k, "hgvs", kp)?)?, + pmids.replace(';', ", ") + ); + } + Ok(out) +} + +/// The copy-number line: the copies per cell, or why there are none. +pub fn mito_copies_text(c: &Value) -> Result { + let p = "sections.mito.data.copies"; + if is_str(get(c, "status", p)?, "called") { + return Ok(format!( + "mtDNA copy number: {} per cell (chrM depth {} times the autosomal depth)", + py(need(c, "mtdna_copies", p)?)?, + py(need(c, "depth_ratio", p)?)? + )); + } + let reason = get(c, "reason", p)?; + let why = if is_str(reason, "low_autosomal_depth") { + "too few autosomal reads: a sliced BAM, a panel or a shallow genome".to_owned() + } else if is_str(reason, "no_chrM_reads") { + "no chrM reads in the BAM".to_owned() + } else { + or_text(reason, "unknown")? + }; + Ok(format!("mtDNA copy number: not assessed ({why})")) +} + +/// The listed deletions and how many of them the confirmed list names. +fn listed(dl: &Value) -> Result<(&[Value], usize), Malformed> { + let rows = items( + Some(need(dl, "listed", "sections.mito.data.deletions")?), + "sections.mito.data.deletions.listed", + )?; + let mut named = 0; + for r in rows { + if get(r, "known", "sections.mito.data.deletions.listed[]")?.is_some_and(truthy) { + named += 1; + } + } + Ok((rows, named)) +} + +/// Step 20's confirmed variants, large deletions and copy number, the same +/// in both reports. +pub fn mito_extended_lines(d: &Section<'_>) -> Result, Malformed> { + let mut lines = Vec::new(); + if let Some(conf) = d.get("confirmed")? { + let conf = items(Some(conf), "sections.mito.data.confirmed")?; + lines.push(format!( + "Confirmed disease variants (assets/mito_confirmed.tsv): {}", + conf.len() + )); + for v in conf { + lines.push(format!(" {}", mito_confirmed_text(v)?)); + } + } + if let Some(n) = d.get("confirmed_note")?.filter(|v| truthy(v)) { + lines.push(format!( + "Confirmed disease variants: not checked ({})", + py(n)? + )); + } + if let Some(dl) = d.get("deletions")?.filter(|v| truthy(v)) { + let (rows, named) = listed(dl)?; + let dp = "sections.mito.data.deletions"; + lines.push(format!( + "Large deletions (eKLIPse): {}, of them {named} named in assets/mito_confirmed.tsv; listed: named \ + ones and any at {}% or more", + py(need(dl, "total", dp)?)?, + py(need(dl, "list_floor_pct", dp)?)? + )); + for r in rows { + lines.push(format!(" {}", mito_deletion_text(r)?)); + } + lines.push(MITO_DELETION_NOTE.to_owned()); + } else if let Some(n) = d.get("deletions_note")?.filter(|v| truthy(v)) { + lines.push(format!("Large deletions: not looked for ({})", py(n)?)); + } + if let Some(c) = d.get("copies")?.filter(|v| truthy(v)) { + lines.push(mito_copies_text(c)?); + if is_str(get(c, "status", "sections.mito.data.copies")?, "called") { + lines.push(MITO_COPIES_NOTE.to_owned()); + } + } else if let Some(n) = d.get("copies_note")?.filter(|v| truthy(v)) { + lines.push(format!("mtDNA copy number: not counted ({})", py(n)?)); + } + Ok(lines) +} + +/// The text report's block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "mito", Some("Mitochondrial Variants (Mutect2)"))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if head.ran { + let d = s.section("mito"); + w.push(format!(" PASS variants: {}", py(d.need("pass")?)?)); + w.push(format!( + " Heteroplasmic (AF 0.05 to 0.95, outside the D-loop poly-C tracts): {}", + py(d.need("heteroplasmic")?)? + )); + w.extend(mito_lines(&d)?.into_iter().map(|l| format!(" {l}"))); + if d.get("polyc_heteroplasmic")?.is_some_and(truthy) { + w.push(format!(" {MITO_POLYC_NOTE}")); + } + w.push(format!(" {MITO_NOTE}")); + w.push(String::new()); + } + Ok(w) +} + +fn stat(label: &str, value: Inline) -> Line { + Line::Stat(label.to_owned(), value) +} + +fn count(n: String, any: bool) -> Inline { + Inline::Badge(n, if any { Colour::Yellow } else { Colour::Gray }) +} + +/// The HTML report's card: full width when it holds the table of annotated +/// calls. +pub fn html(s: &Summary) -> Result { + let mi = s.section("mito"); + let empty = Value::Object(Default::default()); + let ma = annotation(&mi)?.unwrap_or(&empty); + let ap = "sections.mito.data.annotation"; + let pc = items( + mi.get("polyc_heteroplasmic")?.filter(|v| truthy(v)), + "sections.mito.data.polyc_heteroplasmic", + )?; + let mut caveats = Vec::new(); + let mut body = vec![ + stat( + "chrM variants (PASS)", + Inline::Text(or_none(mi.get("pass")?)?), + ), + stat( + "Heteroplasmic (AF 0.05 to 0.95, outside the D-loop poly-C tracts)", + Inline::Text(or_none(mi.get("heteroplasmic")?)?), + ), + stat( + "Length heteroplasmy in the D-loop poly-C tracts", + Inline::Text(pc.len().to_string()), + ), + ]; + for x in pc { + body.push(Line::Note(polyc_text(x)?)); + } + if !pc.is_empty() { + caveats.push(MITO_POLYC_NOTE); + } + let variants = get(ma, "variants", ap)?.filter(|v| truthy(v)); + if get(ma, "source", ap)?.is_some_and(truthy) { + let plp = get(ma, "clinvar_plp", ap)?; + body.push(stat( + "ClinVar pathogenic or likely pathogenic", + count(plp.map_or(Ok("0".into()), py)?, plp.is_some_and(truthy)), + )); + if let Some(vs) = variants { + let vp = "sections.mito.data.annotation.variants[]"; + let mut rows = Vec::new(); + for v in items(Some(vs), ap)? { + let g = |k: &str| -> Result { + Ok(Cell::Text(or_none(get(v, k, vp)?)?)) + }; + let pos = need(v, "pos", vp)?; + rows.push(vec![ + Cell::Text(format!( + "chrM:{} {}>{}", + py(pos)?, + py(need(v, "ref", vp)?)?, + py(need(v, "alt", vp)?)? + )), + g("af")?, + g("clinvar_significance")?, + g("clinvar_review")?, + g("clinvar_condition")?, + g("gnomad_af_hom")?, + g("gnomad_af_het")?, + Cell::Text(if get(v, "polyc", vp)?.is_some_and(truthy) { + "yes".into() + } else { + String::new() + }), + Cell::Viewer(if truthy(pos) { + Some(format!("chrM:{}", py(pos)?)) + } else { + None + }), + ]); + } + body.push(Line::Table(Table { + head: [ + "Variant", + "AF", + "ClinVar", + "Review", + "Condition", + "gnomAD AF hom", + "gnomAD AF het", + "Poly-C tract", + ] + .map(str::to_owned) + .to_vec(), + rows, + viewer: true, + layout: Layout::OneLine, + })); + body.push(Line::ViewerNote); + } + body.push(Line::Note(format!( + "ClinVar {}; gnomAD {}", + or_text(get(ma, "clinvar", ap)?, "unknown")?, + or_text(get(ma, "gnomad", ap)?, "unknown")? + ))); + } else if let Some(n) = get(ma, "note", ap)?.filter(|v| truthy(v)) { + body.push(Line::Note(format!( + "ClinVar and gnomAD: not annotated ({})", + py(n)? + ))); + } + if let Some(conf) = mi.get("confirmed")? { + let conf = items(Some(conf), "sections.mito.data.confirmed")?; + body.push(stat( + "Confirmed disease variants", + count(conf.len().to_string(), !conf.is_empty()), + )); + for v in conf { + body.push(Line::Note(mito_confirmed_text(v)?)); + } + } + if let Some(dl) = mi.get("deletions")?.filter(|v| truthy(v)) { + let (rows, named) = listed(dl)?; + body.push(stat( + "Large deletions (eKLIPse)", + Inline::Text(py(need(dl, "total", "sections.mito.data.deletions")?)?), + )); + body.push(stat( + "Named in the confirmed list", + count(named.to_string(), named > 0), + )); + for r in rows { + body.push(Line::Note(mito_deletion_text(r)?)); + } + caveats.push(MITO_DELETION_NOTE); + } else if let Some(n) = mi.get("deletions_note")?.filter(|v| truthy(v)) { + body.push(Line::Note(format!( + "Large deletions: not looked for ({})", + py(n)? + ))); + } + if let Some(mc) = mi.get("copies")?.filter(|v| truthy(v)) { + let called = is_str(get(mc, "status", "sections.mito.data.copies")?, "called"); + body.push(stat( + "mtDNA copies per cell", + Inline::Text(if called { + py(need(mc, "mtdna_copies", "sections.mito.data.copies")?)? + } else { + "not assessed".into() + }), + )); + body.push(Line::Note(mito_copies_text(mc)?)); + if called { + caveats.push(MITO_COPIES_NOTE); + } + } else if let Some(n) = mi.get("copies_note")?.filter(|v| truthy(v)) { + body.push(Line::Note(format!( + "mtDNA copy number: not counted ({})", + py(n)? + ))); + } + caveats.push(MITO_NOTE); + let card = html::card( + &mi, + Card { + title: "Mitochondrial Analysis".into(), + full: variants.is_some(), + caveats, + body, + ..Card::default() + }, + )?; + Ok(HtmlPart { + cards: vec![card], + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/mocha.rs b/rust/crates/og-render/src/sections/mocha.rs index 7a5f9d069..6382f7853 100644 --- a/rust/crates/og-render/src/sections/mocha.rs +++ b/rust/crates/og-render/src/sections/mocha.rs @@ -4,8 +4,8 @@ use pycompat::json::Value; use crate::model::{ - Card, Cell, Colour, HtmlPart, Inline, Line, Malformed, Section, State, Summary, Table, is_str, - need, py, + Card, Cell, Colour, HtmlPart, Inline, Layout, Line, Malformed, Section, State, Summary, Table, + is_str, need, py, }; use crate::notes::MCA_NOTE; use crate::{html, text}; @@ -141,20 +141,22 @@ pub fn html(s: &Summary) -> Result { "Cell-fraction floor".to_owned(), Inline::Text(or(&d, "floor", ".")?), ), - if rows.is_empty() { - Line::Note("No mosaic alteration reported.".to_owned()) - } else { - Line::Table(Table { - head: ["Type", "Arm", "Region", "Mb", "Cell fraction"] - .map(str::to_owned) - .to_vec(), - rows, - viewer: true, - one_line: true, - }) - }, - Line::Note(lines(&d)?.swap_remove(0)), ]; + if rows.is_empty() { + card.body + .push(Line::Note("No mosaic alteration reported.".to_owned())); + } else { + card.body.push(Line::Table(Table { + head: ["Type", "Arm", "Region", "Mb", "Cell fraction"] + .map(str::to_owned) + .to_vec(), + rows, + viewer: true, + layout: Layout::OneLine, + })); + card.body.push(Line::ViewerNote); + } + card.body.push(Line::Note(lines(&d)?.swap_remove(0))); } } Ok(HtmlPart { diff --git a/rust/crates/og-render/src/sections/paralogs.rs b/rust/crates/og-render/src/sections/paralogs.rs index fffdfd985..d4465c31c 100644 --- a/rust/crates/og-render/src/sections/paralogs.rs +++ b/rust/crates/og-render/src/sections/paralogs.rs @@ -1,2 +1,321 @@ -//! The report section `paralogs`. Empty until its unit of the 1.0.0 plan -//! lands. +//! The section "Paralog genes and SMN1": Paraphase (step 35b, PacBio HiFi) +//! beside Parascopy and the depth counts of step 35 (short reads), and a +//! "not checked" card when neither left a result, so a report without them +//! never reads as normal. + +use pycompat::json::Value; + +use crate::html; +use crate::model::{ + Card, Cell, Colour, HtmlPart, Inline, Layout, Line, Malformed, Section, State, Summary, Table, + get, is_str, need, py, truthy, +}; +use crate::notes::{PARALOG_NOTE, PARASCOPY_NOTE}; +use crate::sections::{parascopy, smn1::smn1_text}; +use crate::text; + +const PATH: &str = "sections.paralogs.data"; + +/// The genes Paraphase called (`d.get("genes", [])`). +fn genes<'a>(d: &Section<'a>) -> Result<&'a [Value], Malformed> { + match d.get("genes")? { + None => Ok(&[]), + Some(Value::Array(a)) => Ok(a), + Some(_) => Err(Malformed(format!("{PATH}.genes is not a list"))), + } +} + +/// `sum(d.get("variants", {}).values())`, as Python prints the sum. +fn variants(d: &Section<'_>) -> Result { + let Some(v) = d.get("variants")? else { + return Ok("0".to_owned()); + }; + let Value::Object(o) = v else { + return Err(Malformed(format!("{PATH}.variants is not an object"))); + }; + let (mut whole, mut float, mut is_float) = (0_i128, 0.0_f64, false); + for x in o.values() { + match x { + Value::Int(i) => { + whole += *i; + float += *i as f64; + } + Value::Bool(b) => { + whole += i128::from(*b); + float += f64::from(u8::from(*b)); + } + Value::Float(f) => { + is_float = true; + float += f; + } + _ => { + return Err(Malformed(format!( + "{PATH}.variants holds a value that is not a number" + ))); + } + } + } + Ok(if is_float { + pycompat::fmt::repr(float) + } else { + whole.to_string() + }) +} + +/// The text report's lines of Paraphase's result. +fn paraphase_lines(d: &Section<'_>) -> Result, Malformed> { + let path = format!("{PATH}.genes[]"); + let mut out = Vec::new(); + for g in genes(d)? { + let f = |k: &str| need(g, k, &path).and_then(py); + let detail = need(g, "detail", &path)?; + let tail = if is_str(Some(detail), "") || is_str(Some(detail), ".") { + String::new() + } else { + format!("; {}", py(detail)?) + }; + out.push(format!( + "{}: {} ({}){tail}", + f("gene")?, + f("copy_number")?, + f("status")? + )); + } + if let Some(smn) = smn1(d)? { + out.push(format!("SMN1 by both tools: {smn}")); + } + if let Some(n) = d.get("other_regions")?.filter(|v| truthy(v)) { + out.push(format!( + "Other Paraphase regions: {} ({} with a call)", + py(n)?, + py(d.need("other_regions_called")?)? + )); + } + out.push(format!("Variants on phased haplotypes: {}", variants(d)?)); + Ok(out) +} + +/// `smn1_text(d.get("smn1") or {})`. +fn smn1(d: &Section<'_>) -> Result, Malformed> { + match d.get("smn1")?.filter(|v| truthy(v)) { + Some(m) => smn1_text(m), + None => Ok(None), + } +} + +/// A key's state as the summary writes it. +fn raw_state(s: &Summary, key: &str) -> String { + let sec = s.section(key); + match sec.state { + State::Ok => "ok".to_owned(), + State::Stale => "stale".to_owned(), + State::Failed => "failed".to_owned(), + State::Unreadable => "unreadable".to_owned(), + State::Missing => "missing".to_owned(), + State::Other => s + .get("sections") + .and_then(|x| get(x, key, "sections").ok().flatten()) + .and_then(|x| get(x, "state", "sections").ok().flatten()) + .and_then(|v| py(v).ok()) + .unwrap_or_default(), + } +} + +/// `Paralog copy number: not checked (...)` when neither step 35 nor step +/// 35b left a result. +pub fn unchecked(s: &Summary) -> Option { + let states = [ + ("parascopy", "35", s.section("parascopy").state), + ("paralogs", "35b", s.section("paralogs").state), + ]; + if states.iter().any(|(_, _, st)| st.ran()) { + return None; + } + let why = if states.iter().all(|(_, _, st)| *st == State::Missing) { + "neither step 35 nor step 35b ran".to_owned() + } else { + states + .iter() + .map(|(k, step, st)| { + let word = match st { + State::Missing => "not run".to_owned(), + State::Failed => "failed".to_owned(), + State::Unreadable => "unreadable".to_owned(), + _ => raw_state(s, k), + }; + format!("step {step} {word}") + }) + .collect::>() + .join(", ") + }; + Some(format!("Paralog copy number: not checked ({why})")) +} + +/// The text report's block: Paraphase, Parascopy, then the "not checked" +/// block. +pub fn text(s: &Summary) -> Result, Malformed> { + let mut w = Vec::new(); + if let Some(head) = text::head( + s, + "paralogs", + Some("Paralog Genes (Paraphase, PacBio HiFi)"), + )? { + w.extend(head.lines); + if head.ran { + let d = s.section("paralogs"); + w.extend(paraphase_lines(&d)?.iter().map(|l| format!(" {l}"))); + w.push(format!(" {PARALOG_NOTE}")); + w.push(String::new()); + } + } + if let Some(head) = text::head( + s, + "parascopy", + Some("SMN1/SMN2 Copy Number (Parascopy, short reads)"), + )? { + w.extend(head.lines); + if head.ran { + let d = s.section("parascopy"); + w.extend(parascopy::text_lines(&d)?.iter().map(|l| format!(" {l}"))); + w.push(format!(" {PARASCOPY_NOTE}")); + w.push(String::new()); + } + } + if let Some(why) = unchecked(s) { + w.push("## Paralog Copy Number".to_owned()); + w.push("---".to_owned()); + w.push(format!(" {why}")); + w.push(String::new()); + } + Ok(w) +} + +/// The Status badge of a section that ran: Complete, or Stale for a result +/// from an earlier run. +pub fn done_badge(sec: &Section<'_>) -> Inline { + if sec.state == State::Stale { + Inline::Badge("Stale".to_owned(), Colour::Yellow) + } else { + Inline::Badge("Complete".to_owned(), Colour::Green) + } +} + +/// Whether a card shows its own lines: the step ran, or its state is one no +/// collector writes. +pub fn shows_body(sec: &Section<'_>) -> bool { + !matches!( + sec.state, + State::Missing | State::Failed | State::Unreadable + ) +} + +/// Paraphase's card. +fn paraphase_card(d: &Section<'_>) -> Result { + let mut body = vec![Line::Stat("Status".to_owned(), done_badge(d))]; + let mut caveats = Vec::new(); + if shows_body(d) { + let path = format!("{PATH}.genes[]"); + let mut rows = Vec::new(); + for g in genes(d)? { + let f = |k: &str| need(g, k, &path).and_then(py).map(Cell::Text); + rows.push(vec![ + f("gene")?, + f("copy_number")?, + f("status")?, + f("detail")?, + ]); + } + if !rows.is_empty() { + body.push(Line::Table(Table { + head: ["Gene", "Copy number", "Call", "Detail"] + .map(str::to_owned) + .to_vec(), + rows, + layout: Layout::OneLine, + ..Table::default() + })); + } + if let Some(smn) = smn1(d)? { + body.push(Line::Stat( + "SMN1 by both tools".to_owned(), + Inline::Text(smn), + )); + } + if let Some(n) = d.get("other_regions")?.filter(|v| truthy(v)) { + body.push(Line::Stat( + "Other Paraphase regions".to_owned(), + Inline::Text(format!( + "{} ({} with a call)", + py(n)?, + py(d.need("other_regions_called")?)? + )), + )); + } + body.push(Line::Stat( + "Variants on phased haplotypes".to_owned(), + Inline::Text(variants(d)?), + )); + caveats.push(PARALOG_NOTE); + } + html::card( + d, + Card { + title: "Paralog Genes (Paraphase)".to_owned(), + full: true, + caveats, + body, + ..Card::default() + }, + ) +} + +/// Parascopy's card. +fn parascopy_card(d: &Section<'_>) -> Result { + let mut body = vec![Line::Stat("Status".to_owned(), done_badge(d))]; + let mut caveats = Vec::new(); + if shows_body(d) { + let (lines, loci_caveats) = parascopy::html_lines(d)?; + body.extend(lines); + caveats.extend(loci_caveats); + caveats.push(PARASCOPY_NOTE); + } + html::card( + d, + Card { + title: "SMN1/SMN2 Copy Number (Parascopy)".to_owned(), + caveats, + body, + ..Card::default() + }, + ) +} + +/// The HTML report's cards. +pub fn html(s: &Summary) -> Result { + let mut cards = Vec::new(); + let pg = s.section("paralogs"); + if pg.state != State::Missing { + cards.push(paraphase_card(&pg)?); + } + let pc = s.section("parascopy"); + if pc.state != State::Missing { + cards.push(parascopy_card(&pc)?); + } + if let Some(why) = unchecked(s) { + cards.push(Card { + title: "Paralog Copy Number".to_owned(), + body: vec![ + Line::Stat( + "Status".to_owned(), + Inline::Badge("Not checked".to_owned(), Colour::Gray), + ), + Line::Note(why), + ], + ..Card::default() + }); + } + Ok(HtmlPart { + cards, + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/parascopy.rs b/rust/crates/og-render/src/sections/parascopy.rs index e688a9b01..737690a18 100644 --- a/rust/crates/og-render/src/sections/parascopy.rs +++ b/rust/crates/og-render/src/sections/parascopy.rs @@ -1,2 +1,148 @@ -//! The report section `parascopy`. Empty until its unit of the 1.0.0 plan -//! lands. +//! Step 35 (Parascopy, short reads): SMN1 and SMN2 with the SMN1 marker, +//! the other loci of its catalog, and the depth counts it writes beside them +//! (HBA, CYP21A2, the dosage loci, the copy-number intervals). + +use pycompat::json::Value; + +use crate::model::{Cell, Layout, Line, Malformed, Section, Table, get, is_str, need, py, truthy}; +use crate::notes::PARASCOPY_LOCI_NOTE; +use crate::sections::smn1::{SMN1_MARKER_CAVEAT, marker_state}; +use crate::sections::{cn_intervals, cyp21a2::cyp21a2_line, dosage::dosage_lines, hba::hba_line}; + +const PATH: &str = "sections.parascopy.data"; + +/// `(d.get(key) or {})`. +fn dict_or_empty<'a>(d: &Section<'a>, key: &str, empty: &'a Value) -> Result<&'a Value, Malformed> { + Ok(d.get(key)?.filter(|v| truthy(v)).unwrap_or(empty)) +} + +/// `d.get(key, ".")` as text. +fn dot(v: &Value, key: &str, path: &str) -> Result { + get(v, key, path)?.map_or(Ok(".".to_owned()), py) +} + +/// Step 35's lines, the same in both reports (the HTML card splits each at +/// its first `": "` into a label and a value). +pub fn lines(d: &Section<'_>) -> Result, Malformed> { + let empty = Value::Object(Default::default()); + let smn1 = dict_or_empty(d, "smn1", &empty)?; + let smn2 = dict_or_empty(d, "smn2", &empty)?; + let mut out = vec![format!( + "SMN1 copies: {}", + dot(smn1, "text", &format!("{PATH}.smn1"))? + )]; + if let Some(mk) = d.get("smn1_marker")?.filter(|v| truthy(v)) { + let path = format!("{PATH}.smn1_marker"); + let rsid = match get(mk, "rsid", &path)?.filter(|v| truthy(v)) { + Some(v) => py(v)?, + None => ".".to_owned(), + }; + out.push(format!( + "SMN1 g.27134T>G silent-carrier marker ({rsid}): {}", + marker_state(mk)? + )); + out.push(format!( + "SMN1 marker reading: {}", + dot(mk, "message", &path)? + )); + out.push(format!("SMN1 marker caveat: {SMN1_MARKER_CAVEAT}")); + } + out.push(format!( + "SMN2 copies: {}", + dot(smn2, "text", &format!("{PATH}.smn2"))? + )); + let f = |k: &str| d.get(k).and_then(|v| v.map_or(Ok(".".to_owned()), py)); + out.push(format!( + "SMN1 and SMN2 together (agCN) at exon 7: {} (quality {}, filter {})", + f("agcn")?, + f("agcn_qual")?, + f("agcn_filter")? + )); + out.push(format!("Reading: {}", f("reading")?)); + if let Some(h) = d.get("hba")?.filter(|v| truthy(v)) { + out.push(hba_line(h)?); + } + out.push(cyp21a2_line(d.get("cyp21a2")?)?); + out.extend(dosage_lines(d.get("dosage")?)?); + Ok(out) +} + +/// The other loci of the catalog (`d.get("loci") or []`). +fn loci<'a>(d: &Section<'a>) -> Result<&'a [Value], Malformed> { + match d.get("loci")?.filter(|v| truthy(v)) { + None => Ok(&[]), + Some(Value::Array(a)) => Ok(a), + Some(_) => Err(Malformed(format!("{PATH}.loci is not a list"))), + } +} + +/// `genes` with a space after each comma. +fn genes(x: &Value) -> Result { + Ok(py(need(x, "genes", &format!("{PATH}.loci[]"))?)?.replace(',', ", ")) +} + +/// One line per catalog locus other than SMN1, the same in both reports. +pub fn locus_lines(d: &Section<'_>) -> Result, Malformed> { + let path = format!("{PATH}.loci[]"); + let mut out = Vec::new(); + for x in loci(d)? { + let mut line = format!("{} (agCN): {}", genes(x)?, py(need(x, "text", &path)?)?); + if is_str(get(x, "status", &path)?, "called") + && let Some(r) = get(x, "reading", &path)?.filter(|r| !is_str(Some(r), ".")) + { + line += &format!(". Reading: {}", py(r)?); + } + out.push(line); + } + Ok(out) +} + +/// The text report's block, after its head. +pub fn text_lines(d: &Section<'_>) -> Result, Malformed> { + let mut out = lines(d)?; + out.extend(locus_lines(d)?); + out.extend(cn_intervals::lines(d.get("cn_intervals")?)?); + Ok(out) +} + +/// The HTML card's lines after its status: the step's lines as label and +/// value, the table of the other loci with its caveat, the intervals. +pub fn html_lines(d: &Section<'_>) -> Result<(Vec, Vec<&'static str>), Malformed> { + let mut body = Vec::new(); + let mut caveats = Vec::new(); + for line in lines(d)? { + let (label, value) = line.split_once(": ").unwrap_or((&line, "")); + body.push(Line::Stat( + label.to_owned(), + crate::model::Inline::Text(value.to_owned()), + )); + } + let loci = loci(d)?; + if !loci.is_empty() { + let path = format!("{PATH}.loci[]"); + let mut rows = Vec::new(); + for x in loci { + let reading = if is_str(get(x, "status", &path)?, "called") { + py(need(x, "reading", &path)?)? + } else { + ".".to_owned() + }; + rows.push(vec![ + Cell::Text(genes(x)?), + Cell::Text(py(need(x, "text", &path)?)?), + Cell::Text(reading), + ]); + } + body.push(Line::Table(Table { + head: ["Genes", "Copies together (agCN)", "Reading"] + .map(str::to_owned) + .to_vec(), + rows, + layout: Layout::OneLine, + ..Table::default() + })); + caveats.push(PARASCOPY_LOCI_NOTE); + } + body.extend(cn_intervals::html(d.get("cn_intervals")?)?); + Ok((body, caveats)) +} diff --git a/rust/crates/og-render/src/sections/pgx.rs b/rust/crates/og-render/src/sections/pgx.rs index 0591b1e47..8686a7268 100644 --- a/rust/crates/og-render/src/sections/pgx.rs +++ b/rust/crates/og-render/src/sections/pgx.rs @@ -1 +1,708 @@ -//! The report section `pgx`. Empty until its unit of the 1.0.0 plan lands. +//! The medicines sections: PharmCAT (step 07), the CPIC recommendations +//! (step 27), and pypgx (step 32) with Cyrius (step 21) and step 36's +//! CYP2D6 verdict, which both reports show in one section (`cyp2d6`). + +use pycompat::json::Value; +use pycompat::text::{lower, slice, upper}; + +use crate::model::{ + Card, Cell, Colour, HtmlPart, Inline, Layout, Line, Malformed, Section, State, Summary, Table, + contains, get, is_str, need, py, truthy, +}; +use crate::notes::{CPIC_NOTE, PGX_CALLERS_NOTE, PHARMCAT_NOTE, stale_note}; +use crate::{html, text}; + +/// `f"{v: Result { + let s = match v { + Value::Str(_) | Value::Int(_) | Value::Float(_) => py(v)?, + Value::Bool(b) => u8::from(*b).to_string(), + _ => return Err(Malformed(format!("{path} cannot be formatted"))), + }; + Ok(pycompat::fmt::ljust(&s, width)) +} + +/// `v` as a text, or the path of a value that is not one (where Python +/// joins or concatenates strings). +pub fn text_of(v: &Value, path: &str) -> Result { + match v { + Value::Str(s) => Ok(s.clone()), + _ => Err(Malformed(format!("{path} is not a text"))), + } +} + +/// `v or default`, as text. +pub fn or_text(v: Option<&Value>, default: &str) -> Result { + match v.filter(|v| truthy(v)) { + Some(v) => py(v), + None => Ok(default.to_owned()), + } +} + +/// `d.get(key)` as Python's f-string prints it: `None` for a missing key. +pub fn or_none(v: Option<&Value>) -> Result { + v.map_or(Ok("None".to_owned()), py) +} + +/// The items of a list value (`[]` for a missing key or `null`). +pub fn items<'a>(v: Option<&'a Value>, path: &str) -> Result<&'a [Value], Malformed> { + match v { + None | Some(Value::Null) => Ok(&[]), + Some(Value::Array(a)) => Ok(a), + Some(_) => Err(Malformed(format!("{path} is not a list"))), + } +} + +/// `str.capitalize()`: the first character upper case, the rest lower. +fn capitalize(s: &str) -> String { + let mut c = s.chars(); + match c.next() { + Some(f) => upper(&f.to_string()) + &lower(c.as_str()), + None => String::new(), + } +} + +/// `os.path.join(os.path.dirname(path), name)`, on `/` paths. +fn beside(path: &str, name: &str) -> String { + let head = path.rfind('/').map_or("", |i| &path[..=i]); + let dir = if !head.is_empty() && head.chars().any(|c| c != '/') { + head.trim_end_matches('/') + } else { + head + }; + if name.starts_with('/') { + name.to_owned() + } else if dir.is_empty() || dir.ends_with('/') { + format!("{dir}{name}") + } else { + format!("{dir}/{name}") + } +} + +fn stat(label: &str, value: Inline) -> Line { + Line::Stat(label.to_owned(), value) +} + +fn sample(s: &Summary) -> Result { + text_of(s.need("sample")?, "summary.sample") +} + +/// `_cpic_recommendations.txt` beside the CPIC report. +fn cpic_file(s: &Summary, sec: &Section<'_>) -> Result { + let src = sec + .source + .ok_or_else(|| Malformed("sections.cpic has no source".into()))?; + let src = text_of(src, "sections.cpic.source")?; + Ok(beside( + &src, + &format!("{}_cpic_recommendations.txt", sample(s)?), + )) +} + +/// The step ran and the summary names its file. +fn ran_with_source(sec: &Section<'_>) -> bool { + sec.state.ran() && sec.source.is_some_and(truthy) +} + +// --- PharmCAT ---------------------------------------------------------------------- + +/// The text report's PharmCAT block. +pub fn pharmcat_text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "pharmcat", Some("Pharmacogenomics (PharmCAT)"))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if head.ran { + let sec = s.section("pharmcat"); + w.push(format!( + " Report: {} (PharmCAT {})", + or_none(sec.source)?, + or_text(sec.get("version")?, "version unknown")? + )); + if sec.get("html_report")?.is_some_and(truthy) { + w.push(" HTML report beside it (open in a browser for full details)".to_owned()); + } + w.push(format!(" {PHARMCAT_NOTE}")); + w.push(String::new()); + } + Ok(w) +} + +/// The HTML report's PharmCAT card, and the link to PharmCAT's own HTML +/// report beside it. +pub fn pharmcat_html(s: &Summary) -> Result { + let sec = s.section("pharmcat"); + // Never green: a report that was written is not a finding. + let mut body = vec![stat( + "PharmCAT report", + if sec.state.ran() { + Inline::Badge("Complete".into(), Colour::State) + } else { + Inline::Badge("Not run".into(), Colour::Gray) + }, + )]; + if let Some(v) = sec.get("version")?.filter(|v| truthy(v)) { + body.push(stat( + "PharmCAT version", + Inline::Text(text_of(v, "sections.pharmcat.data.version")?), + )); + } + let pg = s.section("pypgx"); + if pg.state.ran() && contains(pg.data, "comparison") { + let c = need( + pg.need("comparison")?, + "conflicts", + "sections.pypgx.data.comparison", + )?; + body.push(stat("PharmCAT vs pypgx conflicts", Inline::Text(py(c)?))); + } + let card = html::card( + &sec, + Card { + title: "Pharmacogenomics".into(), + caveats: vec![PHARMCAT_NOTE], + body, + ..Card::default() + }, + )?; + let mut files = Vec::new(); + if ran_with_source(&sec) && sec.get("html_report")?.is_some_and(truthy) { + let src = text_of(sec.source.expect("checked"), "sections.pharmcat.source")?; + files.push(( + format!("{}.html", slice(&src, None, Some(-5))), + "PharmCAT HTML report".to_owned(), + )); + } + Ok(HtmlPart { + cards: vec![card], + files, + }) +} + +// --- CPIC -------------------------------------------------------------------------- + +/// The genes CPIC lists as non-normal or with more than one possible +/// result: (gene, diplotype, phenotype). +fn unusual<'a>(sec: &Section<'a>) -> Result, Malformed> { + let mut out = Vec::new(); + for (i, g) in items(Some(sec.need("genes")?), "sections.cpic.data.genes")? + .iter() + .enumerate() + { + let p = format!("sections.cpic.data.genes[{i}]"); + let st = need(g, "status", &p)?; + if is_str(Some(st), "non-normal") || is_str(Some(st), "ambiguous") { + out.push([ + need(g, "gene", &p)?, + need(g, "diplotype", &p)?, + need(g, "phenotype", &p)?, + ]); + } + } + Ok(out) +} + +/// The text report's CPIC block. +pub fn cpic_text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "cpic", Some("CPIC Drug-Gene Recommendations"))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if head.ran { + let sec = s.section("cpic"); + if truthy(sec.need("parse_failed")?) { + w.push(" The PharmCAT report could not be parsed: see the CPIC report.".to_owned()); + } + let or0 = |k: &str| -> Result { sec.get(k)?.map_or(Ok("0".into()), py) }; + w.push(format!( + " Genes with a non-normal phenotype: {} More than one possible result: {} Not called: {} \ + Called without a function phenotype (no drug guidance): {}", + py(sec.need("non_normal")?)?, + or0("ambiguous")?, + py(sec.need("not_called")?)?, + or0("unclassified")? + )); + for [g, d, p] in unusual(&sec)? { + w.push(format!( + " {} {} {}", + left(g, 10, "a CPIC gene")?, + left(d, 28, "a CPIC diplotype")?, + py(p)? + )); + } + for warn in items(Some(sec.need("warnings")?), "sections.cpic.data.warnings")? { + w.push(format!(" WARNING: {}", py(warn)?)); + } + w.push(format!(" Drugs per gene: {}", cpic_file(s, &sec)?)); + w.push(format!(" {CPIC_NOTE}")); + w.push(String::new()); + } + Ok(w) +} + +/// The HTML report's CPIC cards: the counts, then the genes with a +/// non-normal or unresolved phenotype; and the link to the recommendations +/// file. +pub fn cpic_html(s: &Summary) -> Result { + let sec = s.section("cpic"); + let smp = sample(s)?; + let mut body = vec![stat( + "Status", + if sec.state == State::Stale { + Inline::Badge("Stale".into(), Colour::Yellow) + } else { + Inline::Badge("Complete".into(), Colour::Green) + }, + )]; + if sec.get("parse_failed")?.is_some_and(truthy) { + body.push(stat( + "PharmCAT report", + Inline::Badge("could not be parsed".into(), Colour::Red), + )); + } else { + for (label, k) in [ + ("Genes with a non-normal phenotype", "non_normal"), + ("Genes with more than one possible result", "ambiguous"), + ("Genes not called", "not_called"), + ( + "Genes called without a function phenotype (no drug guidance)", + "unclassified", + ), + ] { + let v = sec.get(k)?.map_or(Ok("0".into()), py)?; + body.push(stat(label, Inline::Text(v))); + } + } + body.push(stat( + "Tip", + Inline::Quiet(format!( + "Recommendations per gene: cpic/{smp}_cpic_recommendations.txt" + )), + )); + let mut cards = vec![html::card( + &sec, + Card { + title: "CPIC Drug Recommendations".into(), + caveats: vec![CPIC_NOTE], + body, + ..Card::default() + }, + )?]; + if sec.state.ran() + && (truthy(sec.need("non_normal")?) || sec.get("ambiguous")?.is_some_and(truthy)) + { + let mut rows = Vec::new(); + for [g, d, p] in unusual(&sec)? { + rows.push(vec![ + Cell::Text(text_of(g, "a CPIC gene")?), + Cell::Text(text_of(d, "a CPIC diplotype")?), + Cell::Text(text_of(p, "a CPIC phenotype")?), + ]); + } + let mut body = vec![Line::Table(Table { + head: vec!["Gene".into(), "Diplotype".into(), "Phenotype".into()], + rows, + viewer: false, + layout: Layout::RowLines, + })]; + for x in items(Some(sec.need("warnings")?), "sections.cpic.data.warnings")? { + body.push(Line::Warn(format!( + "WARNING: {}", + text_of(x, "a CPIC warning")? + ))); + } + body.push(Line::Para(format!( + "Drugs for each gene: cpic/{smp}_cpic_recommendations.txt" + ))); + cards.push(html::card( + &sec, + Card { + title: "Genes With a Non-Normal or Unresolved Phenotype (CPIC)".into(), + full: true, + body, + ..Card::default() + }, + )?); + } + let mut files = Vec::new(); + if ran_with_source(&sec) { + files.push(( + cpic_file(s, &sec)?, + "CPIC recommendations per gene (text)".to_owned(), + )); + } + Ok(HtmlPart { cards, files }) +} + +// --- pypgx, Cyrius and CYP2D6 ------------------------------------------------------ + +/// Why step 32 could not call a gene on this reference. +fn not_callable_why(g: &Value) -> &'static str { + if is_str(Some(g), "GSTT1") { + "GRCh38 has it only on the ALT contig chr22_KI270879v1_alt, which a reference without ALT \ + contigs lacks" + } else { + "see docs/32-pypgx.md" + } +} + +/// Step 32's lines below its counts, the same in both reports: the genes +/// this reference cannot hold, the calls resting on a low-GQ variant, and +/// the CYP2D6 depth check with its MAPQ shares. +pub fn pypgx_lines(d: &Section<'_>) -> Result, Malformed> { + let mut lines = Vec::new(); + for g in items(d.get("not_callable")?, "sections.pypgx.data.not_callable")? { + lines.push(format!( + "Not assessable on this reference: {} ({})", + py(g)?, + not_callable_why(g) + )); + } + for (i, x) in items(d.get("low_gq")?, "sections.pypgx.data.low_gq")? + .iter() + .enumerate() + { + let p = format!("sections.pypgx.data.low_gq[{i}]"); + lines.push(format!( + "Low-confidence: {} {} rests on {}: check the reads", + py(need(x, "gene", &p)?)?, + py(need(x, "diplotype", &p)?)?, + or_text(Some(need(x, "weakest", &p)?), "a low-GQ variant")? + )); + } + if let Some(dc) = d.get("depth_check")?.filter(|v| truthy(v)) { + let p = "sections.pypgx.data.depth_check"; + let share = |k: &str| -> Result { + get(dc, k, p)?.map_or(Ok("not measured".into()), py) + }; + lines.push(format!( + "CYP2D6 depth check: {}; share of the depth at MAPQ >= 1 / >= 20: Cyrius's region with REP6 {} / {}, \ + gene body without REP6 {} / {}", + or_text(get(dc, "status", p)?, "unreadable")?, + share("gene_mapq1_share")?, + share("gene_mapq20_share")?, + share("body_mapq1_share")?, + share("body_mapq20_share")? + )); + } + Ok(lines) +} + +/// (gene, copy number) rows, the same in both reports: pypgx's call from +/// read depth for each BAM-mode gene of assets/pgx_genes.tsv. +pub fn pgx_copy_number(s: &Summary) -> Result, Malformed> { + let pg = s.section("pypgx"); + match pg.get("cnv_call")?.filter(|v| truthy(v)) { + None => Ok(Vec::new()), + Some(Value::Object(o)) => o + .iter() + .map(|(g, v)| Ok((g.clone(), format!("{} (pypgx, read depth)", py(v)?)))) + .collect(), + Some(_) => Err(Malformed( + "sections.pypgx.data.cnv_call is not an object".into(), + )), + } +} + +/// The summary's CYP2D6 block (calls, agreement, step 36's consensus, the +/// probable call), or `None` for a summary without one (a part collected +/// with `--only`), which shows no CYP2D6 across the callers. +fn block(s: &Summary) -> Result, Malformed> { + match s.get("cyp2d6") { + None | Some(Value::Null) => Ok(None), + Some(c @ Value::Object(_)) => Ok(Some(c)), + Some(_) => Err(Malformed("summary.cyp2d6 is not an object".into())), + } +} + +/// The callers' CYP2D6 calls, in the summary's order. +fn calls(c: &Value) -> Result, Malformed> { + match need(c, "calls", "summary.cyp2d6")? { + Value::Object(o) => Ok(o.iter().collect()), + _ => Err(Malformed("summary.cyp2d6.calls is not an object".into())), + } +} + +/// The column label of a caller: PharmCAT calls no CYP2D6 from a VCF, it +/// shows the call step 36 passed to it. +fn caller_label(k: &str) -> &str { + if k == "PharmCAT" { + "PharmCAT (step 36's call)" + } else { + k + } +} + +/// Step 36's CYP2D6 verdict as one line, or `""` without one. +pub fn cyp2d6_verdict(c: &Value) -> Result { + let Some(v) = get(c, "consensus", "summary.cyp2d6")?.filter(|v| truthy(v)) else { + return Ok(String::new()); + }; + let p = "summary.cyp2d6.consensus"; + let passed = get(v, "passed_to_pharmcat", p)?.is_some_and(truthy); + let where_ = if passed { + "passed to PharmCAT" + } else { + "not passed to PharmCAT" + }; + Ok(format!( + "{}, {where_} ({})", + or_text(get(v, "result", p)?, "?")?, + or_text(get(v, "reason", p)?, "no reason given")? + )) +} + +/// The probable CYP2D6 call as one line. +fn probable_text(p: &Value) -> Result { + let path = "summary.cyp2d6.probable"; + let pheno = match get(p, "phenotype", path)?.filter(|v| truthy(v)) { + Some(x) => format!(", {}", py(x)?), + None => String::new(), + }; + Ok(format!( + "{}{pheno}; not passed to PharmCAT, so no drug guidance rests on it", + or_none(get(p, "diplotype", path)?)? + )) +} + +/// The probable call's label and line, when the summary has one. +fn probable(c: &Value) -> Result, Malformed> { + let Some(p) = get(c, "probable", "summary.cyp2d6")?.filter(|v| truthy(v)) else { + return Ok(None); + }; + let label = text_of( + need(p, "label", "summary.cyp2d6.probable")?, + "summary.cyp2d6.probable.label", + )?; + Ok(Some((capitalize(&label), probable_text(p)?))) +} + +/// The text report's block of the section: pypgx, its copy numbers, then +/// CYP2D6 across the callers. +pub fn cyp2d6_text(s: &Summary) -> Result, Malformed> { + let mut w = Vec::new(); + if let Some(head) = text::head(s, "pypgx", Some("pypgx Pharmacogenomics"))? { + w.extend(head.lines); + if head.ran { + let d = s.section("pypgx"); + w.push(format!( + " Genes called: {}/{}", + py(d.need("genes_called")?)?, + py(d.need("genes_total")?)? + )); + w.push(format!(" CYP2D6 diplotype: {}", py(d.need("cyp2d6")?)?)); + if contains(d.data, "comparison") { + let c = d.need("comparison")?; + let p = "sections.pypgx.data.comparison"; + w.push(format!( + " PharmCAT vs pypgx conflicts: {}", + py(need(c, "conflicts", p)?)? + )); + let one = need(c, "one_tool_only", p)?; + if truthy(one) { + w.push(format!( + " Scope differences (only one tool called): {}", + py(one)? + )); + } + } + w.extend(pypgx_lines(&d)?.into_iter().map(|l| format!(" {l}"))); + w.push(String::new()); + } + } + let cn = pgx_copy_number(s)?; + if !cn.is_empty() { + w.push("## PGx copy number".to_owned()); + w.push("---".to_owned()); + for (g, v) in cn { + w.push(format!(" {} {v}", pycompat::fmt::ljust(&g, 8))); + } + w.push(String::new()); + } + let Some(c) = block(s)? else { + return Ok(w); + }; + let calls = calls(c)?; + let verdict = cyp2d6_verdict(c)?; + if calls.iter().any(|(_, v)| truthy(v)) || !verdict.is_empty() { + w.push("## CYP2D6 across callers".to_owned()); + w.push("---".to_owned()); + // The consensus change is filed under pypgx. + if let Some(line) = text::new_since(s, &["pypgx", "cyrius"])? { + w.push(line); + } + for (k, v) in &calls { + w.push(format!( + " {} {}", + pycompat::fmt::ljust(caller_label(k), 26), + or_text(Some(v), "not run")? + )); + } + let agree = need(c, "agree", "summary.cyp2d6")?; + w.push(format!( + " pypgx and Cyrius agree: {}", + if truthy(agree) { + "yes" + } else if *agree == Value::Bool(false) { + "NO, review before acting on CYP2D6" + } else { + "fewer than two usable calls" + } + )); + if !verdict.is_empty() { + w.push(format!(" Step 36: {verdict}")); + } + if let Some((label, line)) = probable(c)? { + w.push(format!(" {label}: {line}")); + } + w.push(format!(" {PGX_CALLERS_NOTE}")); + w.push(String::new()); + } + Ok(w) +} + +/// The HTML report's cards of the section: pypgx (when its step ran or +/// failed), its copy numbers, and CYP2D6 across the callers, which is +/// always there. +pub fn cyp2d6_html(s: &Summary) -> Result { + let mut cards = Vec::new(); + let pd = s.section("pypgx"); + if pd.state != State::Missing { + let mut body = vec![ + stat( + "Genes called", + Inline::Text(format!( + "{}/{}", + or_none(pd.get("genes_called")?)?, + or_none(pd.get("genes_total")?)? + )), + ), + stat( + "CYP2D6 diplotype", + Inline::Text(or_text(pd.get("cyp2d6")?, "not called")?), + ), + ]; + body.extend(pypgx_lines(&pd)?.into_iter().map(Line::Para)); + cards.push(html::card( + &pd, + Card { + title: "pypgx Pharmacogenomics".into(), + body, + ..Card::default() + }, + )?); + } + let cn = pgx_copy_number(s)?; + if !cn.is_empty() { + cards.push(Card { + title: "PGx Copy Number".into(), + body: cn + .into_iter() + .map(|(g, v)| stat(&g, Inline::Text(v))) + .collect(), + ..Card::default() + }); + } + let Some(c) = block(s)? else { + return Ok(HtmlPart { + cards, + files: Vec::new(), + }); + }; + let mut body: Vec = Vec::new(); + for (k, v) in calls(c)? { + body.push(stat( + caller_label(k), + Inline::Text(or_text(Some(v), "not run")?), + )); + } + let agree = need(c, "agree", "summary.cyp2d6")?; + body.push(stat( + "pypgx and Cyrius agree", + if truthy(agree) { + Inline::Badge("yes".into(), Colour::Green) + } else if *agree == Value::Bool(false) { + Inline::Badge("no".into(), Colour::Red) + } else { + Inline::Badge("fewer than two usable calls".into(), Colour::Gray) + }, + )); + if let Some(cons) = get(c, "consensus", "summary.cyp2d6")?.filter(|v| truthy(v)) { + let p = "summary.cyp2d6.consensus"; + let passed = get(cons, "passed_to_pharmcat", p)?.is_some_and(truthy); + body.push(stat( + "Step 36", + Inline::Badge( + or_text(get(cons, "result", p)?, "?")?, + if passed { + Colour::State + } else { + Colour::Yellow + }, + ), + )); + body.push(Line::Note(format!( + "{}{}", + if passed { + "Passed to PharmCAT: " + } else { + "Not passed to PharmCAT: " + }, + or_text(get(cons, "reason", p)?, "no reason given")? + ))); + } + if let Some((label, line)) = probable(c)? { + body.push(stat(&label, Inline::Text(line))); + } + let mut stale = Vec::new(); + for k in ["cpic", "pypgx", "cyrius"] { + let sec = s.section(k); + if matches!(sec.state, State::Stale | State::Failed) { + stale.push(format!( + "{}: {}", + text_of(sec.title, &format!("sections.{k}.title"))?, + stale_note(&sec)? + )); + } + } + cards.push(Card { + title: "CYP2D6 Across Callers".into(), + caveats: vec![PGX_CALLERS_NOTE], + stale, + body, + ..Card::default() + }); + Ok(HtmlPart { + cards, + files: Vec::new(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn paths_join_as_os_path_does() { + assert_eq!(beside("cpic/S_cpic.json", "S_x.txt"), "cpic/S_x.txt"); + assert_eq!(beside("S_cpic.json", "S_x.txt"), "S_x.txt"); + assert_eq!(beside("/r.json", "x"), "/x"); + assert_eq!(beside("a//b", "x"), "a/x"); + assert_eq!(beside("a/b", "/abs"), "/abs"); + } + + #[test] + fn capitalize_lowers_the_rest() { + assert_eq!(capitalize("probable CYP2D6 call"), "Probable cyp2d6 call"); + assert_eq!(capitalize(""), ""); + } + + #[test] + fn left_pads_as_format_does() { + assert_eq!(left(&Value::Str("ab".into()), 4, "x").unwrap(), "ab "); + assert_eq!(left(&Value::Bool(true), 3, "x").unwrap(), "1 "); + assert!(left(&Value::Null, 3, "x").is_err()); + } +} diff --git a/rust/crates/og-render/src/sections/prs.rs b/rust/crates/og-render/src/sections/prs.rs index 3d832baa8..7ac93adc1 100644 --- a/rust/crates/og-render/src/sections/prs.rs +++ b/rust/crates/og-render/src/sections/prs.rs @@ -1 +1,341 @@ -//! The report section `prs`. Empty until its unit of the 1.0.0 plan lands. +//! Polygenic risk scores (step 25, pgsc_calc): each score with the variants +//! it matched, its percentile among the most similar reference group when +//! the ancestry panel is installed, the effect a published evaluation +//! measured in that group, and the line that says what the numbers can be +//! compared with: the ancestry caveat, the chrX rows left out and the +//! sex-specific scores when no sex was declared. +//! +//! The scores are shown as the summary lists them. + +use pycompat::json::Value; + +use crate::model::{ + Card, Cell, HtmlPart, Layout, Line, Malformed, Summary, Table, get, is_str, need, py, truthy, +}; +use crate::notes::PRS_ANCESTRY_NOTE; +use crate::sections::ancestry::{Align, items, need_align, need_text, py_or}; +use crate::{html, text}; + +const PATH: &str = "sections.prs.data"; +const ROW: &str = "sections.prs.data.scores[]"; + +/// What a score of the other sex says in place of a percentile. +pub const NOT_APPLICABLE: &str = "not applicable to the declared sex"; + +/// Part of the line under the scores, so not a caveat of its own. +pub const EFFECT_TEXT: &str = "Effect: the odds ratio (OR) or hazard ratio (HR) a published \ + evaluation of the score measured in people of the same reference group, in that \ + evaluation's unit (its PGS Catalog PPM page says which). It describes the score in that \ + group, not your own risk, and is shown only for a group the score was evaluated in."; + +/// What a score, matched or total cell says when the summary has none: og +/// summary collect writes null for a short row of pgsc_calc's table. +pub const NOT_AVAILABLE: &str = "not available"; + +/// `str(r[key])`, or [`NOT_AVAILABLE`] for null. +fn count(r: &Value, key: &str) -> Result { + match need(r, key, ROW)? { + Value::Null => Ok(NOT_AVAILABLE.to_owned()), + v => py(v), + } +} + +fn field<'a>(r: &'a Value, key: &str) -> Result, Malformed> { + get(r, key, ROW) +} + +/// `r.get(key)` is true. +fn on<'a>(r: &'a Value, key: &str) -> Result, Malformed> { + Ok(field(r, key)?.filter(|v| truthy(v))) +} + +fn other_sex(r: &Value) -> Result { + Ok(is_str(field(r, "sex_check")?, NOT_APPLICABLE)) +} + +/// The scores of the summary. +fn scores(d: &Value) -> Result, Malformed> { + items(need(d, "scores", PATH)?, &format!("{PATH}.scores")) +} + +/// The percentile cell: the percentile with its group, the note for a score +/// of the other sex, or "" when there is none. +pub fn percentile(r: &Value) -> Result { + if other_sex(r)? { + let sex = py_or(on(r, "applies_to")?, "sex")?; + return Ok(format!("{NOT_APPLICABLE} ({sex}-specific score)")); + } + match on(r, "percentile")? { + Some(p) => Ok(format!( + "{} ({})", + py(p)?, + py_or(on(r, "group")?, "group unknown")? + )), + None => Ok(String::new()), + } +} + +/// `", ".join(v)`: the items of a list of texts, or a text's characters. +fn join(v: &Value, at: &str) -> Result { + let parts = items(v, at)? + .into_iter() + .map(|x| match x { + Value::Str(s) => Ok(s), + _ => Err(Malformed(format!("{at} holds an item that is not a text"))), + }) + .collect::, _>>()?; + Ok(parts.join(", ")) +} + +/// The effect cell: the odds or hazard ratio the PGS Catalog records for +/// the score in the sample's group, a line saying the score has none for +/// that group, or "" when the sample has no group, the score does not apply +/// to the declared sex, or the table names no evaluations. The bool says +/// whether the cell is the ratio itself. +pub fn effect(r: &Value) -> Result<(String, bool), Malformed> { + let evaluated = match field(r, "evaluated_in")? { + None | Some(Value::Null) => return Ok((String::new(), false)), + Some(v) => v, + }; + if on(r, "percentile")?.is_some() && on(r, "group")?.is_some() && !other_sex(r)? { + if let Some(x) = on(r, "effect")? { + return Ok((py(x)?, true)); + } + let group = py(need(r, "group", ROW)?)?; + let place = if truthy(evaluated) { + format!( + "; evaluated in {}", + join(evaluated, &format!("{ROW}.evaluated_in"))? + ) + } else { + String::new() + }; + return Ok((format!("no published OR or HR for {group}{place}"), false)); + } + Ok((String::new(), false)) +} + +/// The chrX rows left out of the scores because no sex was given. +fn chrx_note(rs: &[Value]) -> Result { + let mut left = Vec::new(); + for r in rs { + let Some(c) = on(r, "chrx")? else { continue }; + let Value::Str(c) = c else { + return Err(Malformed(format!("{ROW}.chrx is not a text"))); + }; + if c.ends_with("left out") { + let first = pycompat::text::split(c.as_str(), None, None) + .first() + .copied() + .unwrap_or_default() + .to_owned(); + left.push(format!("{} ({first})", need_text(r, "pgs_id", ROW)?)); + } + } + if left.is_empty() { + return Ok(String::new()); + } + Ok(format!( + " Rows on chrX were left out, because the sample's sex was not given: {}. Give the sex \ + to score them (docs/25-prs.md).", + left.join(", ") + )) +} + +/// The sex-specific scores shown for any sex, when no sex was declared. +fn sex_note(rs: &[Value]) -> Result { + let mut named = Vec::new(); + for r in rs { + if is_str(field(r, "sex_check")?, "no sex declared") + && let Some(a) = on(r, "applies_to")? + { + named.push(format!("{} ({})", need_text(r, "pgs_id", ROW)?, py(a)?)); + } + } + if named.is_empty() { + return Ok(String::new()); + } + Ok(format!( + " No sex was declared, so these sex-specific scores are shown for any sex: {}. Give the \ + sex to mark a score of the other sex not applicable (docs/25-prs.md).", + named.join(", ") + )) +} + +/// Whether the summary's table names evaluations (its Evaluated_In column). +fn names_evaluations(rs: &[Value]) -> Result { + for r in rs { + if !matches!(field(r, "evaluated_in")?, None | Some(Value::Null)) { + return Ok(true); + } + } + Ok(false) +} + +/// The line under the scores, in both reports: what the numbers can be +/// compared with. +pub fn note(d: &Value) -> Result { + let rs = scores(d)?; + let tail = chrx_note(&rs)? + &sex_note(&rs)?; + if !get(d, "adjusted", PATH)?.is_some_and(truthy) { + return Ok(format!( + "Raw score only: no ancestry reference panel was installed, so there is no percentile \ + and the sum cannot be compared with anyone (scripts/setup.sh --ancestry-panel adds \ + it). {PRS_ANCESTRY_NOTE} See docs/25-prs.md.{tail}" + )); + } + let empty = Value::Object(indexmap::IndexMap::new()); + let anc = get(d, "ancestry", PATH)? + .filter(|v| truthy(v)) + .unwrap_or(&empty); + let at = format!("{PATH}.ancestry"); + let mut group = match get(anc, "population", &at)?.filter(|v| truthy(v)) { + Some(p) => py(p)?, + None => { + let mut g = String::new(); + for r in &rs { + if let Some(x) = on(r, "group")? { + g = py(x)?; + break; + } + } + g + } + }; + if group.is_empty() { + group = "most similar".to_owned(); + } + let panel = match get(anc, "panel", &at)?.filter(|v| truthy(v)) { + Some(p) => format!(" of the {} reference panel", py(p)?), + None => " of the reference panel".to_owned(), + }; + let low = if is_str(get(anc, "low_confidence", &at)?, "True") { + " The ancestry match is low-confidence: read the percentile with care." + } else { + "" + }; + let eff = if names_evaluations(&rs)? { + format!(" {EFFECT_TEXT}") + } else { + String::new() + }; + Ok(format!( + "Percentile: where the score falls among the {group} samples{panel}, the group whose \ + genetic ancestry is most similar to this sample's (pgsc_calc).{low} A percentile is not \ + a risk.{eff} {PRS_ANCESTRY_NOTE} See docs/25-prs.md.{tail}" + )) +} + +/// The text report's block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "prs", Some("Polygenic Risk Scores"))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if !head.ran { + return Ok(w); + } + let d = s.section("prs").data; + for r in scores(d)? { + let condition = need_align(&r, "condition", 35, Align::Left, ROW)?; + let n = |k: &str| count(&r, k); + let line = format!( + " {condition} {} ({}/{} variants matched) {}", + n("score")?, + n("matched")?, + n("total")?, + need_text(&r, "pgs_id", ROW)? + ); + let pct = percentile(&r)?; + let pct = if other_sex(&r)? { + format!(" {pct}") + } else if pct.is_empty() { + String::new() + } else { + format!(" percentile {pct}") + }; + let (e, is_ratio) = effect(&r)?; + let eff = if e.is_empty() { + String::new() + } else if is_ratio { + format!(" effect {e}") + } else { + format!(" {e}") + }; + w.push(format!("{line}{pct}{eff}")); + } + w.push(format!(" {}", note(d)?)); + w.push(String::new()); + Ok(w) +} + +/// The HTML report's card, when step 25 has a result: one row per score, +/// with the effect column when the table names evaluations. +pub fn html(s: &Summary) -> Result { + let sec = s.section("prs"); + if !sec.state.ran() { + return Ok(HtmlPart::default()); + } + let d = sec.data; + let rs = scores(d)?; + let adjusted = get(d, "adjusted", PATH)?.is_some_and(truthy); + let show_effect = adjusted && names_evaluations(&rs)?; + let mut rows = Vec::new(); + for x in &rs { + let pct = percentile(x)?; + let mut row = vec![ + Cell::Text(need_text(x, "condition", ROW)?), + Cell::Text(need_text(x, "pgs_id", ROW)?), + Cell::Text(count(x, "score")?), + Cell::Text(format!("{}/{}", count(x, "matched")?, count(x, "total")?)), + Cell::Text(if !pct.is_empty() { + pct + } else if matches!(need(x, "score", ROW)?, Value::Null) { + NOT_AVAILABLE.to_owned() + } else { + "raw score only".to_owned() + }), + ]; + if show_effect { + row.push(Cell::Text(effect(x)?.0)); + } + rows.push(row); + } + let mut head = [ + "Condition", + "PGS", + "Score", + "Variants matched", + "Percentile", + ] + .map(str::to_owned) + .to_vec(); + if show_effect { + head.push("Effect in your group".to_owned()); + } + let card = Card { + title: format!( + "Polygenic Risk Scores{}", + if adjusted { + "" + } else { + " (raw, not percentiles)" + } + ), + full: true, + body: vec![ + Line::Table(Table { + head, + rows, + layout: Layout::RowLines, + ..Table::default() + }), + Line::Para(note(d)?), + ], + ..Card::default() + }; + Ok(HtmlPart { + cards: vec![html::card(&sec, card)?], + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/rfc1.rs b/rust/crates/og-render/src/sections/rfc1.rs index aff217dcf..aceaf96ac 100644 --- a/rust/crates/og-render/src/sections/rfc1.rs +++ b/rust/crates/og-render/src/sections/rfc1.rs @@ -1 +1,19 @@ -//! The report section `rfc1`. Empty until its unit of the 1.0.0 plan lands. +//! Step 9b's motif count of a flagged repeat locus: RFC1's AARRG unit, and +//! any other locus whose motif decides more than its length. + +use pycompat::json::Value; + +use crate::model::{Malformed, get, py}; + +/// ` motif count: : `, as both reports print it. +pub fn motif_text(locus: &str, m: &Value) -> Result { + let at = |k: &str| get(m, k, "the motif count"); + let verdict = at("verdict")?.map_or(Ok(".".to_owned()), py)?; + let note = at("note")?.map_or(Ok(String::new()), py)?; + Ok(format!("{locus} motif count: {verdict}: {note}")) +} + +/// RFC1's motif count. +pub fn rfc1_text(m: &Value) -> Result { + motif_text("RFC1", m) +} diff --git a/rust/crates/og-render/src/sections/smn1.rs b/rust/crates/og-render/src/sections/smn1.rs index eb4742abe..10abfcbdc 100644 --- a/rust/crates/og-render/src/sections/smn1.rs +++ b/rust/crates/og-render/src/sections/smn1.rs @@ -1 +1,63 @@ -//! The report section `smn1`. Empty until its unit of the 1.0.0 plan lands. +//! SMN1 read two ways: the copy-aware marker (g.27134T>G, the silent +//! carrier sign, and LPA's KIV-2 4925G>A uses the same reading), and +//! Paraphase's count beside Parascopy's. + +use pycompat::json::Value; + +use crate::model::{Malformed, get, is_str, need, py, truthy}; + +/// What the SMN1 marker can and cannot say. +pub const SMN1_MARKER_CAVEAT: &str = "The marker raises the chance that two SMN1 copies sit on one \ + chromosome (2+0, a silent carrier); it does not prove 2+0, and a carrier test in a clinical \ + laboratory confirms it."; + +/// A copy-aware marker's state in a few words: present or uncertain with +/// the reads and the copies that carry it, not seen with the reads, too +/// little depth, or not assessed. +pub fn marker_state(m: &Value) -> Result { + let at = |k: &str| get(m, k, "the marker"); + let st = match at("status")?.filter(|v| truthy(v)) { + Some(v) => py(v)?, + None => "not assessed".to_owned(), + }; + if st == "not assessed" { + return Ok(st); + } + let dot = |k: &str| at(k).and_then(|v| v.map_or(Ok(".".to_owned()), py)); + let mut out = format!( + "{st}, {} of {} reads carry it", + dot("alt_reads")?, + dot("depth")? + ); + if let Some(c) = at("copies")? + && !matches!(c, Value::Null) + && !is_str(Some(c), "") + && !is_str(Some(c), ".") + { + out += &format!(", about {} copies", py(c)?); + } + Ok(out) +} + +/// SMN1 from Paraphase beside Parascopy (step 35), with the verdict; `None` +/// when Parascopy has no SMN1 count. +pub fn smn1_text(m: &Value) -> Result, Malformed> { + let at = |k: &str| get(m, k, "sections.paralogs.data.smn1"); + let Some(pc) = at("parascopy")?.filter(|v| !matches!(v, Value::Null)) else { + return Ok(None); + }; + let pp = match at("paraphase")?.filter(|v| !matches!(v, Value::Null)) { + Some(v) => py(v)?, + None => "no call".to_owned(), + }; + let agree = at("agree")?; + let verdict = if is_str(agree, "agree") { + "the two agree" + } else if is_str(agree, "disagree") { + "they DISAGREE: check both before using either" + } else { + "no comparison: one of them has no call" + }; + let pc = py(need(pc, "text", "sections.paralogs.data.smn1.parascopy")?)?; + Ok(Some(format!("Paraphase {pp}, Parascopy {pc}: {verdict}"))) +} diff --git a/rust/crates/og-render/src/sections/str.rs b/rust/crates/og-render/src/sections/str.rs index 047c968c6..e90fd8736 100644 --- a/rust/crates/og-render/src/sections/str.rs +++ b/rust/crates/og-render/src/sections/str.rs @@ -1 +1,475 @@ -//! The report section `str`. Empty until its unit of the 1.0.0 plan lands. +//! Step 09's repeat expansions (ExpansionHunter) with Stranger's status +//! (step 9b): the loci outside the normal range, the loci it does not +//! grade, the readings of the repeats STRchive leaves out, and every locus +//! with its count, status and notes, the key loci first. + +use pycompat::fmt::ljust; +use pycompat::json::Value; + +use crate::html; +use crate::model::{ + Card, Cell, Colour, HtmlPart, Inline, Layout, Line, Malformed, Summary, Table, get, is_str, + need, py, truthy, +}; +use crate::notes::{EH_NOTE, NOT_GRADED_NOTE, RFC1_NOTE}; +use crate::sections::paralogs::{done_badge, shows_body}; +use crate::sections::rfc1::{motif_text, rfc1_text}; +use crate::text; + +const PATH: &str = "sections.expansions.data"; + +/// What the report says of an allele that was not sized from spanning +/// reads. +fn sizing_text(sizing: &str) -> Option<&'static str> { + Some(match sizing { + "flanking_only" => "size not determined (flanking reads only)", + "in_repeat" => { + "longer than a read (in-repeat reads only): shown as at least the interval's lower end" + } + "low_confidence" => { + "low confidence (fewer than 2 spanning reads, or the interval reaches the disease range)" + } + "no_call" => "no call", + _ => return None, + }) +} + +/// `l.get(key)`. +fn at<'a>(l: &'a Value, key: &str) -> Result, Malformed> { + get(l, key, &format!("{PATH}.loci[]")) +} + +/// `(l.get(key) or "")` as text. +fn text_or_empty(l: &Value, key: &str) -> Result { + match at(l, key)?.filter(|v| truthy(v)) { + Some(v) => py(v), + None => Ok(String::new()), + } +} + +/// `d.get(key) or []`. +fn list<'a>(d: &'a Value, key: &str) -> Result<&'a [Value], Malformed> { + match get(d, key, PATH)?.filter(|v| truthy(v)) { + None => Ok(&[]), + Some(Value::Array(a)) => Ok(a), + Some(_) => Err(Malformed(format!("{PATH}.{key} is not a list"))), + } +} + +/// The repeat count of a locus: an in-repeat allele as at least the lower +/// end of its interval, or `not sized` without one. +pub fn count(l: &Value) -> Result { + let counts = py(need(l, "repeat_count", &format!("{PATH}.loci[]"))?)?; + let counts: Vec<&str> = counts.split('/').collect(); + let sizing = text_or_empty(l, "allele_sizing")?; + let sizing: Vec<&str> = sizing.split('/').collect(); + let cis = text_or_empty(l, "ci")?; + let cis: Vec<&str> = cis.split('/').collect(); + let mut out = Vec::new(); + for (i, c) in counts.iter().enumerate() { + let z = sizing.get(i).copied().unwrap_or(""); + let ci = if cis.len() == counts.len() { + cis[i] + } else if cis.len() == 1 { + cis[0] + } else { + "" + }; + let lo = ci.split('-').next().unwrap_or(""); + out.push(if z == "in_repeat" { + if pycompat::text::isdigit(lo) { + format!(">={lo}") + } else { + "not sized".to_owned() + } + } else { + (*c).to_owned() + }); + } + Ok(out.join("/")) +} + +/// Stranger's status, never `normal` for a locus that was not sized. +pub fn status(l: &Value) -> Result { + let st = at(l, "status")?.filter(|v| truthy(v)); + if st.is_none() && at(l, "not_graded")?.is_some_and(truthy) { + return Ok("not graded".to_owned()); + } + if is_str(st, "normal") + && !is_str(at(l, "sizing")?, "sized") + && !is_str(at(l, "sizing")?, "unknown") + { + return Ok("not sized".to_owned()); + } + st.map_or(Ok(String::new()), py) +} + +/// What the report says beside a locus: sizing, the catalog's normal +/// maximum, an intermediate range, no thresholds, the motif count. +pub fn notes(l: &Value) -> Result { + let mut out = Vec::new(); + if let Some(Value::Str(z)) = at(l, "sizing")? + && let Some(t) = sizing_text(z) + { + let ci = at(l, "ci")?; + let shown = match ci { + None | Some(Value::Null) => false, + Some(v) => !is_str(Some(v), "") && !is_str(Some(v), "."), + }; + out.push(if shown { + format!("{t}, interval {}", py(ci.expect("checked above"))?) + } else { + t.to_owned() + }); + } + if let Some(m) = at(l, "normal_max")?.filter(|v| truthy(v)) { + out.push(format!("normal max {}", py(m)?)); + } + if let Some(z) = at(l, "zone")?.filter(|v| truthy(v)) { + let path = format!("{PATH}.loci[]"); + out.push(format!( + "{}: {}; {}", + py(z)?, + py(need(l, "zone_label", &path)?)?, + py(need(l, "zone_citation", &path)?)? + )); + } + if let Some(n) = at(l, "not_graded")?.filter(|v| truthy(v)) { + out.push(format!("no thresholds in the catalog ({})", py(n)?)); + } + if let Some(m) = at(l, "motif")?.filter(|v| truthy(v)) { + let locus = py(need(l, "locus", &format!("{PATH}.loci[]"))?)?; + out.push(motif_text(&locus, m)?); + } + Ok(out.join("; ")) +} + +/// `(locus, count, status, notes)` for every locus, the key loci first; a +/// key locus missing from the output says so. +pub fn rows(d: &Value) -> Result, Malformed> { + let mut rows = Vec::new(); + for l in list(d, "loci")? { + rows.push([ + py(need(l, "locus", &format!("{PATH}.loci[]"))?)?, + count(l)?, + status(l)?, + notes(l)?, + ]); + } + let mut out = Vec::new(); + for k in list(d, "key_loci")? { + let locus = need(k, "locus", &format!("{PATH}.key_loci[]"))?; + let name = py(locus)?; + // `k["locus"] not in have`: compared by value, as Python compares. + let have = list(d, "loci")?.iter().any(|l| { + get(l, "locus", PATH) + .ok() + .flatten() + .is_some_and(|x| crate::model::py_eq(x, locus)) + }); + if !have { + out.push([ + name, + "not in output".to_owned(), + String::new(), + String::new(), + ]); + } + } + out.extend(rows); + Ok(out) +} + +/// The first line of one reading of the repeats STRchive leaves out (CFTR, +/// UGT1A1, 5-HTTLPR). +fn reading_head(r: &Value) -> Result { + let path = format!("{PATH}.readings[]"); + let f = |k: &str| need(r, k, &path).and_then(py); + Ok(format!( + "{} {}: {}", + f("locus")?, + f("repeat")?, + reading_what(r)? + )) +} + +/// A reading's genotype when it was read, else its status in words. +fn reading_what(r: &Value) -> Result { + let path = format!("{PATH}.readings[]"); + let st = need(r, "status", &path)?; + if is_str(Some(st), "read") { + py(need(r, "genotype", &path)?) + } else { + Ok(py(st)?.replace('_', " ")) + } +} + +/// The joined items of a list or the keys of a dict, as `", ".join(x)`. +fn joined(v: &Value, key: &str) -> Result { + match v { + Value::Array(a) => Ok(a.iter().map(py).collect::, _>>()?.join(", ")), + Value::Object(o) => Ok(o.keys().cloned().collect::>().join(", ")), + Value::Str(s) => Ok(s.chars().map(String::from).collect::>().join(", ")), + _ => Err(Malformed(format!("{PATH}.{key} is not a list"))), + } +} + +/// `len(x)` of a list, a dict or a string. +fn len(v: &Value, key: &str) -> Result { + match v { + Value::Array(a) => Ok(a.len()), + Value::Object(o) => Ok(o.len()), + Value::Str(s) => Ok(s.chars().count()), + _ => Err(Malformed(format!("{PATH}.{key} has no length"))), + } +} + +/// The text report's block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "expansions", Some("Repeat Expansions (ExpansionHunter)"))? + else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if !head.ran { + return Ok(w); + } + let sec = s.section("expansions"); + let d = sec.data; + let g = |k: &str| get(d, k, PATH); + w.push(format!(" Loci in output: {}", py(sec.need("records")?)?)); + let flagged = list(d, "flagged")?; + if g("stranger_outdated")?.is_some_and(truthy) { + w.push( + " Stranger's result is older than ExpansionHunter's: rerun step 9b. No locus is \ + marked as normal or expanded." + .to_owned(), + ); + } else if !g("stranger")?.is_some_and(truthy) { + w.push( + " Stranger (step 9b) did not run: no locus is marked as normal or expanded." + .to_owned(), + ); + } else if !flagged.is_empty() { + w.push(format!( + " Outside the normal range (Stranger): {}", + len(sec.need("flagged")?, "flagged")? + )); + let loci = list(d, "loci")?; + for l in flagged { + let path = format!("{PATH}.flagged[]"); + let locus = need(l, "locus", &path)?; + // `by.get(l["locus"], l)`: the full locus row when there is one; + // the last row of a locus listed twice, as a dict keeps it. + let full = loci + .iter() + .rev() + .find(|x| { + get(x, "locus", PATH) + .ok() + .flatten() + .is_some_and(|y| crate::model::py_eq(y, locus)) + }) + .unwrap_or(l); + w.push(format!( + " {} {} {}", + ljust(&py(locus)?, 8), + count(full)?, + py(need(l, "status", &path)?)? + )); + if let Some(m) = get(l, "rfc1_motif", &path)?.filter(|v| truthy(v)) { + w.push(format!(" {}", rfc1_text(m)?)); + } + if let Some(m) = get(l, "motif", &path)?.filter(|v| truthy(v)) { + w.push(format!(" {}", motif_text(&py(locus)?, m)?)); + } + } + } else { + w.push(" Outside the normal range (Stranger): none".to_owned()); + } + if let Some(ng) = g("not_graded")?.filter(|v| truthy(v)) { + let Value::Object(o) = ng else { + return Err(Malformed(format!("{PATH}.not_graded is not an object"))); + }; + let items: Vec = o + .iter() + .map(|(k, v)| py(v).map(|v| format!("{k}: {v}"))) + .collect::>()?; + w.push(format!( + " Not graded (no thresholds in the catalog): {} ({})", + o.len(), + items.join("; ") + )); + w.push(format!(" {NOT_GRADED_NOTE}")); + } + for r in list(d, "readings")? { + let path = format!("{PATH}.readings[]"); + w.push(format!(" {}", reading_head(r)?)); + w.push(format!( + " {} ({})", + py(need(r, "reading", &path)?)?, + py(need(r, "source", &path)?)? + )); + } + if g("stranger")?.is_some_and(truthy) + && let Some(n) = g("no_status")?.filter(|v| truthy(v)) + { + w.push(format!( + " Loci without a Stranger status (not in its catalog): {}", + py(n)? + )); + } + if let Some(u) = g("unsized")?.filter(|v| truthy(v)) { + w.push(format!( + " Not sized from short reads, so not counted as normal: {} ({})", + len(u, "unsized")?, + joined(u, "unsized")? + )); + } + if let Some(i) = g("intermediate")?.filter(|v| truthy(v)) { + w.push(format!( + " In an intermediate range (bin/str_intermediate_ranges.tsv): {}", + joined(i, "intermediate")? + )); + } + let rows = rows(d)?; + if !rows.is_empty() { + w.push(" Every locus (key loci first): repeat count, Stranger status, notes".to_owned()); + for [locus, count, status, notes] in &rows { + let status = if status.is_empty() { "-" } else { status }; + let tail = if notes.is_empty() { + String::new() + } else { + format!(" {notes}") + }; + w.push(format!( + " {} {} {status}{tail}", + ljust(locus, 8), + ljust(count, 10) + )); + } + } + if flagged.iter().any(|l| { + get(l, "rfc1_motif", PATH) + .ok() + .flatten() + .is_some_and(truthy) + }) { + w.push(format!(" {RFC1_NOTE}")); + } + w.push(format!(" {EH_NOTE}")); + w.push(String::new()); + Ok(w) +} + +/// The HTML report's card, shown on every report: "Not run" when step 09 +/// is not in the summary. +pub fn html(s: &Summary) -> Result { + let sec = s.section("expansions"); + let mut body = vec![Line::Stat("Status".to_owned(), done_badge(&sec))]; + let mut caveats = Vec::new(); + if shows_body(&sec) { + let d = sec.data; + let g = |k: &str| get(d, k, PATH); + let label = "Outside the normal range (Stranger)".to_owned(); + let flagged = list(d, "flagged")?; + body.push(if g("stranger_outdated")?.is_some_and(truthy) { + Line::Stat( + label, + Inline::Badge( + "older than ExpansionHunter's result: rerun step 9b".to_owned(), + Colour::Yellow, + ), + ) + } else if !g("stranger")?.is_some_and(truthy) { + Line::Stat(label, Inline::Badge("not run".to_owned(), Colour::Gray)) + } else { + let colour = if flagged.is_empty() { + Colour::Gray + } else { + Colour::Yellow + }; + Line::Stat(label, Inline::Badge(flagged.len().to_string(), colour)) + }); + if let Some(u) = g("unsized")?.filter(|v| truthy(v)) { + body.push(Line::Stat( + "Not sized from short reads (not counted as normal)".to_owned(), + Inline::Badge(len(u, "unsized")?.to_string(), Colour::Yellow), + )); + } + if let Some(i) = g("intermediate")?.filter(|v| truthy(v)) { + body.push(Line::Stat( + "In an intermediate range".to_owned(), + Inline::Badge(joined(i, "intermediate")?, Colour::Yellow), + )); + } + if let Some(n) = g("not_graded")?.filter(|v| truthy(v)) { + body.push(Line::Stat( + "Not graded (no thresholds in the catalog)".to_owned(), + Inline::Badge(joined(n, "not_graded")?, Colour::Gray), + )); + caveats.push(NOT_GRADED_NOTE); + } + for r in list(d, "readings")? { + let path = format!("{PATH}.readings[]"); + let colour = if get(r, "five_t", &path)?.is_some_and(truthy) + || is_str(get(r, "agreement", &path)?, "differs") + { + Colour::Yellow + } else { + Colour::Gray + }; + body.push(Line::Stat( + format!( + "{} {}", + py(need(r, "locus", &path)?)?, + py(need(r, "repeat", &path)?)? + ), + Inline::Badge(reading_what(r)?, colour), + )); + body.push(Line::Note(format!( + "{} ({})", + py(need(r, "reading", &path)?)?, + py(need(r, "source", &path)?)? + ))); + } + for l in flagged { + let path = format!("{PATH}.flagged[]"); + if let Some(m) = get(l, "rfc1_motif", &path)?.filter(|v| truthy(v)) { + body.push(Line::Note(rfc1_text(m)?)); + caveats.push(RFC1_NOTE); + } + if let Some(m) = get(l, "motif", &path)?.filter(|v| truthy(v)) { + body.push(Line::Note(motif_text(&py(need(l, "locus", &path)?)?, m)?)); + } + } + let rows = rows(d)?; + if !rows.is_empty() { + body.push(Line::Table(Table { + head: ["Locus", "Repeat Count", "Stranger", "Notes"] + .map(str::to_owned) + .to_vec(), + rows: rows + .into_iter() + .map(|r| r.map(Cell::Text).to_vec()) + .collect(), + layout: Layout::OneLine, + ..Table::default() + })); + } + caveats.push(EH_NOTE); + } + let card = html::card( + &sec, + Card { + title: "Repeat Expansions".to_owned(), + caveats, + body, + ..Card::default() + }, + )?; + Ok(HtmlPart { + cards: vec![card], + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/traits.rs b/rust/crates/og-render/src/sections/traits.rs index a087281ea..eedce03d1 100644 --- a/rust/crates/og-render/src/sections/traits.rs +++ b/rust/crates/og-render/src/sections/traits.rs @@ -1,2 +1,260 @@ -//! The report section `traits`. Empty until its unit of the 1.0.0 plan -//! lands. +//! Step 42's traits panel: one row per catalog site with its genotype and +//! the cited reading, the genotype rules that read several sites together +//! (APOL1 with its G2, SERPINA1, HFE and the others), and the APOE lines +//! kept apart behind a warning. + +use pycompat::json::Value; + +use crate::html; +use crate::model::{ + Card, Cell, Colour, HtmlPart, Inline, Line, Malformed, Section, Summary, Table, is_str, need, + py, truthy, +}; +use crate::notes::TRAITS_NOTE; +use crate::sections::{apoe, g2}; +use crate::text; + +const ROWS: &str = "sections.traits.data.rows[]"; +const RULES: &str = "sections.traits.data.rules[]"; + +/// A list in the section's data, `[]` when it is absent or empty. +fn list<'a>(d: &Section<'a>, key: &str, need_it: bool) -> Result<&'a [Value], Malformed> { + let v = if need_it { + Some(d.need(key)?) + } else { + d.get(key)?.filter(|v| truthy(v)) + }; + match v { + None => Ok(&[]), + Some(Value::Array(a)) => Ok(a), + Some(_) => Err(Malformed(format!( + "sections.traits.data.{key} is not a list" + ))), + } +} + +/// ` [ancestry]` for a row whose reading depends on ancestry. +fn ancestry(r: &Value, path: &str) -> Result { + Ok(is_str(Some(need(r, "ancestry", path)?), "yes")) +} + +/// The text report's lines (before the closing blank line). +pub fn text_lines(d: &Section<'_>, sample: &str) -> Result, Malformed> { + let n = |k: &str| d.need(k).and_then(py); + let mut out = vec![format!( + " Read: {} No genotype at the position: {} Other allele: {}", + n("called")?, + n("not_covered")?, + n("other_allele")? + )]; + let rows = list(d, "rows", true)?; + let mut missing = Vec::new(); + for r in rows { + let f = |k: &str| need(r, k, ROWS).and_then(py); + if is_str(Some(need(r, "status", ROWS)?), "not_covered") { + missing.push(f("rsid")?); + continue; + } + let mark = if ancestry(r, ROWS)? { + " [ancestry]" + } else { + "" + }; + out.push(format!( + " {} ({} {}): {}{mark}. {} (PMID {})", + f("trait")?, + f("gene")?, + f("rsid")?, + f("genotype")?, + f("reading")?, + f("pmids")? + )); + } + if !missing.is_empty() { + out.push(format!( + " Not read (no genotype in the gVCF): {}", + missing.join(", ") + )); + } + out.push(format!( + " {TRAITS_NOTE} Notes per row: traits/{sample}_traits.tsv" + )); + let rules = list(d, "rules", false)?; + if !rules.is_empty() { + out.push(format!( + " Genotype rules (several sites read together; traits/{sample}_genotype_rules.tsv):" + )); + for r in rules { + let f = |k: &str| need(r, k, RULES).and_then(py); + out.push(format!( + " {} ({} {}): {}. {} (PMID {})", + f("trait")?, + f("gene")?, + f("rsids")?.replace(',', ", "), + g2::rule_result(r)?, + f("reading")?, + f("pmids")? + )); + if let Some(line) = g2::g2_text(r)? { + out.push(line); + } + } + } + out.extend(apoe::text_lines(list(d, "apoe", false)?)?); + Ok(out) +} + +/// The text report's block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "traits", Some("Traits Panel (step 42)"))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if head.ran { + let sample = py(s.need("sample")?)?; + w.extend(text_lines(&s.section("traits"), &sample)?); + w.push(String::new()); + } + Ok(w) +} + +/// The ancestry badge after a genotype or a result, when the row has one. +fn ancestry_badge(r: &Value, path: &str) -> Result, Malformed> { + Ok(if ancestry(r, path)? { + vec![ + Inline::Text(" ".to_owned()), + Inline::Badge("ancestry".to_owned(), Colour::Yellow), + ] + } else { + Vec::new() + }) +} + +/// The column titles of the traits table and of the folded APOE table. +fn head() -> Vec { + ["Trait", "Gene", "Genotype", "Reading", "Source"] + .map(str::to_owned) + .to_vec() +} + +/// One row of the traits table (and of the folded APOE table). +fn row(r: &Value) -> Result, Malformed> { + let f = |k: &str| need(r, k, ROWS).and_then(py); + let trait_ = f("trait")?; + let gene = Cell::Sub(f("gene")?, f("rsid")?); + let mut genotype = vec![Inline::Text(f("genotype")?)]; + genotype.extend(ancestry_badge(r, ROWS)?); + Ok(vec![ + Cell::Text(trait_), + gene, + Cell::Parts(genotype), + Cell::Sub(f("reading")?, f("note")?), + Cell::Text(format!("PMID {}", f("pmids")?)), + ]) +} + +/// The genotype rules table, none without rules. +fn rules_html(d: &Section<'_>, sample: &str) -> Result, Malformed> { + let rules = list(d, "rules", false)?; + if rules.is_empty() { + return Ok(Vec::new()); + } + let mut rows = Vec::new(); + for r in rules { + let f = |k: &str| need(r, k, RULES).and_then(py); + let trait_ = f("trait")?; + let gene = Cell::Sub(f("gene")?, f("rsids")?.replace(',', ", ")); + let genotypes = f("genotypes")?.replace(',', ", "); + let mut result = vec![Inline::Text(g2::rule_result(r)?)]; + result.extend(ancestry_badge(r, RULES)?); + result.extend(g2::g2_result_html(r)?); + let mut reading = vec![Inline::Text(f("reading")?)]; + reading.extend(g2::g2_reading_html(r)?); + reading.push(Inline::Small(f("note")?)); + rows.push(vec![ + Cell::Text(trait_), + gene, + Cell::Text(genotypes), + Cell::Parts(result), + Cell::Parts(reading), + Cell::Text(format!("PMID {}", f("pmids")?)), + ]); + } + Ok(vec![ + Line::Raw("

    Genotype rules (several sites read together)

    ".to_owned()), + Line::Table(Table { + head: [ + "Trait", + "Gene and sites", + "Genotypes", + "Result", + "Reading", + "Source", + ] + .map(str::to_owned) + .to_vec(), + rows, + ..Table::default() + }), + Line::Note(format!( + "Every rule line: traits/{sample}_genotype_rules.tsv" + )), + ]) +} + +/// The traits card's lines: the counts, one table row per site read or not +/// read, the rules, and the APOE lines folded away behind a warning. +pub fn html_lines(d: &Section<'_>, sample: &str) -> Result, Malformed> { + let n = |k: &str| d.need(k).and_then(py); + let mut out = vec![ + Line::Stat( + "Read / no genotype / other allele".to_owned(), + Inline::Text(format!( + "{} / {} / {}", + n("called")?, + n("not_covered")?, + n("other_allele")? + )), + ), + Line::Table(Table { + head: head(), + rows: list(d, "rows", true)? + .iter() + .map(row) + .collect::>()?, + ..Table::default() + }), + ]; + out.extend(rules_html(d, sample)?); + out.extend(apoe::html_line(list(d, "apoe", false)?, head(), row)?); + out.push(Line::Note(format!( + "Every row with its note: traits/{sample}_traits.tsv" + ))); + Ok(out) +} + +/// The HTML report's card, shown on every report: "Not run" when step 42 +/// is not in the summary. +pub fn html(s: &Summary) -> Result { + let sec = s.section("traits"); + let (body, caveats) = if sec.state.ran() { + let sample = py(s.need("sample")?)?; + (html_lines(&sec, &sample)?, vec![TRAITS_NOTE]) + } else { + (Vec::new(), Vec::new()) + }; + let card = html::card( + &sec, + Card { + title: "Traits".to_owned(), + full: true, + caveats, + body, + ..Card::default() + }, + )?; + Ok(HtmlPart { + cards: vec![card], + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/two_hits.rs b/rust/crates/og-render/src/sections/two_hits.rs index 695939b43..5fac84288 100644 --- a/rust/crates/og-render/src/sections/two_hits.rs +++ b/rust/crates/og-render/src/sections/two_hits.rs @@ -1,2 +1,71 @@ -//! The report section `two_hits`. Empty until its unit of the 1.0.0 plan -//! lands. +//! The genes a called deletion and a small variant both reach (og summary +//! collect's two-hit rule), shown under the carrier list: each gene with +//! its verdict, then which deletion callers were read. + +use crate::model::Malformed; +use crate::sections::clinvar::D; + +/// `str.capitalize()`: the first character upper case, the rest lower. +fn capitalize(s: &str) -> String { + let mut c = s.chars(); + match c.next() { + Some(first) => { + pycompat::text::upper(&first.to_string()) + &pycompat::text::lower(c.as_str()) + } + None => String::new(), + } +} + +/// `"; ".join(items)` of a list of strings. +fn joined(v: &D<'_>) -> Result { + let items = v.items()?; + let parts: Vec<&str> = items.iter().map(D::string).collect::>()?; + Ok(parts.join("; ")) +} + +/// The lines, none for a summary written before the rule. +pub fn lines(t: Option<&D<'_>>) -> Result, Malformed> { + let Some(t) = t.filter(|t| t.truthy()) else { + return Ok(Vec::new()); + }; + let mut out = Vec::new(); + let genes = t.at("genes")?; + for x in genes.items()? { + let allele = match x.opt("allele")? { + Some(a) => format!(" ({})", a.text()?), + None => String::new(), + }; + out.push(format!( + "{}: {} ({}; AR), both copies may be hit: {} and {} {}{allele}; {}", + capitalize(x.at("verdict")?.string()?), + x.s("gene")?, + x.s("condition")?, + x.s("deletion")?, + x.s("variant")?, + x.s("genotype")?, + x.s("why")? + )); + } + let found = if genes.truthy() { + format!("{} gene(s); ", genes.len()?) + } else { + "none found; ".to_owned() + }; + let read = joined(&t.at("deletions_read")?)?; + let read = if read.is_empty() { + "no caller".to_owned() + } else { + read + }; + let not_read = t.at("deletions_not_read")?; + let not_read = if not_read.truthy() { + format!("; not read: {}", joined(¬_read)?) + } else { + String::new() + }; + out.push(format!( + "Two hits across callers (a deletion plus a variant on the other copy): {found}deletions read from \ + {read}{not_read}" + )); + Ok(out) +} diff --git a/rust/crates/og-render/src/sections/viral.rs b/rust/crates/og-render/src/sections/viral.rs index 1de0d75ba..62a65af87 100644 --- a/rust/crates/og-render/src/sections/viral.rs +++ b/rust/crates/og-render/src/sections/viral.rs @@ -8,7 +8,8 @@ use pycompat::json::Value; use crate::model::{ - Card, Cell, HtmlPart, Inline, Line, Malformed, Section, Summary, Table, is_str, need, py, + Card, Cell, HtmlPart, Inline, Layout, Line, Malformed, Section, Summary, Table, is_str, need, + py, }; use crate::notes::VIRAL_NOTE; use crate::{html, text}; @@ -154,7 +155,7 @@ pub fn html(s: &Summary) -> Result { .to_vec(), rows, viewer: false, - one_line: true, + layout: Layout::OneLine, }) }); } diff --git a/rust/crates/og-render/src/sections/y_tree.rs b/rust/crates/og-render/src/sections/y_tree.rs index 892584a1c..042d22c01 100644 --- a/rust/crates/og-render/src/sections/y_tree.rs +++ b/rust/crates/og-render/src/sections/y_tree.rs @@ -1,2 +1,151 @@ -//! The report section `y_tree`. Empty until its unit of the 1.0.0 plan -//! lands. +//! The second Y placement (step 37, `og y tree-place`): the branch on a +//! public tree release newer than Yleaf's, the SNP counts on the path to +//! it, and one row per level of the walk. + +use crate::model::{Card, Cell, Inline, Layout, Line, Malformed, State, Summary, Table}; +use crate::notes::{EXPLORE_NOTE, Y_TREE_NOTE}; +use crate::sections::ancestry::{ + Align, explore_html, get_text, list, need_align, need_text, py_or, shown, +}; +use crate::{html, text}; + +const PATH: &str = "sections.y_tree.data"; +const ROW: &str = "sections.y_tree.data.path[]"; + +/// The two lists of ancestral calls a reader should see, with their labels. +const ANCESTRAL: [(&str, &str); 2] = [ + ( + "Ancestral inside the terminal branch", + "ancestral_in_terminal", + ), + ( + "Ancestral on the path, outvoted by the SNPs below it", + "outvoted_ancestral", + ), +]; + +/// The text report's block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "y_tree", Some("Y Placement on the Current Public Tree"))? + else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if !head.ran { + return Ok(w); + } + let d = s.section("y_tree").data; + let n = |k: &str| need_text(d, k, PATH); + w.push(format!( + " Branch: {} (level {}, {})", + n("haplogroup")?, + n("level")?, + n("tree")? + )); + w.push(format!( + " Path SNPs: {} derived, {} ancestral, {} no call; {} SNPs called at depth {} or more", + n("path_derived")?, + n("path_ancestral")?, + n("path_no_call")?, + n("snps_called")?, + n("min_depth")? + )); + for (label, key) in ANCESTRAL { + if shown(d, key, PATH)?.is_some() { + w.push(format!(" {label}: {}", n(key)?)); + } + } + for x in list(d, "path", PATH)? { + let a = |k: &str, width: usize, side: Align| need_align(&x, k, width, side, ROW); + w.push(format!( + " {} {} {}D {}A {}N", + a("Level", 3, Align::Right)?, + a("Branch", 22, Align::Left)?, + a("Derived", 3, Align::Right)?, + a("Ancestral", 3, Align::Right)?, + a("No_call", 4, Align::Right)? + )); + } + w.push(format!(" {Y_TREE_NOTE}")); + w.push(String::new()); + Ok(w) +} + +/// The HTML report's card, when step 37 placed the sample on the tree. +pub fn card(s: &Summary) -> Result, Malformed> { + let sec = s.section("y_tree"); + if sec.state == State::Missing { + return Ok(None); + } + let mut c = Card { + title: "Y Placement on the Current Public Tree".to_owned(), + full: true, + ..Card::default() + }; + if sec.state.ran() { + let d = sec.data; + let g = |k: &str| get_text(d, k, ".", PATH); + for (label, key) in [ + ("Branch", "haplogroup"), + ("Level", "level"), + ("Tree", "tree"), + ] { + c.body + .push(Line::Stat(label.to_owned(), Inline::Text(g(key)?))); + } + c.body.push(Line::Para(format!( + "Path SNPs: {} derived, {} ancestral, {} no call; {} SNPs called at depth {} or more", + g("path_derived")?, + g("path_ancestral")?, + g("path_no_call")?, + g("snps_called")?, + g("min_depth")? + ))); + for (label, key) in ANCESTRAL { + if let Some(v) = shown(d, key, PATH)? { + c.body + .push(Line::Para(format!("{label}: {}", py_or(Some(v), "")?))); + } + } + let mut rows = Vec::new(); + for x in list(d, "path", PATH)? { + rows.push( + [ + "Level", + "Branch", + "Derived", + "Ancestral", + "No_call", + "Ancestral_SNPs", + ] + .iter() + .map(|k| need_text(&x, k, ROW).map(Cell::Text)) + .collect::, Malformed>>()?, + ); + } + if !rows.is_empty() { + c.body.push(Line::Table(Table { + head: [ + "Level", + "Branch", + "Derived", + "Ancestral", + "No call", + "Ancestral SNPs", + ] + .map(str::to_owned) + .to_vec(), + rows, + layout: Layout::OneLine, + ..Table::default() + })); + } + let (links, note) = explore_html(d, PATH)?; + c.body.extend(links); + if note { + c.caveats.push(EXPLORE_NOTE); + } + c.caveats.push(Y_TREE_NOTE); + } + Ok(Some(html::card(&sec, c)?)) +} diff --git a/rust/crates/og-render/src/sections/ystr.rs b/rust/crates/og-render/src/sections/ystr.rs index 7e7922e39..f15eb1518 100644 --- a/rust/crates/og-render/src/sections/ystr.rs +++ b/rust/crates/og-render/src/sections/ystr.rs @@ -1 +1,105 @@ -//! The report section `ystr`. Empty until its unit of the 1.0.0 plan lands. +//! The Y-STR markers (step 45, HipSTR): the repeat count of each marker +//! called, in FamilyTreeDNA's Y-111 order and panels, so a count lines up +//! with FamilyTreeDNA results and surname-project tables. + +use pycompat::json::Value; + +use crate::model::{Card, Cell, Layout, Line, Malformed, State, Summary, Table, need, truthy}; +use crate::notes::YSTR_NOTE; +use crate::sections::ancestry::{Align, items, need_align, need_text, py_len}; +use crate::{html, text}; + +const PATH: &str = "sections.y_str.data"; +const ROW: &str = "sections.y_str.data.markers[]"; + +/// How many markers were called, and with what confidence. +pub fn counts(d: &Value) -> Result { + let n = |k: &str| need_text(d, k, PATH); + let high = n("high")?; + let low = n("low")?; + let all = py_len(need(d, "markers", PATH)?, &format!("{PATH}.markers"))?; + Ok(format!( + "Markers called: {high} high and {low} low confidence of {all}; {} no call, {} not \ + callable from short reads", + n("no_call")?, + n("not_callable")? + )) +} + +/// The markers with a count, in the order the summary lists them (the +/// panel's order). +fn called(markers: &Value) -> Result, Malformed> { + let mut out = Vec::new(); + for x in items(markers, &format!("{PATH}.markers"))? { + if truthy(need(&x, "count", ROW)?) { + out.push(x); + } + } + Ok(out) +} + +/// The text report's block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "y_str", Some("Y-STR Markers (HipSTR)"))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if !head.ran { + return Ok(w); + } + let d = s.section("y_str").data; + w.push(format!(" {}", counts(d)?)); + for x in called(need(d, "markers", PATH)?)? { + w.push(format!( + " {} {} {} ({})", + need_align(&x, "panel", 6, Align::Left, ROW)?, + need_align(&x, "marker", 13, Align::Left, ROW)?, + need_align(&x, "count", 6, Align::Left, ROW)?, + need_text(&x, "confidence", ROW)? + )); + } + w.push(format!(" {YSTR_NOTE}")); + w.push(String::new()); + Ok(w) +} + +/// The HTML report's card, when step 45 has a result. +pub fn card(s: &Summary) -> Result, Malformed> { + let sec = s.section("y_str"); + if sec.state == State::Missing { + return Ok(None); + } + let mut c = Card { + title: "Y-STR Markers (HipSTR)".to_owned(), + full: true, + ..Card::default() + }; + if sec.state.ran() { + let d = sec.data; + let markers = crate::model::get(d, "markers", PATH)?; + let mut rows = Vec::new(); + if let Some(m) = markers { + for x in called(m)? { + rows.push( + ["panel", "marker", "count", "confidence"] + .iter() + .map(|k| need_text(&x, k, ROW).map(Cell::Text)) + .collect::, Malformed>>()?, + ); + } + } + c.body = vec![ + Line::Para(counts(d)?), + Line::Table(Table { + head: ["Panel", "Marker", "Repeats", "Confidence"] + .map(str::to_owned) + .to_vec(), + rows, + layout: Layout::OneLine, + ..Table::default() + }), + ]; + c.caveats.push(YSTR_NOTE); + } + Ok(Some(html::card(&sec, c)?)) +} diff --git a/rust/crates/og-render/tests/ancestry_ancient.rs b/rust/crates/og-render/tests/ancestry_ancient.rs new file mode 100644 index 000000000..47bc88bcc --- /dev/null +++ b/rust/crates/og-render/tests/ancestry_ancient.rs @@ -0,0 +1,760 @@ +//! `og ancient` (step 44) on synthetic files, run as the pipeline runs it, +//! then the summary and both reports. It replaces +//! `tests/test_ancient_report.py`. +//! +//! 1. ranking: on a synthetic .evec the nearest ancient group is the one +//! whose mean position is closest on the weighted PCs, a group far on PC1 +//! but with a small eigenvalue on PC3 is ranked by the weighted distance, +//! and each row carries its dates (oldest first, AADR style BCE), country +//! and member count; +//! 2. axes: a run whose PC2 has the opposite sign is turned back before the +//! person is placed, and a run whose present-day coordinates differ +//! stops with the reason instead of placing the person on other axes; +//! 3. liftover: lift keeps a site inside a block, a site on a minus-strand +//! chain with its alleles complemented, and drops a site in a gap, one +//! whose alleles miss the GRCh38 base, chrX and a monomorphic marker; +//! 4. genotypes: person-geno appends one column, 9 where the person has no +//! usable genotype, and stops on a genotype file shorter than the .snp +//! file; +//! 5. select: the present-day reference people come from MODERN_GROUPS, the +//! ancient ones need a date, a passing assessment, enough sites and a +//! place in West Eurasia, one genetic id per individual; +//! 6. the report command writes the table, the summary, the text and an SVG +//! with the person marked; +//! 7. both reports show them: an HTML card with the table and the plot, and +//! a section of the text report. +//! +//! The unit tests in og-ancestry and og-intervals check the date labels, +//! the dosages and the single-position lift in detail. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use og_ancestry::ancient::cli; +use og_render::model::Summary; +use og_render::{html, text}; +use pycompat::json::{Dump, dumps}; + +const CAVEAT: &str = "Closeness on a PCA is genetic similarity, not descent"; +const EVALS: [f64; 4] = [9.0, 4.0, 0.01, 0.01]; + +fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +/// A scratch folder that is removed when the test ends, also when it fails. +struct Tmp(PathBuf); + +impl std::ops::Deref for Tmp { + type Target = Path; + fn deref(&self) -> &Path { + &self.0 + } +} + +impl AsRef for Tmp { + fn as_ref(&self) -> &Path { + &self.0 + } +} + +impl AsRef for Tmp { + fn as_ref(&self) -> &std::ffi::OsStr { + self.0.as_os_str() + } +} + +impl Tmp { + /// A new folder that only this test owns. `create_dir` fails when the + /// path exists, so a run never shares or deletes another run's folder. + fn new(kind: &str, tag: &str) -> Tmp { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let d = std::env::temp_dir().join(format!( + "og-ancestry-{kind}-{}-{tag}-{nanos}", + std::process::id() + )); + std::fs::create_dir(&d).unwrap(); + Tmp(d) + } +} + +impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +fn tmp(tag: &str) -> Tmp { + Tmp::new("ancient", tag) +} + +fn write(d: &Path, name: &str, text: &str) -> String { + let p = d.join(name); + std::fs::write(&p, text).unwrap(); + p.display().to_string() +} + +fn read(p: &str) -> String { + std::fs::read_to_string(p).unwrap_or_default() +} + +fn og(args: &[&str]) -> (u8, String, String) { + let args: Vec = args.iter().map(|a| (*a).to_owned()).collect(); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = cli(&args, &mut out, &mut err); + ( + code, + String::from_utf8(out).unwrap(), + String::from_utf8(err).unwrap(), + ) +} + +/// One person of an .evec: id, label, four PCs. +type Pc = (String, &'static str, [f64; 4]); + +fn evec(rows: &[Pc], evals: [f64; 4]) -> String { + let mut out = vec![format!( + "#eigvals: {}", + evals + .iter() + .map(|v| format!("{v:.3}")) + .collect::>() + .join(" ") + )]; + for (i, label, pcs) in rows { + out.push(format!( + "{i} {} {label}", + pcs.iter() + .map(|v| format!("{v:.6}")) + .collect::>() + .join(" ") + )); + } + out.join("\n") + "\n" +} + +/// Present-day people M0..M19: they build the axes. +fn moderns() -> Vec { + (0..20) + .map(|i| { + let f = f64::from(i); + ( + format!("M{i}"), + if i % 2 == 1 { "French" } else { "Sardinian" }, + [ + f * 0.01 - 0.1, + f64::from(i % 5) * 0.02 - 0.04, + f64::from(i % 3) * 0.01 - 0.01, + f64::from(i % 4) * 0.01 - 0.015, + ], + ) + }) + .collect() +} + +/// The panel: the present-day people and three ancient groups. +fn panel() -> Vec { + let mut p = moderns(); + p.extend([ + ("A1".to_owned(), "Ancient", [0.10, 0.10, 0.0, 0.0]), // Steppe + ("A2".to_owned(), "Ancient", [0.12, 0.10, 0.0, 0.0]), + ("B1".to_owned(), "Ancient", [-0.20, 0.00, 0.0, 0.0]), // Farmer + // Close on PC1 and PC2, far on PC3 (eigenvalue 0.01). + ("C1".to_owned(), "Ancient", [0.00, 0.00, 0.9, 0.0]), + ]); + p +} + +/// The person's run: the present-day people with PC2 flipped, as smartpca +/// may write it, and the person. +fn run_rows() -> Vec { + let mut r: Vec = moderns() + .into_iter() + .map(|(k, l, p)| (k, l, [p[0], -p[1], p[2], p[3]])) + .collect(); + r.push(("S".to_owned(), "Person", [0.08, -0.09, 0.0, 0.0])); + r +} + +/// The files of one report run. +struct Report { + tsv: String, + info: String, + txt: String, + svg: String, + ancients: String, + stats: String, +} + +impl Report { + fn new(d: &Path) -> Report { + let mut anc = "id\tgroup\tdate_bp\tdate_sd\tfull_date\tlocality\tcountry\tlat\tlon\tsnps\n" + .to_owned(); + for (id, group, bp, country, locality) in [ + ("A1", "Steppe_EBA", "4800", "Russia", "Samara"), + ("A2", "Steppe_EBA", "4600", "Russia", "Samara"), + ("B1", "Anatolia_N", "8200", "Turkey", "Barcin"), + ("C1", "Odd_C", "3000", "Italy", "Somewhere"), + ] { + anc += &format!("{id}\t{group}\t{bp}\t50\tx\t{locality}\t{country}\t50\t40\t500000\n"); + } + let o = |k: &str| d.join(format!("S_{k}")).display().to_string(); + Report { + tsv: o("ancient.tsv"), + info: o("info.tsv"), + txt: o("ancient.txt"), + svg: o("pca.svg"), + ancients: write(d, "anc.tsv", &anc), + stats: write( + d, + "stats.tsv", + "panel_sites\t4\nsites_genotyped\t3\nother_allele\t1\nsites_used\t2\n", + ), + } + } + + fn run(&self, panel_evec: &str, run_evec: &str, with_stats: bool) -> (u8, String, String) { + let mut a = vec![ + "report", + "--panel-evec", + panel_evec, + "--run-evec", + run_evec, + "--ancients", + &self.ancients, + "--sample", + "S", + "--release", + "test", + ]; + if with_stats { + a.extend(["--stats", &self.stats]); + } + a.extend([ + "--out-tsv", + &self.tsv, + "--out-info", + &self.info, + "--out-txt", + &self.txt, + "--out-svg", + &self.svg, + ]); + og(&a) + } + + fn table(&self) -> Vec> { + let t = read(&self.tsv); + let mut lines = t.lines(); + let head: Vec<&str> = lines.next().unwrap_or_default().split('\t').collect(); + lines + .map(|l| { + head.iter() + .zip(l.split('\t')) + .map(|(h, v)| ((*h).to_owned(), v.to_owned())) + .collect() + }) + .collect() + } + + fn groups(&self) -> Vec { + self.table() + .into_iter() + .map(|r| r["group"].clone()) + .collect() + } +} + +#[test] +fn the_nearest_group_on_the_weighted_pcs_comes_first() { + let d = tmp("rank"); + let r = Report::new(&d); + let pe = write(&d, "panel.evec", &evec(&panel(), EVALS)); + let re = write(&d, "run.evec", &evec(&run_rows(), EVALS)); + let (code, _, err) = r.run(&pe, &re, true); + assert_eq!(code, 0, "{err}"); + assert_eq!( + r.groups(), + ["Steppe_EBA", "Odd_C", "Anatolia_N"], + "Steppe_EBA first from the flipped run turned back; Odd_C, far only on a PC with a tiny \ + eigenvalue, second" + ); + let rows = r.table(); + let st = &rows[0]; + assert_eq!( + [&st["n"], &st["dates"], &st["country"], &st["locality"]], + ["2", "2850-2650 BCE", "Russia", "Samara"] + ); + let ranks: Vec<&str> = rows.iter().map(|x| x["rank"].as_str()).collect(); + assert_eq!(ranks, ["1", "2", "3"]); + + // Control: without the eigenvalue weights Odd_C falls behind Anatolia_N. + let fe = write(&d, "flat_panel.evec", &evec(&panel(), [1.0; 4])); + let fr = write(&d, "flat_run.evec", &evec(&run_rows(), [1.0; 4])); + let (code, _, err) = r.run(&fe, &fr, true); + assert_eq!(code, 0, "{err}"); + assert_eq!(r.groups(), ["Steppe_EBA", "Anatolia_N", "Odd_C"]); +} + +#[test] +fn a_flipped_axis_is_turned_back_and_other_axes_stop() { + let d = tmp("align"); + let pe = write(&d, "panel.evec", &evec(&panel(), EVALS)); + let re = write(&d, "run.evec", &evec(&run_rows(), EVALS)); + let (code, out, err) = og(&[ + "align", + "--panel-evec", + &pe, + "--run-evec", + &re, + "--pcs", + "2", + ]); + assert_eq!(code, 0, "{err}"); + assert!(out.contains("PC1+ PC2-"), "{out}"); + let bad: Vec = moderns() + .into_iter() + .map(|(k, l, p)| { + let n: u32 = k[1..].parse().unwrap(); + let s = if n.is_multiple_of(3) { -1.0 } else { 1.0 }; + (k, l, [p[0] * s, p[1], p[2], p[3]]) + }) + .collect(); + let be = write(&d, "bad.evec", &evec(&bad, EVALS)); + let (code, _, err) = og(&[ + "align", + "--panel-evec", + &pe, + "--run-evec", + &be, + "--pcs", + "2", + ]); + assert_eq!(code, 1, "{err}"); + assert!(err.contains("PC1"), "{err}"); +} + +#[test] +fn lift_keeps_sites_in_blocks_and_complements_the_minus_strand() { + let d = tmp("lift"); + let chain = write( + &d, + "test.chain", + &([ + // chr1 source 1000-1100 -> target 2000-2100 (+), gap of 50, then + // 1150-1200 -> 2150-2200. + "chain 100 chr1 100000 + 1000 1200 chr1 100000 + 2000 2200 1", + "100 50 50", + "50", + "", + // chr2 source 500-600 -> target on the minus strand of a 10,000 bp + // chr2, reverse start 300. + "chain 90 chr2 50000 + 500 600 chr2 10000 - 300 400 2", + "100", + "", + // chr3: two chains over 100-200. + "chain 80 chr3 50000 + 100 200 chr3 50000 + 100 200 3", + "100", + "", + "chain 70 chr3 50000 + 150 250 chr3 50000 + 5150 5250 4", + "100", + "", + ] + .join("\n") + + "\n"), + ); + let mut seq1 = vec![b'N'; 3000]; + seq1[2050] = b'G'; // 1-based 2051 + seq1[2159] = b'T'; // 1-based 2160 + let mut seq2 = vec![b'N'; 10000]; + seq2[9689] = b'C'; // 1-based 9690: the complement of the source G + let (mut fa, mut fai, mut off) = (String::new(), String::new(), 0usize); + for (name, seq) in [("chr1", &seq1), ("chr2", &seq2)] { + let head = format!(">{name}\n"); + fa += &head; + off += head.len(); + for c in seq.chunks(60) { + fa += std::str::from_utf8(c).unwrap(); + fa.push('\n'); + } + fai += &format!("{name}\t{}\t{off}\t60\t61\n", seq.len()); + off += seq.len() + seq.len().div_ceil(60); + } + let fa = write(&d, "ref.fa", &fa); + write(&d, "ref.fa.fai", &fai); + let snp = write( + &d, + "src.snp", + &([ + "rs1 1 0.0 1051 G A", // ref G at target: kept + "rs2 1 0.0 1160 C T", // second allele is the target base: kept + "rs3 1 0.0 1120 A G", // gap: dropped + "rs4 2 0.0 511 G T", // minus strand: alleles C/A, target C: kept + "rs5 1 0.0 1051 A C", // neither allele is G: dropped + "rs6 23 0.0 1051 G A", // X: dropped + "rs7 1 0.0 1051 G X", // monomorphic marker: dropped + ] + .join("\n") + + "\n"), + ); + let out = d.join("sites.tsv").display().to_string(); + let (code, _, err) = og(&[ + "lift", "--snp", &snp, "--chain", &chain, "--fasta", &fa, "--out", &out, + ]); + assert_eq!(code, 0, "{err}"); + assert_eq!( + read(&out).lines().collect::>(), + [ + "snp\tchrom\tpos\tallele1\tallele2", + "rs1\tchr1\t2051\tG\tA", + "rs2\tchr1\t2160\tC\tT", + "rs4\tchr2\t9690\tC\tA" + ] + ); + + // person-geno on those sites. + let msnp = write( + &d, + "moderns.snp", + "rs1 1 0.0 1051 G A\nrs2 1 0.0 1160 C T\nrsX 23 0.0 5 A G\nrs4 2 0.0 511 G T\n", + ); + let mgeno = write(&d, "moderns.geno", "012\n220\n999\n101\n"); + let gts = write( + &d, + "gts.tsv", + "chr1\t2051\tG\tA\tG/A\nchr1\t2160\tT\t.\tT/T\nchr2\t9690\tC\tG\tC/G\n", + ); + let merged = d.join("merged.geno").display().to_string(); + let pstats = d.join("pstats.tsv").display().to_string(); + let geno = |m: &str| { + og(&[ + "person-geno", + "--sites", + &out, + "--genotypes", + >s, + "--snp", + &msnp, + "--moderns-geno", + m, + "--out", + &merged, + "--stats", + &pstats, + ]) + }; + let (code, _, err) = geno(&mgeno); + assert_eq!(code, 0, "{err}"); + assert_eq!( + read(&merged).lines().collect::>(), + ["0121", "2200", "9999", "1019"], + "one column: 1, 0, 9 for X, 9 for an allele outside the pair" + ); + let st: BTreeMap = read(&pstats) + .lines() + .map(|l| { + let (k, v) = l.split_once('\t').unwrap(); + (k.to_owned(), v.to_owned()) + }) + .collect(); + let want: BTreeMap = [ + ("panel_sites", "4"), + ("sites_genotyped", "3"), + ("other_allele", "1"), + ("sites_used", "2"), + ] + .map(|(k, v)| (k.to_owned(), v.to_owned())) + .into(); + assert_eq!(st, want); + let short = write(&d, "short.geno", "012\n220\n"); + let (code, _, err) = geno(&short); + assert_eq!( + code, 1, + "a genotype file shorter than the .snp file stops: {err}" + ); +} + +#[test] +fn select_takes_reference_and_ancient_people_by_the_rules() { + let d = tmp("select"); + let head = [ + "Genetic ID (long text)", + "Persistent Genetic ID", + "Individual ID", + "\"Date mean in BP in years before 1950\"", + "Date standard deviation in BP [x]", + "Full Date One of two formats", + "Group ID", + "Locality", + "Political Entity", + "Latitude", + "Longitude", + "Data type", + "SNPs hit on autosomal targets (Computed using easystats on 1240k snpset)", + "SNPs hit on autosomal targets (Computed using easystats on HO snpset)", + "ASSESSMENT", + "ASSESSMENT WARNINGS (long)", + ] + .map(str::to_owned) + .to_vec(); + #[allow(clippy::too_many_arguments)] + fn arow( + gid: &str, + ind: &str, + bp: u32, + group: &str, + lat: &str, + lon: &str, + dtype: &str, + snps: u32, + ass: &str, + ) -> Vec { + [ + gid, + gid, + ind, + &bp.to_string(), + "50", + "x", + group, + "Loc", + "Land", + lat, + lon, + dtype, + &snps.to_string(), + "0", + ass, + "", + ] + .map(str::to_owned) + .to_vec() + } + let mut rows: Vec> = (0..60) + .map(|i| { + arow( + &format!("F{i}.DG"), + &format!("F{i}"), + 0, + "French", + "46", + "2", + "Shotgun.diploid", + 1_100_000, + "PROVISIONAL_PASS", + ) + }) + .collect(); + rows.extend([ + // The same individual with fewer sites. + arow( + "F0.SG", + "F0", + 0, + "French", + "46", + "2", + "Shotgun.diploid", + 900_000, + "PASS", + ), + // Not a reference group. + arow( + "Y1.DG", + "Y1", + 0, + "Yakut", + "62", + "129", + "Shotgun.diploid", + 1_100_000, + "PASS", + ), + // Not shotgun diploid. + arow( + "H1.HO", + "H1", + 0, + "French", + "46", + "2", + "HumanOrigins", + 600_000, + "PASS", + ), + arow( + "A1.AG", + "A1", + 4800, + "Russia_Steppe", + "53", + "50", + "1240k", + 500_000, + "PASS", + ), + // Too few sites. + arow( + "A2.AG", + "A2", + 4800, + "Russia_Steppe", + "53", + "50", + "1240k", + 50_000, + "PASS", + ), + arow( + "A3.AG", + "A3", + 4800, + "Russia_Steppe", + "53", + "50", + "1240k", + 500_000, + "Questionable", + ), + // Outside West Eurasia. + arow( + "A4.AG", "A4", 4800, "China_N", "40", "116", "1240k", 500_000, "PASS", + ), + // No place. + arow( + "A5.AG", "A5", 4800, "Unknown", "..", "..", "1240k", 500_000, "PASS", + ), + ]); + let anno_text: String = std::iter::once(&head) + .chain(rows.iter()) + .map(|r| r.join("\t") + "\n") + .collect(); + let anno = write(&d, "x.anno", &anno_text); + let ind_text: String = rows + .iter() + .map(|r| format!("{} U {}\n", r[0], r[6])) + .collect(); + let ind = write(&d, "x.ind", &ind_text); + let o = |n: &str| d.join(n).display().to_string(); + let (oind, plist, anc) = (o("panel.ind"), o("poplist.txt"), o("ancients.tsv")); + let (code, _, err) = og(&[ + "select", + "--anno", + &anno, + "--ind", + &ind, + "--out-ind", + &oind, + "--poplist", + &plist, + "--ancients", + &anc, + ]); + assert_eq!(code, 0, "{err}"); + let labels: Vec<(String, String)> = read(&oind) + .lines() + .map(|l| { + let c: Vec<&str> = l.split('\t').collect(); + (c[0].to_owned(), c[2].to_owned()) + }) + .collect(); + let label = |k: &str| labels.iter().find(|(i, _)| i == k).map(|(_, v)| v.as_str()); + assert_eq!( + labels.iter().filter(|(_, v)| v == "French").count(), + 60, + "the 60 French shotgun genomes are the reference" + ); + assert_eq!(label("F0.SG"), Some("Ignore"), "one id per person (F0.DG)"); + assert_eq!(label("Y1.DG"), Some("Ignore")); + assert_eq!(label("H1.HO"), Some("Ignore")); + let projected: Vec<&str> = labels + .iter() + .filter(|(_, v)| v == "Ancient") + .map(|(k, _)| k.as_str()) + .collect(); + assert_eq!(projected, ["A1.AG"]); + assert_eq!(read(&plist), "French\n"); + assert!( + read(&anc) + .lines() + .nth(1) + .unwrap() + .starts_with("A1.AG\tRussia_Steppe\t4800") + ); +} + +/// The summary og summary collect writes for S, as text. +fn collect(sd: &Path) -> String { + let input = og_summary::collect::Input { + sample: "S".to_owned(), + sample_dir: sd.display().to_string(), + declared_sex: None, + assets: repo().join("assets").display().to_string(), + group: og_summary::collect::Group::All, + tables: repo().join("bin").display().to_string(), + }; + dumps( + &og_summary::collect::post::collect(&input).unwrap(), + &Dump::default(), + ) + .unwrap() +} + +#[test] +fn the_report_writes_its_files_and_both_reports_show_them() { + let d = tmp("report"); + let r = Report::new(&d); + let pe = write(&d, "panel.evec", &evec(&panel(), EVALS)); + let re = write(&d, "run.evec", &evec(&run_rows(), EVALS)); + let (code, _, err) = r.run(&pe, &re, true); + assert_eq!(code, 0, "{err}"); + let info: BTreeMap = read(&r.info) + .lines() + .skip(1) + .map(|l| { + let (k, v) = l.split_once('\t').unwrap(); + (k.to_owned(), v.to_owned()) + }) + .collect(); + assert_eq!( + info["pc2"], "0.09000", + "the person's PC2 in the panel's sign" + ); + assert_eq!(info["sites_used"], "2"); + let svg = read(&r.svg); + assert!(svg.starts_with("S"), "the person's label"); + assert!(svg.trim_end().ends_with("")); + let txt = read(&r.txt); + assert!(txt.contains(CAVEAT) && txt.contains("qpAdm"), "{txt}"); + + // Both reports read the outputs. + let sd = d.join("S"); + std::fs::create_dir_all(sd.join("ancient")).unwrap(); + for (from, to) in [ + (&r.tsv, "S_ancient.tsv"), + (&r.info, "S_ancient_info.tsv"), + (&r.txt, "S_ancient.txt"), + (&r.svg, "S_ancient_pca.svg"), + ] { + std::fs::copy(from, sd.join("ancient").join(to)).unwrap(); + } + let s = Summary::parse(&collect(&sd)).unwrap(); + let page = html::report(&s).unwrap(); + assert!(page.contains("

    Ancient Populations

    ")); + assert!(page.contains("1Steppe_EBA2850-2650 BCERussia")); + assert!(page.contains("data:image/svg+xml;base64,"), "the plot"); + let report = text::report(&s).unwrap(); + assert!(report.contains("Ancient Populations (AADR PCA, step 44)")); + assert!(report.contains("1. Steppe_EBA ("), "{report}"); + assert!(report.contains("not descent")); + assert!(report.contains("Plot: ancient/S_ancient_pca.svg")); + + let (code, _, err) = r.run(&pe, &pe, false); + assert_eq!(code, 1, "a run without the person stops"); + assert!(err.contains("no row for S"), "{err}"); +} diff --git a/rust/crates/og-render/tests/ancestry_archaic.rs b/rust/crates/og-render/tests/ancestry_archaic.rs new file mode 100644 index 000000000..e6c53a5af --- /dev/null +++ b/rust/crates/og-render/tests/ancestry_archaic.rs @@ -0,0 +1,1349 @@ +//! `og archaic` (step 43) on a synthetic archaic panel, then the summary and +//! the report lines; on amd64 in CI also with the real IBDmix image. It +//! replaces `tests/test_archaic_report.py`. +//! +//! 1. table: a 1000 Genomes site keeps its frequencies and gets each archaic +//! genome's ALT count, 10 plus it outside that genome's mask or inside an +//! excluded region; an archaic record with another ALT or another REF is +//! unknown (9); a site only an archaic genome varies at gets frequency 0; +//! a site no archaic genome was called at is left out; a position given +//! twice by 1000 Genomes is left out; +//! 2. strict: a lifted row outside the GRCh38 strict mask is kept once per +//! position with every archaic call marked masked (9 stays 9); +//! 3. genotypes: a masked archaic call goes to IBDmix as its ALT count and +//! its position into that genome's mask file; the stand-in columns carry +//! the population's ALT count; your column is your ALT count, 9 for a +//! non-PASS record, and where your VCF has no record 0 inside a reference +//! block of the gVCF and 9 outside every one; a site where your VCF has +//! another ALT is left out; a chromosome your VCF has no record on is not +//! analysed; the split-VCF rules; unsorted input stops and keeps the +//! previous outputs; +//! 4. report: Altai and Vindija segments merge into Neanderthal ones, a +//! Denisova segment that overlaps one is not Denisovan, segments under +//! 50 kb or LOD 4 are dropped, the share is of the analysed span, and the +//! genes of the longest segments are named; og summary collect reads the +//! summary and marks a damaged one unreadable, and the report lines give +//! both shares; +//! 5. with IBDmix itself (`IBDMIX_IMAGE`, docker; amd64 only, run in CI or +//! with `OG_DOCKER_TESTS=1`): a planted Neanderthal and a planted +//! Denisovan stretch come out as one segment each, the same person +//! without them gets none; masked mismatches stop a segment where the +//! same calls written as unknown let it run through; a stretch with no +//! call gives a segment only when a gVCF reference block says it was read. + +use std::collections::BTreeMap; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; + +use flate2::Compression; +use flate2::read::GzDecoder; +use flate2::write::GzEncoder; +use og_ancestry::archaic::cli; +use pycompat::json::Value; + +const TABLE_HEADER: &str = "chrom\tpos\tref\talt\tAltai\tVindija\tDenisova\tAF_ALL\tAF_AFR\tAF_AMR\tAF_EAS\tAF_EUR\tAF_SAS"; +const VCF_HEAD: &str = + "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n"; +const ARCHAICS: [&str; 3] = ["Altai", "Vindija", "Denisova"]; + +fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +/// A scratch folder that is removed when the test ends, also when it fails. +struct Tmp(PathBuf); + +impl std::ops::Deref for Tmp { + type Target = Path; + fn deref(&self) -> &Path { + &self.0 + } +} + +impl AsRef for Tmp { + fn as_ref(&self) -> &Path { + &self.0 + } +} + +impl AsRef for Tmp { + fn as_ref(&self) -> &std::ffi::OsStr { + self.0.as_os_str() + } +} + +impl Tmp { + /// A new folder that only this test owns. `create_dir` fails when the + /// path exists, so a run never shares or deletes another run's folder. + fn new(kind: &str, tag: &str) -> Tmp { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let d = std::env::temp_dir().join(format!( + "og-ancestry-{kind}-{}-{tag}-{nanos}", + std::process::id() + )); + std::fs::create_dir(&d).unwrap(); + Tmp(d) + } +} + +impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +fn tmp(tag: &str) -> Tmp { + Tmp::new("archaic", tag) +} + +fn p(d: &Path, name: &str) -> String { + d.join(name).display().to_string() +} + +fn write(path: &str, text: &str) { + std::fs::write(path, text).unwrap(); +} + +fn gzip(path: &str, text: &str) { + let mut z = GzEncoder::new(std::fs::File::create(path).unwrap(), Compression::default()); + z.write_all(text.as_bytes()).unwrap(); + z.finish().unwrap(); +} + +fn gunzip(path: &str) -> String { + let mut s = String::new(); + GzDecoder::new(std::fs::File::open(path).unwrap()) + .read_to_string(&mut s) + .unwrap(); + s +} + +fn og(args: &[&str]) -> (u8, String) { + let args: Vec = args.iter().map(|a| (*a).to_owned()).collect(); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = cli(&args, &mut out, &mut err); + (code, String::from_utf8(err).unwrap()) +} + +fn ref_block(chrom: &str, start: u32, end: u32) -> String { + format!("{chrom}\t{start}\t.\tN\t<*>\t0\t.\tEND={end}\tGT:GQ:MIN_DP\t0/0:50:30\n") +} + +/// The records of a table VCF by position, and each one's INFO. +type Rows = BTreeMap>; +type Info = BTreeMap>; + +fn vcf_rows(path: &str) -> (Rows, Info) { + let rows: Rows = std::fs::read_to_string(path) + .unwrap() + .lines() + .filter(|l| !l.starts_with('#')) + .map(|l| { + let c: Vec = l.split('\t').map(str::to_owned).collect(); + (c[1].parse().unwrap(), c) + }) + .collect(); + let info = rows + .iter() + .map(|(k, c)| { + ( + *k, + c[7].split(';') + .map(|kv| { + let (a, b) = kv.split_once('=').unwrap(); + (a.to_owned(), b.to_owned()) + }) + .collect(), + ) + }) + .collect(); + (rows, info) +} + +fn codes(i: &Info, pos: u32) -> [&str; 3] { + ARCHAICS.map(|g| i[&pos][g].as_str()) +} + +/// The inputs of part 1 in `d`. +fn table_inputs(d: &Path) -> Vec { + let afs = "0.1\t0.2\t0.05\t0\t0.15\t0.1"; + write( + &p(d, "modern.tsv"), + &format!( + // every archaic genome called; Vindija has another ALT and + // Denisova another REF; no archaic record; a split multi-allelic + // site. + "100\tA\tG\t{afs}\n200\tC\tT\t{afs}\n300\tG\tA\t{afs}\n400\tT\tC\t{afs}\n400\tT\tG\t{afs}\n" + ), + ); + write( + &p(d, "altai.tsv"), + "100\tA\tG\t1/1\n200\tC\t.\t0/0\n500\tG\tC\t0/1\n600\tA\t.\t0/0\n", + ); + write( + &p(d, "vindija.tsv"), + "100\tA\t.\t0/0\n200\tC\tA\t0/1\n500\tG\t.\t0/0\n600\tA\t.\t0/0\n", + ); + write( + &p(d, "denisova.tsv"), + "100\tA\tG\t0|1\n200\tT\tC\t0/1\n400\tT\tC\t1/1\n500\tG\tT\t1/1\n", + ); + [ + "table".to_owned(), + "--chrom".to_owned(), + "22".to_owned(), + "--modern".to_owned(), + p(d, "modern.tsv"), + "--archaic".to_owned(), + format!("Altai={}", p(d, "altai.tsv")), + "--archaic".to_owned(), + format!("Vindija={}", p(d, "vindija.tsv")), + "--archaic".to_owned(), + format!("Denisova={}", p(d, "denisova.tsv")), + ] + .to_vec() +} + +fn table(common: &[String], extra: &[&str], out: &str) -> (u8, String) { + let mut a: Vec<&str> = common.iter().map(String::as_str).collect(); + a.extend(extra); + a.extend(["--out", out]); + og(&a) +} + +#[test] +fn the_table_codes_each_archaic_call() { + let d = tmp("table"); + let common = table_inputs(&d); + let (code, err) = table(&common, &[], &p(&d, "t.vcf")); + assert_eq!(code, 0, "{err}"); + let (recs, info) = vcf_rows(&p(&d, "t.vcf")); + assert_eq!( + recs.keys().copied().collect::>(), + [100, 200, 500], + "the modern sites with an archaic call and the archaic-only site" + ); + assert_eq!( + codes(&info, 100), + ["2", "0", "1"], + "each genome's ALT count" + ); + assert_eq!( + [&info[&100]["AF_ALL"], &info[&100]["AF_EUR"]], + ["0.1", "0.15"] + ); + assert_eq!( + codes(&info, 200), + ["0", "9", "9"], + "another ALT or another REF is unknown" + ); + assert_eq!(recs[&500][4], "C", "the first non-reference ALT"); + assert_eq!(codes(&info, 500), ["1", "0", "9"]); + assert_eq!(info[&500]["AF_ALL"], "0"); + assert_eq!(recs[&100][0], "chr22", "the chr prefix for the liftover"); +} + +#[test] +fn a_call_outside_its_mask_or_inside_an_excluded_region_is_masked() { + let d = tmp("mask"); + let common = table_inputs(&d); + // 1-based 51-150 and 451-550: 100 and 500, not 200. + write(&p(&d, "altai.bed"), "22\t50\t150\n22\t450\t550\n"); + write(&p(&d, "vindija.bed"), "22\t150\t250\n"); // holds 200, not 100 + gzip(&p(&d, "denisova.bed.gz"), "chr22\t99\t100\n"); // 100 only, named chr22 + + // Control: without --mask the same inputs keep every call. + let (code, err) = table(&common, &[], &p(&d, "nomask.vcf")); + assert_eq!(code, 0, "{err}"); + let (nm_rows, nm) = vcf_rows(&p(&d, "nomask.vcf")); + assert_eq!(nm[&100]["Altai"], "2"); + assert_eq!(nm_rows.keys().copied().collect::>(), [100, 200, 500]); + + let masks = [ + format!("Altai={}", p(&d, "altai.bed")), + format!("Vindija={}", p(&d, "vindija.bed")), + format!("Denisova={}", p(&d, "denisova.bed.gz")), + ]; + let (code, err) = table( + &common, + &[ + "--mask", &masks[0], "--mask", &masks[1], "--mask", &masks[2], + ], + &p(&d, "masked.vcf"), + ); + assert_eq!(code, 0, "{err}"); + let (m_rows, mi) = vcf_rows(&p(&d, "masked.vcf")); + assert_eq!( + [&mi[&100]["Altai"], &mi[&100]["Denisova"]], + ["2", "1"], + "inside its mask a genome keeps its call; a mask is read with and without chr" + ); + assert_eq!( + [&mi[&100]["Vindija"], &nm[&100]["Vindija"]], + ["10", "0"], + "outside its mask a call is masked (10 for 0/0)" + ); + assert_eq!(m_rows.keys().copied().collect::>(), [100, 200, 500]); + assert_eq!( + mi[&200]["Altai"], "10", + "a row whose only call is masked stays" + ); + assert_eq!(m_rows[&500][4], "C"); + assert_eq!([&mi[&500]["Altai"], &mi[&500]["Vindija"]], ["1", "10"]); + assert_eq!( + [&mi[&200]["Vindija"], &mi[&500]["Denisova"]], + ["9", "9"], + "unknown stays 9" + ); + + // The excluded regions mask every call at a site inside one. + write(&p(&d, "excl.bed"), "22\t90\t120\tsegdup\n"); + let excl = p(&d, "excl.bed"); + let (code, err) = table(&common, &["--exclude", &excl], &p(&d, "excl.vcf")); + assert_eq!(code, 0, "{err}"); + let (ex_rows, ex) = vcf_rows(&p(&d, "excl.vcf")); + assert_eq!(codes(&ex, 100), ["12", "10", "11"]); + assert_eq!(codes(&nm, 100), ["2", "0", "1"]); + assert_eq!(ex_rows.keys().copied().collect::>(), [100, 200, 500]); + assert_eq!([&ex[&200]["Altai"], &ex[&500]["Altai"]], ["0", "1"]); + + write(&p(&d, "excl_other.bed"), "7\t90\t120\tsegdup\n"); + let other = p(&d, "excl_other.bed"); + let (code, err) = table(&common, &["--exclude", &other], &p(&d, "x.vcf")); + assert!(code == 1 && err.contains("no interval"), "{code} {err}"); + let (code, err) = table( + &common, + &["--mask", &masks[0], "--mask", &masks[0]], + &p(&d, "x.vcf"), + ); + assert!(code == 2 && err.contains("twice"), "{code} {err}"); + let nobody = format!("Nobody={}", p(&d, "altai.bed")); + let (code, err) = table(&common, &["--mask", &nobody], &p(&d, "x.vcf")); + assert_eq!(code, 2, "{err}"); + write(&p(&d, "other.bed"), "7\t50\t150\n"); + let elsewhere = format!("Altai={}", p(&d, "other.bed")); + let (code, err) = table(&common, &["--mask", &elsewhere], &p(&d, "x.vcf")); + assert!( + code == 1 && err.contains("no interval"), + "a mask with no interval on the chromosome stops: {code} {err}" + ); +} + +#[test] +fn strict_marks_rows_outside_the_mask_and_keeps_each_position_once() { + let d = tmp("strict"); + // Strict mask 1-based 101-300 on chr22. + let rows = [ + "chr22\t100\tA\tG\t2\t0\t9\t0.5\t0\t0\t0\t0\t0", + "chr22\t150\tA\tG\t1\t1\t1\t0.5\t0\t0\t0\t0\t0", + "chr22\t300\tG\tA\t0\t0\t0\t0.5\t0\t0\t0\t0\t0", + "chr22\t301\tC\tT\t0\t10\t12\t0.5\t0\t0\t0\t0\t0", + "chr22\t301\tC\tT\t2\t2\t2\t0.5\t0\t0\t0\t0\t0", + ]; + write(&p(&d, "lifted.tsv"), &(rows.join("\n") + "\n")); + write(&p(&d, "strict.bed"), "chr22\t100\t300\n"); + let strict = |mask: &str, rows: &str, out: &str| { + og(&[ + "strict", "--chrom", "22", "--strict", mask, "--rows", rows, "--out", out, + ]) + }; + let (code, err) = strict( + &p(&d, "strict.bed"), + &p(&d, "lifted.tsv"), + &p(&d, "marked.tsv"), + ); + assert_eq!(code, 0, "{err}"); + let got: Vec> = std::fs::read_to_string(p(&d, "marked.tsv")) + .unwrap() + .lines() + .map(|l| l.split('\t').map(str::to_owned).collect()) + .collect(); + assert_eq!( + got.iter().map(|c| c[1].as_str()).collect::>(), + ["100", "150", "300", "301"] + ); + let by = |pos: &str| -> Vec<&str> { + got.iter().find(|c| c[1] == pos).unwrap()[4..7] + .iter() + .map(String::as_str) + .collect() + }; + assert_eq!(by("100"), ["12", "10", "9"], "outside: 2, 0, unknown"); + assert_eq!(by("150"), ["1", "1", "1"], "inside: untouched"); + assert_eq!(by("300"), ["0", "0", "0"], "the last base inside too"); + assert_eq!( + by("301"), + ["10", "10", "12"], + "a masked call stays, the first of a duplicate wins" + ); + write( + &p(&d, "unsorted.tsv"), + &([rows[0], rows[3], rows[2]].join("\n") + "\n"), + ); + let (code, err) = strict( + &p(&d, "strict.bed"), + &p(&d, "unsorted.tsv"), + &p(&d, "marked3.tsv"), + ); + assert!(code == 2 && err.contains("sorted"), "{code} {err}"); + assert!(!d.join("marked3.tsv").exists()); + write(&p(&d, "strict_other.bed"), "chr21\t0\t10\n"); + let (code, err) = strict( + &p(&d, "strict_other.bed"), + &p(&d, "lifted.tsv"), + &p(&d, "marked2.tsv"), + ); + assert!(code != 0 && err.contains("no interval"), "{code} {err}"); +} + +/// A genotype file by position. +fn gt_rows(path: &str) -> BTreeMap> { + let mut rows = BTreeMap::new(); + for l in gunzip(path).lines().skip(1) { + let c: Vec = l.split('\t').map(str::to_owned).collect(); + let pos: u32 = c[1].parse().unwrap(); + assert!( + rows.insert(pos, c).is_none(), + "two genotype rows for position {pos} in {path}" + ); + } + rows +} + +const ROWS: [&str; 5] = [ + "chr22\t1000\tA\tG\t2\t2\t0\t0.5\t0.5\t0.5\t0.5\t0.5\t0.5", + "chr22\t2000\tC\tT\t0\t0\t0\t0.01\t0\t0\t0\t0.02\t0", + "chr22\t3000\tG\tA\t1\t1\t2\t0\t0\t0\t0\t0\t0", + "chr22\t4000\tT\tC\t2\t9\t0\t0.2\t0\t0\t0\t0\t0", + "chr22\t5000\tA\tC\t0\t0\t0\t0.2\t0\t0\t0\t0\t0", +]; + +/// The table of part 2 and the person's VCF. +fn genotype_inputs(d: &Path) -> String { + let tdir = p(d, "table"); + std::fs::create_dir_all(&tdir).unwrap(); + gzip( + &format!("{tdir}/chr22.tsv.gz"), + &format!("{TABLE_HEADER}\n{}\n", ROWS.join("\n")), + ); + gzip( + &format!("{tdir}/chr21.tsv.gz"), + &format!("{TABLE_HEADER}\nchr21\t1000\tA\tG\t2\t2\t0\t0.5\t0.5\t0.5\t0.5\t0.5\t0.5\n"), + ); + write( + &p(d, "s.vcf"), + &format!( + "{VCF_HEAD}chr22\t1000\t.\tA\tG\t50\tPASS\t.\tGT:GQ\t0/1:30\n\ + chr22\t3000\t.\tG\tA\t50\tRefCall\t.\tGT\t1/1\n\ + chr22\t5000\t.\tA\tT\t50\tPASS\t.\tGT\t1/1\n" + ), + ); + tdir +} + +#[test] +fn genotypes_code_your_calls_and_the_stand_ins() { + let d = tmp("genotypes"); + let tdir = genotype_inputs(&d); + // A reference block over 1-2500: 2000 was read; 4000 was not. + write( + &p(&d, "s.g.vcf"), + &format!("{VCF_HEAD}{}", ref_block("chr22", 1, 2500)), + ); + let (code, err) = og(&[ + "genotypes", + "--table-dir", + &tdir, + "--vcf", + &p(&d, "s.vcf"), + "--gvcf", + &p(&d, "s.g.vcf"), + "--pop", + "EUR", + "--pseudo", + "10", + "--out-dir", + &p(&d, "gt"), + ]); + assert_eq!(code, 0, "{err}"); + let text = gunzip(&p(&d, "gt/chr22.gt.gz")); + let head: Vec<&str> = text.lines().next().unwrap().split('\t').collect(); + assert_eq!( + head[..8], + [ + "chrom", "pos", "ref", "alt", "Altai", "Vindija", "Denisova", "SAMPLE" + ] + ); + assert_eq!(head.len(), 18, "ten stand-ins"); + let g = gt_rows(&p(&d, "gt/chr22.gt.gz")); + assert_eq!(g[&1000][0], "22", "an integer chromosome for IBDmix"); + assert_eq!(g[&1000][7], "1", "a heterozygous call"); + assert_eq!(g[&2000][7], "0", "no record inside a reference block"); + assert_eq!(g[&4000][7], "9", "no record outside every block"); + assert_eq!(g[&3000][7], "9", "a non-PASS record"); + assert!(!g.contains_key(&5000), "another ALT is left out"); + let sum = |pos: u32| -> u32 { g[&pos][8..].iter().map(|x| x.parse::().unwrap()).sum() }; + assert_eq!(sum(1000), 10, "0.5 of 20 alleles"); + assert_eq!(sum(2000), 0, "0.02 of 20 rounds to 0"); + assert!(g[&1000][4..].iter().all(|x| x.len() == 1)); + assert!( + !d.join("gt/chr21.gt.gz").exists(), + "a chromosome the VCF has no record on is not analysed" + ); + let spans = std::fs::read_to_string(p(&d, "gt/spans.tsv")).unwrap(); + let span: Vec<&str> = spans.lines().nth(1).unwrap().split('\t').collect(); + assert_eq!(span[..4], ["chr22", "1000", "2000", "2"]); + let samples = std::fs::read_to_string(p(&d, "gt/samples.txt")).unwrap(); + let samples: Vec<&str> = samples.split_whitespace().collect(); + assert_eq!((samples[0], samples.len()), ("SAMPLE", 11)); + assert_eq!( + std::fs::read_to_string(p(&d, "gt/chr22.Altai.mask.bed")).unwrap(), + "", + "an unmasked table writes empty masks" + ); + + write( + &p(&d, "two.vcf"), + "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tA\tB\n", + ); + let (code, err) = og(&[ + "genotypes", + "--table-dir", + &tdir, + "--vcf", + &p(&d, "two.vcf"), + "--out-dir", + &p(&d, "gt2"), + ]); + assert!(code != 0 && err.contains("one-sample"), "{code} {err}"); +} + +#[test] +fn a_split_vcf_keeps_each_snv_call() { + let d = tmp("split"); + let sdir = p(&d, "table_split"); + std::fs::create_dir_all(&sdir).unwrap(); + let rows: String = [ + (1000, "A\tG"), + (2000, "C\tT"), + (3000, "G\tA"), + (4000, "T\tC"), + (5000, "A\tG"), + (6000, "C\tT"), + (7000, "G\tA"), + (8000, "T\tC"), + (9000, "A\tC"), + ] + .iter() + .map(|(pos, ra)| format!("chr22\t{pos}\t{ra}\t0\t0\t0\t0.2\t0\t0\t0\t0\t0\n")) + .collect(); + gzip( + &format!("{sdir}/chr22.tsv.gz"), + &format!("{TABLE_HEADER}\n{rows}"), + ); + let recs = [ + "1000\tA\tAT\tPASS\t0/1", // an insertion, then the SNV + "1000\tA\tG\tPASS\t0/1", + "2000\tC\tT\tPASS\t1/1", // the SNV, then an insertion + "2000\tC\tCA\tPASS\t0/1", + "3000\tG\tA,GT\tPASS\t1/2", // a multi-allelic record, then the SNV + "3000\tG\tA\tPASS\t0/1", + "4000\tT\tC\tPASS\t0/1", // two SNVs, both carried + "4000\tT\tG\tPASS\t0/1", + "4000\tT\tC\tPASS\t0/1", + "5000\tA\tT\tPASS\t0/0", // a split 0/1 at A>G,T: 0/0, then 0/1 + "5000\tA\tG\tPASS\t0/1", + "6000\tC\tT\tPASS\t0/1", // 0/1, then the 0/0 half + "6000\tC\tG\tPASS\t0/0", + "7000\tG\tA\tLowQual\t0/1", // filtered, then usable + "7000\tG\tA\tPASS\t1/1", + "8000\tT\tA\tPASS\t0/0", // two 0/0 halves: the reference + "8000\tT\tG\tPASS\t0/0", + ]; + let body: String = recs + .iter() + .map(|r| { + let c: Vec<&str> = r.split('\t').collect(); + format!( + "chr22\t{}\t.\t{}\t{}\t50\t{}\t.\tGT\t{}\n", + c[0], c[1], c[2], c[3], c[4] + ) + }) + .collect(); + write(&p(&d, "split.vcf"), &format!("{VCF_HEAD}{body}")); + let (code, err) = og(&[ + "genotypes", + "--table-dir", + &sdir, + "--vcf", + &p(&d, "split.vcf"), + "--pseudo", + "10", + "--out-dir", + &p(&d, "gt3"), + ]); + assert_eq!(code, 0, "{err}"); + let g = gt_rows(&p(&d, "gt3/chr22.gt.gz")); + let code = |pos: u32| g.get(&pos).map_or("left out", |c| c[7].as_str()); + assert_eq!(code(1000), "1", "an insertion before the SNV"); + assert_eq!(code(2000), "2", "an insertion after the SNV"); + assert_eq!(code(3000), "1", "a multi-allelic record beside the SNV"); + assert_eq!(code(4000), "9", "two carried SNVs at one position"); + assert_eq!(code(5000), "1", "a 0/0 half before the carried SNV"); + assert_eq!(code(6000), "1", "a 0/0 half after the carried SNV"); + assert_eq!(code(7000), "2", "a usable record wins over a filtered one"); + assert_eq!(code(8000), "0", "two 0/0 halves are the reference"); +} + +#[test] +fn masked_calls_go_to_ibdmix_with_a_mask_per_genome() { + let d = tmp("masked"); + let mdir = p(&d, "table_masked"); + std::fs::create_dir_all(&mdir).unwrap(); + let rows: String = [ + (100, "A\tG", "12\t0\t9"), + (200, "C\tT", "10\t11\t0"), + (300, "G\tA", "2\t0\t0"), + (400, "T\tC", "10\t0\t0"), + (500, "A\tC", "0\t0\t0"), + (600, "C\tT", "11\t0\t0"), + ] + .iter() + .map(|(pos, ra, c)| format!("chr22\t{pos}\t{ra}\t{c}\t0.2\t0\t0\t0\t0\t0\n")) + .collect(); + gzip( + &format!("{mdir}/chr22.tsv.gz"), + &format!("{TABLE_HEADER}\n{rows}"), + ); + // Another ALT: row 500 is left out. + write( + &p(&d, "m.vcf"), + &format!("{VCF_HEAD}chr22\t500\t.\tA\tT\t50\tPASS\t.\tGT\t0/1\n"), + ); + write( + &p(&d, "m.g.vcf"), + &format!("{VCF_HEAD}{}", ref_block("chr22", 1, 1000)), + ); + let (code, err) = og(&[ + "genotypes", + "--table-dir", + &mdir, + "--vcf", + &p(&d, "m.vcf"), + "--gvcf", + &p(&d, "m.g.vcf"), + "--pseudo", + "10", + "--out-dir", + &p(&d, "gtm"), + ]); + assert_eq!(code, 0, "{err}"); + let g = gt_rows(&p(&d, "gtm/chr22.gt.gz")); + let calls = |pos: u32| g[&pos][4..7].to_vec(); + assert_eq!(calls(100), ["2", "0", "9"], "12 is 2"); + assert_eq!(calls(200), ["0", "1", "0"], "10 and 11 are 0 and 1"); + assert_eq!(calls(600), ["1", "0", "0"]); + let mask = + |a: &str| std::fs::read_to_string(p(&d, &format!("gtm/chr22.{a}.mask.bed"))).unwrap(); + assert_eq!( + mask("Altai"), + "22\t99\t200\n22\t399\t600\n", + "the left-out row 500 does not close the run" + ); + assert_eq!(mask("Vindija"), "22\t199\t200\n"); + assert_eq!(mask("Denisova"), "", "written and empty"); + + let bdir = p(&d, "table_bad"); + std::fs::create_dir_all(&bdir).unwrap(); + gzip( + &format!("{bdir}/chr22.tsv.gz"), + &format!("{TABLE_HEADER}\nchr22\t100\tA\tG\t7\t0\t0\t0.2\t0\t0\t0\t0\t0\n"), + ); + let (code, err) = og(&[ + "genotypes", + "--table-dir", + &bdir, + "--vcf", + &p(&d, "m.vcf"), + "--out-dir", + &p(&d, "gtb"), + ]); + assert!(code == 1 && err.contains("code '7'"), "{code} {err}"); +} + +#[test] +fn unsorted_input_stops_and_keeps_the_previous_outputs() { + let d = tmp("order"); + let tdir = genotype_inputs(&d); + for (label, records) in [ + ( + "within-table", + [(3000, "G", "A", "GT"), (1000, "A", "G", "GT")], + ), + ( + "past-table", + [(9000, "A", "G", "GT"), (8000, "A", "G", "GT")], + ), + ( + "ignored-call", + [(3000, "G", "A", "GT"), (1000, "A", ".", "GQ")], + ), + ] { + let bad = p(&d, &format!("{label}.vcf")); + let body: String = records + .iter() + .map(|(pos, r, a, f)| format!("chr22\t{pos}\t.\t{r}\t{a}\t.\tPASS\t.\t{f}\t0/1\n")) + .collect(); + write(&bad, &format!("{VCF_HEAD}{body}")); + let out = p(&d, &format!("order-{label}")); + std::fs::create_dir_all(&out).unwrap(); + let protected: Vec = std::iter::once("chr22.gt.gz".to_owned()) + .chain(ARCHAICS.iter().map(|g| format!("chr22.{g}.mask.bed"))) + .collect(); + for name in &protected { + write(&format!("{out}/{name}"), "previous output\n"); + } + let (code, err) = og(&[ + "genotypes", + "--table-dir", + &tdir, + "--vcf", + &bad, + "--out-dir", + &out, + ]); + assert!( + code == 2 && err.contains("not sorted"), + "{label}: {code} {err}" + ); + for name in &protected { + assert_eq!( + std::fs::read(format!("{out}/{name}")).unwrap(), + b"previous output\n", + "{label} keeps {name}" + ); + } + } + let udir = p(&d, "table_unordered"); + std::fs::create_dir_all(&udir).unwrap(); + gzip( + &format!("{udir}/chr22.tsv.gz"), + &format!( + "{TABLE_HEADER}\n{}\n", + [ROWS[1], ROWS[0], ROWS[2], ROWS[3], ROWS[4]].join("\n") + ), + ); + let out = p(&d, "gt_unordered"); + let (code, err) = og(&[ + "genotypes", + "--table-dir", + &udir, + "--vcf", + &p(&d, "s.vcf"), + "--out-dir", + &out, + ]); + assert!(code == 2 && err.contains("not sorted"), "{code} {err}"); + let left: Vec = match std::fs::read_dir(&out) { + Ok(r) => r + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect(), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(), + Err(e) => panic!("cannot read {out}: {e}"), + }; + assert!( + !left + .iter() + .any(|n| n.ends_with(".gt.gz") || n.ends_with(".mask.bed")), + "no partial genotype or mask: {left:?}" + ); +} + +/// The archaic section og summary collect writes for S1 in `dir`. +fn archaic_section(dir: &Path) -> Value { + let input = og_summary::collect::Input { + sample: "S1".to_owned(), + sample_dir: dir.display().to_string(), + declared_sex: None, + assets: repo().join("assets").display().to_string(), + group: og_summary::collect::Group::All, + tables: repo().join("bin").display().to_string(), + }; + let v = og_summary::collect::post::collect(&input).unwrap(); + let get = |v: &Value, k: &str| pycompat::value::get(v, k).unwrap().unwrap().clone(); + get(&get(&v, "sections"), "archaic") +} + +/// Both reports of the summary og summary collect writes for S1. +fn reports(dir: &Path) -> (String, String) { + let input = og_summary::collect::Input { + sample: "S1".to_owned(), + sample_dir: dir.display().to_string(), + declared_sex: None, + assets: repo().join("assets").display().to_string(), + group: og_summary::collect::Group::All, + tables: repo().join("bin").display().to_string(), + }; + let v = og_summary::collect::post::collect(&input).unwrap(); + let s = og_render::model::Summary::parse( + &pycompat::json::dumps(&v, &pycompat::json::Dump::default()).unwrap(), + ) + .unwrap(); + ( + og_render::text::report(&s).unwrap(), + og_render::html::report(&s).unwrap(), + ) +} + +fn field<'a>(v: &'a Value, k: &str) -> &'a Value { + pycompat::value::get(v, k).unwrap().unwrap() +} + +#[test] +fn the_report_merges_segments_and_both_reports_read_it() { + let d = tmp("report"); + let idir = p(&d, "ibd"); + std::fs::create_dir_all(&idir).unwrap(); + let head = "ID\tchrom\tstart\tend\tslod\n"; + write( + &format!("{idir}/chr22.Altai.txt"), + &format!( + "{head}SAMPLE\t22\t1000000\t1300000\t40\nP001\t22\t5000000\t9000000\t99\n\ + SAMPLE\t22\t2000000\t2040000\t30\nSAMPLE\t22\t3000000\t3100000\t3.5\n" + ), + ); + write( + &format!("{idir}/chr22.Vindija.txt"), + &format!("{head}SAMPLE\t22\t1200000\t1500000\t50\n"), + ); + write( + &format!("{idir}/chr22.Denisova.txt"), + &format!("{head}SAMPLE\t22\t1400000\t1450000\t12\nSAMPLE\t22\t6000000\t6200000\t20\n"), + ); + write( + &p(&d, "spans.tsv"), + "chrom\tfirst\tlast\tsites\nchr22\t1\t10000000\t5000\n", + ); + write( + &p(&d, "genes.gtf"), + "chr22\tHAVANA\tgene\t1100000\t1110000\t.\t+\t.\tgene_id \"G1\"; gene_type \"protein_coding\"; gene_name \"GENEA\";\n\ + chr22\tHAVANA\tgene\t1120000\t1130000\t.\t+\t.\tgene_id \"G2\"; gene_type \"lncRNA\"; gene_name \"LNC1\";\n\ + chr22\tHAVANA\tgene\t6100000\t6300000\t.\t-\t.\tgene_id \"G3\"; gene_type \"protein_coding\"; gene_name \"GENEB\";\n", + ); + let (code, err) = og(&[ + "report", + "--sample", + "S1", + "--ibdmix-dir", + &idir, + "--spans", + &p(&d, "spans.tsv"), + "--genes", + &p(&d, "genes.gtf"), + "--out-dir", + &p(&d, "out"), + ]); + assert_eq!(code, 0, "{err}"); + let segs = std::fs::read_to_string(p(&d, "out/S1_archaic_segments.tsv")).unwrap(); + let segs: Vec> = segs.lines().map(|l| l.split('\t').collect()).collect(); + assert_eq!( + segs[0], + [ + "chrom", + "start", + "end", + "length_bp", + "ancestry", + "matched", + "lod", + "genes" + ] + ); + let body = &segs[1..]; + assert_eq!(body.len(), 2, "{body:?}"); + assert_eq!( + body[0][..6], + [ + "chr22", + "1000000", + "1500000", + "500000", + "Neanderthal", + "Altai,Vindija,Denisova" + ], + "Altai 1.0-1.3 and Vindija 1.2-1.5 Mb merge" + ); + assert_eq!(body[0][6], "50.0", "the higher LOD"); + assert_eq!( + body[1][..5], + ["chr22", "6000000", "6200000", "200000", "Denisovan"], + "a Denisova stretch inside a Neanderthal one is not Denisovan; one apart is" + ); + assert_eq!(body[0][7], "GENEA", "protein-coding genes only"); + let summ: BTreeMap = std::fs::read_to_string(p(&d, "out/S1_archaic.tsv")) + .unwrap() + .lines() + .skip(1) + .map(|l| { + let (k, v) = l.split_once('\t').unwrap(); + (k.to_owned(), v.to_owned()) + }) + .collect(); + assert_eq!(summ["neanderthal_percent"], "5.00"); + assert_eq!(summ["denisovan_percent"], "2.00"); + assert_eq!( + [&summ["neanderthal_segments"], &summ["denisovan_segments"]], + ["1", "1"] + ); + assert!(summ["longest_1"].starts_with("chr22:1000000-1500000 Neanderthal")); + assert!(summ["longest_1"].contains("GENEA")); + let txt = std::fs::read_to_string(p(&d, "out/S1_archaic.txt")).unwrap(); + assert!(txt.contains("An estimate, not a measurement."), "{txt}"); + let (code, err) = og(&[ + "report", + "--sample", + "S1", + "--ibdmix-dir", + &p(&d, "nothing"), + "--spans", + &p(&d, "spans.tsv"), + "--out-dir", + &p(&d, "out2"), + ]); + assert!( + code != 0 && err.contains("no IBDmix output"), + "{code} {err}" + ); + + // The summary and the report lines read it. + let sd = d.join("sample"); + std::fs::create_dir_all(sd.join("archaic")).unwrap(); + let tsv = sd.join("archaic/S1_archaic.tsv"); + std::fs::copy(p(&d, "out/S1_archaic.tsv"), &tsv).unwrap(); + let sec = archaic_section(&sd); + assert_eq!(field(&sec, "state"), &Value::Str("ok".into())); + assert_eq!(field(&sec, "step"), &Value::Str("43".into())); + let data = field(&sec, "data"); + assert_eq!( + field(data, "neanderthal_percent"), + &Value::Str("5.00".into()) + ); + assert_eq!(field(data, "denisovan_segments"), &Value::Int(1)); + let Value::Array(longest) = field(data, "longest") else { + panic!("longest is not a list") + }; + assert_eq!(longest.len(), 2); + let lines = og_render::sections::archaic::lines(data).unwrap(); + assert!( + lines[0].starts_with("Neanderthal: 5.00% (0.5 Mb in 1 segments)"), + "{lines:?}" + ); + assert!(lines[1].starts_with("Denisovan: 2.00%"), "{lines:?}"); + let (txt, page) = reports(&sd); + let shares = "Neanderthal: 5.00% (0.5 Mb in 1 segments)"; + assert!(txt.contains(shares), "the text report: {txt}"); + assert!( + page.contains("

    Archaic Ancestry (Neanderthal, Denisovan)

    ") + && page.contains(shares), + "the HTML report" + ); + for damaged in [ + "key\tvalue\nneanderthal_percent\tlots\n", + "key\tvalue\nneanderthal_segments\n", + ] { + std::fs::write(&tsv, damaged).unwrap(); + assert_eq!( + field(&archaic_section(&sd), "state"), + &Value::Str("unreadable".into()), + "{damaged:?}" + ); + } + let nowhere = archaic_section(&d.join("nowhere")); + assert_eq!(field(&nowhere, "state"), &Value::Str("missing".into())); + assert_eq!(field(&nowhere, "data"), &Value::Object(Default::default())); +} + +/// Python's `random.Random(seed)` for a small seed: MT19937 seeded by +/// `init_by_array([seed])`, and `random()` from two draws. The IBDmix +/// fixture below is the one the Python test drew. +struct PyRandom { + mt: [u32; 624], + i: usize, +} + +impl PyRandom { + fn new(seed: u32) -> PyRandom { + let mut mt = [0u32; 624]; + mt[0] = 19_650_218; + for i in 1..624 { + mt[i] = 1_812_433_253u32 + .wrapping_mul(mt[i - 1] ^ (mt[i - 1] >> 30)) + .wrapping_add(i as u32); + } + let key = [seed]; + let (mut i, mut j) = (1usize, 0usize); + for _ in 0..624 { + mt[i] = (mt[i] ^ (mt[i - 1] ^ (mt[i - 1] >> 30)).wrapping_mul(1_664_525)) + .wrapping_add(key[j]) + .wrapping_add(j as u32); + i += 1; + j += 1; + if i >= 624 { + mt[0] = mt[623]; + i = 1; + } + if j >= key.len() { + j = 0; + } + } + for _ in 0..623 { + mt[i] = (mt[i] ^ (mt[i - 1] ^ (mt[i - 1] >> 30)).wrapping_mul(1_566_083_941)) + .wrapping_sub(i as u32); + i += 1; + if i >= 624 { + mt[0] = mt[623]; + i = 1; + } + } + mt[0] = 0x8000_0000; + PyRandom { mt, i: 624 } + } + + fn next_u32(&mut self) -> u32 { + if self.i >= 624 { + for k in 0..624 { + let y = (self.mt[k] & 0x8000_0000) | (self.mt[(k + 1) % 624] & 0x7fff_ffff); + let mut v = self.mt[(k + 397) % 624] ^ (y >> 1); + if y & 1 == 1 { + v ^= 0x9908_b0df; + } + self.mt[k] = v; + } + self.i = 0; + } + let mut y = self.mt[self.i]; + self.i += 1; + y ^= y >> 11; + y ^= (y << 7) & 0x9d2c_5680; + y ^= (y << 15) & 0xefc6_0000; + y ^ (y >> 18) + } + + fn random(&mut self) -> f64 { + let a = f64::from(self.next_u32() >> 5); + let b = f64::from(self.next_u32() >> 6); + (a * 67_108_864.0 + b) / 9_007_199_254_740_992.0 + } + + fn uniform(&mut self, a: f64, b: f64) -> f64 { + a + (b - a) * self.random() + } +} + +#[test] +fn the_fixture_generator_draws_what_python_drew() { + // Python 3: random.Random(11), three random() then uniform(0.01, 0.04). + let mut r = PyRandom::new(11); + let got = [r.random(), r.random(), r.random(), r.uniform(0.01, 0.04)]; + assert_eq!( + got, + [ + 0.452_379_553_509_818_6, + 0.559_772_386_080_496, + 0.924_210_584_023_729_4, + 0.023_969_502_102_993_2 + ] + ); +} + +#[cfg(target_arch = "x86_64")] +mod ibdmix { + //! Part 4: the chain with IBDmix itself. + + use super::*; + use std::process::Command; + + const NEA: (u32, u32) = (2_000_000, 2_600_000); + const DEN: (u32, u32) = (3_200_000, 3_600_000); + const MID: (u32, u32) = (2_200_000, 2_400_000); + const MASKED: u32 = 10; + + /// Whether the docker tests run here: always in CI. + fn docker_tests(test: &str) -> bool { + let on = std::env::var_os("CI").is_some_and(|v| !v.is_empty()) + || std::env::var("OG_DOCKER_TESTS").as_deref() == Ok("1"); + if !on { + eprintln!( + "skip: {test} runs IBDmix in docker; set OG_DOCKER_TESTS=1 (CI always runs it)" + ); + } + on + } + + fn image() -> String { + let text = std::fs::read_to_string(repo().join("versions.env")).unwrap(); + let line = text + .lines() + .find(|l| l.starts_with("IBDMIX_IMAGE=")) + .expect("IBDMIX_IMAGE in versions.env"); + let v = &line["IBDMIX_IMAGE=".len()..]; + v.split('#') + .next() + .unwrap() + .trim() + .trim_matches('"') + .to_owned() + } + + fn id(flag: &str) -> String { + let o = Command::new("id").arg(flag).output().unwrap(); + String::from_utf8(o.stdout).unwrap().trim().to_owned() + } + + fn open_dir(name: &str) -> Tmp { + let d = tmp(name); + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&d, std::fs::Permissions::from_mode(0o777)).unwrap(); + d + } + + /// A 4 Mb chr22 table and one person, drawn as the Python test drew + /// them. A third of the sites carry an allele the Neanderthals share + /// and modern people rarely do, a third one only the Denisovan carries, + /// a third are common. Inside a planted stretch one copy carries the + /// archaic allele. `mid`: the middle of the planted Neanderthal stretch + /// holds sites the person contradicts, with the Neanderthals' calls + /// written masked (10 plus the call) or unknown (9). + fn synthetic(d: &Path, plant: bool, mid: Option<&str>) { + let mut rnd = PyRandom::new(11); + std::fs::create_dir_all(d.join("table")).unwrap(); + let (mut rows, mut calls) = (Vec::new(), Vec::new()); + for (i, pos) in (1_000_000u32..5_000_000).step_by(1000).enumerate() { + let (f, al, vi, de) = match i % 3 { + 0 => (rnd.uniform(0.01, 0.04), 2, 2, 0), + 1 => (rnd.uniform(0.01, 0.04), 0, 0, 2), + _ => { + let f = rnd.uniform(0.1, 0.9); + let n = u32::from(rnd.random() < f) + u32::from(rnd.random() < f); + (f, n, n, n) + } + }; + let (mut ta, mut tv) = (al, vi); + let mut g = [rnd.random() < f, rnd.random() < f]; + if plant && NEA.0 <= pos && pos < NEA.1 { + g[0] = al == 2; + } + if let Some(m) = mid + && MID.0 <= pos + && pos < MID.1 + { + if i.is_multiple_of(3) { + g = [false, false]; + } + (ta, tv) = if m == "masked" { + (al + MASKED, vi + MASKED) + } else { + (9, 9) + }; + } + let fs = format!("{f:.4}"); + rows.push(format!( + "chr22\t{pos}\tA\tG\t{ta}\t{tv}\t{de}\t{}", + [fs.as_str(); 6].join("\t") + )); + if plant && DEN.0 <= pos && pos < DEN.1 { + g[0] = de == 2; + } + if g.iter().any(|x| *x) { + let gt = if g.iter().all(|x| *x) { "1/1" } else { "0/1" }; + calls.push(format!("chr22\t{pos}\t.\tA\tG\t50\tPASS\t.\tGT\t{gt}")); + } + } + gzip( + &p(d, "table/chr22.tsv.gz"), + &format!("{TABLE_HEADER}\n{}\n", rows.join("\n")), + ); + write(&p(d, "s.vcf"), &format!("{VCF_HEAD}{}\n", calls.join("\n"))); + } + + /// genotypes, IBDmix with the flags of scripts/43-archaic.sh, report; + /// the segments as (start, end, length, ancestry). + fn chain(d: &Path, gvcf: Option) -> Vec<(u32, u32, u32, String)> { + let gvcf = + gvcf.unwrap_or_else(|| format!("{VCF_HEAD}{}", ref_block("chr22", 1, 6_000_000))); + write(&p(d, "s.g.vcf"), &gvcf); + let (code, err) = og(&[ + "genotypes", + "--table-dir", + &p(d, "table"), + "--vcf", + &p(d, "s.vcf"), + "--gvcf", + &p(d, "s.g.vcf"), + "--out-dir", + &p(d, "gt"), + ]); + assert_eq!(code, 0, "{err}"); + std::fs::create_dir_all(d.join("ibd")).unwrap(); + let sh = "set -e; gzip -dc /w/gt/chr22.gt.gz > /w/gt/chr22.gt; for a in Altai Vindija Denisova; do \ + ibdmix -g /w/gt/chr22.gt -s /w/gt/samples.txt -n $a -r /w/gt/chr22.$a.mask.bed -o /w/ibd/chr22.$a.txt \ + -d 4 -m 1 -a 0.01 -e 0.002 -c 2; done"; + let o = Command::new("docker") + .args(["run", "--rm", "--network", "none", "-u"]) + .arg(format!("{}:{}", id("-u"), id("-g"))) + .arg("-v") + .arg(format!("{}:/w", d.display())) + .arg(image()) + .args(["sh", "-c", sh]) + .output() + .unwrap_or_else(|e| panic!("docker: {e}")); + assert!( + o.status.success(), + "IBDmix: {}{}", + String::from_utf8_lossy(&o.stderr), + String::from_utf8_lossy(&o.stdout) + ); + let (code, err) = og(&[ + "report", + "--sample", + "S", + "--ibdmix-dir", + &p(d, "ibd"), + "--spans", + &p(d, "gt/spans.tsv"), + "--out-dir", + &p(d, "out"), + ]); + assert_eq!(code, 0, "{err}"); + let body: Vec<(u32, u32, u32, String)> = + std::fs::read_to_string(p(d, "out/S_archaic_segments.tsv")) + .unwrap() + .lines() + .skip(1) + .map(|l| { + let c: Vec<&str> = l.split('\t').collect(); + ( + c[1].parse().unwrap(), + c[2].parse().unwrap(), + c[3].parse().unwrap(), + c[4].to_owned(), + ) + }) + .collect(); + eprintln!("segments in {}: {body:?}", d.display()); + body + } + + fn cover(segs: &[(u32, u32, u32, String)], kind: &str, (lo, hi): (u32, u32)) -> u32 { + segs.iter() + .filter(|s| s.3 == kind) + .map(|s| s.1.min(hi).saturating_sub(s.0.max(lo))) + .sum() + } + + #[test] + fn planted_stretches_come_out_as_one_segment_each() { + if !docker_tests("planted_stretches_come_out_as_one_segment_each") { + return; + } + let d = open_dir("real_planted"); + synthetic(&d, true, None); + let body = chain(&d, None); + let nea: Vec<_> = body + .iter() + .filter(|s| s.3 == "Neanderthal") + .cloned() + .collect(); + let den: Vec<_> = body + .iter() + .filter(|s| s.3 == "Denisovan") + .cloned() + .collect(); + assert_eq!(nea.len(), 1, "{body:?}"); + assert!( + cover(&nea, "Neanderthal", NEA) * 10 >= 9 * (NEA.1 - NEA.0), + "{nea:?}" + ); + assert!( + nea[0].2 <= NEA.1 - NEA.0 + 100_000, + "no more than 100 kb beyond it" + ); + assert_eq!(den.len(), 1, "{body:?}"); + assert!( + cover(&den, "Denisovan", DEN) * 10 >= 9 * (DEN.1 - DEN.0), + "{den:?}" + ); + + let d = open_dir("real_control"); + synthetic(&d, false, None); + assert!(chain(&d, None).is_empty(), "without the planted stretches"); + } + + #[test] + fn masked_mismatches_stop_a_segment_that_unknown_calls_let_through() { + if !docker_tests("masked_mismatches_stop_a_segment_that_unknown_calls_let_through") { + return; + } + let span = MID.1 - MID.0; + let d = open_dir("real_mid_unknown"); + synthetic(&d, true, Some("unknown")); + let unknown = cover(&chain(&d, None), "Neanderthal", MID); + assert!( + unknown * 10 >= 9 * span, + "control: written as unknown the segment runs through ({unknown})" + ); + let d = open_dir("real_mid_masked"); + synthetic(&d, true, Some("masked")); + let masked = cover(&chain(&d, None), "Neanderthal", MID); + assert!( + masked * 10 <= span, + "written as masked no segment covers the middle ({masked})" + ); + } + + #[test] + fn a_stretch_with_no_call_needs_a_reference_block() { + if !docker_tests("a_stretch_with_no_call_needs_a_reference_block") { + return; + } + let (lo, hi) = (200_000u32, 349_000u32); + let mut got = Vec::new(); + for covered in [true, false] { + let d = open_dir(&format!("real_gap_{covered}")); + std::fs::create_dir_all(d.join("table")).unwrap(); + let rows: String = (lo..=hi) + .step_by(1000) + .map(|pos| format!("chr22\t{pos}\tC\tT\t0\t0\t0\t{}\n", ["0.9"; 6].join("\t"))) + .collect(); + gzip( + &p(&d, "table/chr22.tsv.gz"), + &format!("{TABLE_HEADER}\n{rows}"), + ); + write( + &p(&d, "s.vcf"), + &format!("{VCF_HEAD}chr22\t10000000\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\n"), + ); + // Without the block the person has one read site left, at the start. + let block = if covered { + ref_block("chr22", 1, 400_000) + } else { + ref_block("chr22", 1, lo) + }; + let body = chain(&d, Some(format!("{VCF_HEAD}{block}"))); + got.push( + body.iter() + .filter(|s| s.3 == "Neanderthal") + .map(|s| s.2) + .sum::(), + ); + } + assert!( + got[0] >= 100_000, + "control: a stretch read as the reference gives a segment ({got:?})" + ); + assert_eq!(got[1], 0, "no call and no reference block: no segment"); + } +} diff --git a/rust/crates/og-render/tests/ancestry_fine.rs b/rust/crates/og-render/tests/ancestry_fine.rs new file mode 100644 index 000000000..6387423ec --- /dev/null +++ b/rust/crates/og-render/tests/ancestry_fine.rs @@ -0,0 +1,194 @@ +//! Both reports of step 26b's table (`og ancestry-fine report` writes it): +//! the groups with their bootstrap range, the group the model leaves out, +//! the chromosome RFMix failed on, the painted shares and the painting. +//! It replaces the render part of `tests/test_ancestry_fine_report.py`; +//! og-ancestry's own tests check the table the report command writes. + +use std::path::{Path, PathBuf}; + +use og_render::model::Summary; +use og_render::{html, text}; +use pycompat::json::{Dump, dumps}; + +fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +/// The table `og ancestry-fine report` wrote for the Python test's inputs. +const TABLE: &str = "level\tcode\tgroup\tfraction\tse\tlow\thigh +continent\tWEA\tWest Eurasia (Europe, Middle East, North Africa)\t0.9000\t0.0332\t0.8025\t0.8975 +continent\tEAS\tEast Asia\t0.0500\t0.0000\t0.0500\t0.0500 +continent\tAFR\tAfrica\t0.0500\t0.0332\t0.0525\t0.1475 +region\tIBE\tIberia\t0.6000\t.\t.\t. +region\tITA\tItaly\t0.4000\t.\t.\t. +painting\tWEA\tWest Eurasia (Europe, Middle East, North Africa)\t0.7000\t.\t.\t. +painting\tAFR\tAfrica\t0.3000\t.\t.\t. +left_out\tOCE\tOceania (continental group)\t.\t.\t.\t. +left_out\tOCE\tOceania (regional group)\t.\t.\t.\t. +not_painted\tchr19\tRFMix failed\t.\t.\t.\t. +"; + +/// A scratch folder that is removed when the test ends, also when it fails. +struct Tmp(PathBuf); + +impl std::ops::Deref for Tmp { + type Target = Path; + fn deref(&self) -> &Path { + &self.0 + } +} + +impl AsRef for Tmp { + fn as_ref(&self) -> &Path { + &self.0 + } +} + +impl AsRef for Tmp { + fn as_ref(&self) -> &std::ffi::OsStr { + self.0.as_os_str() + } +} + +impl Tmp { + /// A new folder that only this test owns. `create_dir` fails when the + /// path exists, so a run never shares or deletes another run's folder. + fn new(kind: &str, tag: &str) -> Tmp { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let d = std::env::temp_dir().join(format!( + "og-ancestry-{kind}-{}-{tag}-{nanos}", + std::process::id() + )); + std::fs::create_dir(&d).unwrap(); + Tmp(d) + } +} + +impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +fn sample(tag: &str, painting: bool) -> Tmp { + let d = Tmp::new("fine", tag); + std::fs::create_dir(d.join("ancestry_fine")).unwrap(); + std::fs::write(d.join("ancestry_fine/S_ancestry_fine.tsv"), TABLE).unwrap(); + if painting { + std::fs::write( + d.join("ancestry_fine/S_painting.png"), + b"\x89PNG\r\n\x1a\nnot a real picture", + ) + .unwrap(); + } + d +} + +fn reports(d: &Path) -> (String, String) { + reports_with(d, |s| s) +} + +/// Both reports of the summary og summary collect writes, after `edit`. +fn reports_with(d: &Path, edit: impl Fn(String) -> String) -> (String, String) { + let input = og_summary::collect::Input { + sample: "S".to_owned(), + sample_dir: d.display().to_string(), + declared_sex: None, + assets: repo().join("assets").display().to_string(), + group: og_summary::collect::Group::All, + tables: repo().join("bin").display().to_string(), + }; + let summary = dumps( + &og_summary::collect::post::collect(&input).unwrap(), + &Dump::default(), + ) + .unwrap(); + let s = Summary::parse(&edit(summary)).unwrap(); + (text::report(&s).unwrap(), html::report(&s).unwrap()) +} + +#[test] +fn both_reports_name_the_group_left_out_and_the_chromosome_not_painted() { + let (txt, page) = reports(&sample("full", true)); + for doc in [&txt, &page] { + assert!( + doc.contains( + "Left out of the model: Oceania (continental and regional group). The panel has \ + fewer than 10" + ), + "{doc}" + ); + assert!(doc.contains("not painted: RFMix failed on chr19"), "{doc}"); + } + assert!(txt.contains("Fine-Grained Ancestry (ADMIXTURE, RFMix)")); + assert!( + txt.contains(" West Eurasia (Europe, Middle East, North Africa) 90% (80% to 90%)"), + "{txt}" + ); + assert!( + txt.contains( + " Chromosome painting, share of the painted genome: West Eurasia (Europe, Middle \ + East, North Africa) 70%, Africa 30%; not painted: RFMix failed on chr19" + ), + "{txt}" + ); + assert!(page.contains("

    Fine-Grained Ancestry

    ")); + assert!(page.contains( + "West Eurasia (Europe, Middle East, North Africa)90%80% to \ + 90%" + )); + assert!( + page.contains("Iberia60%."), + "a level without bootstrap replicates has no range" + ); + assert!( + page.contains("src=\"data:image/png;base64,iVBORw0KGg"), + "the painting" + ); + assert!( + page.contains("■ not painted

    "), + "its legend" + ); + let [r, g, b] = og_ancestry::COLOURS + .iter() + .find(|(code, _)| *code == "WEA") + .unwrap() + .1; + let entry = format!( + "■ West Eurasia (Europe, \ + Middle East, North Africa)" + ); + assert!( + page.contains(&entry), + "a group of the legend, in its colour" + ); +} + +#[test] +fn a_legend_colour_that_is_not_a_hex_colour_is_shown_grey() { + let d = sample("colour", true); + let (_, page) = reports_with(&d, |s| { + let at = s.find("\"colour\": \"#").expect("a legend colour"); + let start = at + "\"colour\": \"".len(); + format!("{}red;display:none{}", &s[..start], &s[start + 7..]) + }); + assert!(!page.contains("display:none"), "a planted CSS declaration"); + assert!( + page.contains("■"), + "the fixed grey" + ); +} + +#[test] +fn without_a_picture_the_card_has_no_image() { + let (_, page) = reports(&sample("nopng", false)); + assert!(page.contains("

    Fine-Grained Ancestry

    ")); + assert!(!page.contains("data:image/png;base64,")); + assert!( + !page.contains("not painted

    "), + "no legend without the painting" + ); +} diff --git a/rust/crates/og-render/tests/ancestry_haplogroups.rs b/rust/crates/og-render/tests/ancestry_haplogroups.rs new file mode 100644 index 000000000..79419f488 --- /dev/null +++ b/rust/crates/og-render/tests/ancestry_haplogroups.rs @@ -0,0 +1,271 @@ +//! Both reports point to where a haplogroup can be explored further (steps +//! 12 and 37). It replaces `tests/test_haplogroup_explore.py`. +//! +//! Each case writes the haplogroup files of one made-up sample S into an +//! output folder, collects the summary and renders both reports: +//! +//! 1. a male sample (Yleaf table with R-M269, haplogrep3 with H5a7): the Y +//! and the mtDNA links are built from those names, in the text and the +//! HTML report, with the note that nothing is uploaded; +//! 2. Yleaf's paragroup form R-M269*(xR-L23): the links use R-M269; +//! 3. a female sample (no Yleaf table): no Y links; the mtDNA links stay; +//! 4. no call (Hg NA, haplogroup ''): no links at all; +//! 5. a branch name with characters a URL cannot carry raw is encoded; +//! 6. an address that would close the link's attribute and open a tag is +//! escaped in the HTML report, so the page never runs it. +//! +//! The branch names are the fixture's and public examples, not anyone's +//! result. + +use std::path::{Path, PathBuf}; + +use og_render::model::Summary; +use og_render::{html, text}; +use pycompat::json::{Dump, dumps}; + +const Y_HEAD: &str = + "Sample_name\tHg\tHg_marker\tTotal_reads\tValid_markers\tQC-score\tQC-1\tQC-2\tQC-3\n"; + +fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +fn put(path: &Path, text: &str) { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, text).unwrap(); +} + +/// A scratch folder that is removed when the test ends, also when it fails. +struct Tmp(PathBuf); + +impl std::ops::Deref for Tmp { + type Target = Path; + fn deref(&self) -> &Path { + &self.0 + } +} + +impl AsRef for Tmp { + fn as_ref(&self) -> &Path { + &self.0 + } +} + +impl AsRef for Tmp { + fn as_ref(&self) -> &std::ffi::OsStr { + self.0.as_os_str() + } +} + +impl Tmp { + /// A new folder that only this test owns. `create_dir` fails when the + /// path exists, so a run never shares or deletes another run's folder. + fn new(kind: &str, tag: &str) -> Tmp { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let d = std::env::temp_dir().join(format!( + "og-ancestry-{kind}-{}-{tag}-{nanos}", + std::process::id() + )); + std::fs::create_dir(&d).unwrap(); + Tmp(d) + } +} + +impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +/// A sample folder with haplogrep3's call and, for a male, Yleaf's table. +fn sample(name: &str, y_hg: Option<&str>, mt_hg: &str) -> Tmp { + let d = Tmp::new("hg", name); + put( + &d.join("mito/S_haplogroup.txt"), + &format!("\"SampleID\"\t\"Haplogroup\"\t\"Rank\"\n\"S\"\t\"{mt_hg}\"\t\"0.98\"\n"), + ); + if let Some(hg) = y_hg { + put( + &d.join("y_haplogroup/S_y_haplogroup.txt"), + &format!("{Y_HEAD}S_sorted\t{hg}\tM269\t100\t42\t0.97\t1\t1\t0.97\n"), + ); + } + d +} + +/// The summary og summary collect writes for S, as text. +fn collect(d: &Path) -> String { + let input = og_summary::collect::Input { + sample: "S".to_owned(), + sample_dir: d.display().to_string(), + declared_sex: None, + assets: repo().join("assets").display().to_string(), + group: og_summary::collect::Group::All, + tables: repo().join("bin").display().to_string(), + }; + dumps( + &og_summary::collect::post::collect(&input).unwrap(), + &Dump::default(), + ) + .unwrap() +} + +/// Both reports of a summary. +fn render(summary: &str) -> (String, String) { + let s = Summary::parse(summary).unwrap(); + (text::report(&s).unwrap(), html::report(&s).unwrap()) +} + +fn reports(d: &Path) -> (String, String) { + render(&collect(d)) +} + +#[test] +fn a_male_sample_gets_the_y_and_the_mtdna_links() { + let (txt, page) = reports(&sample("male", Some("R-M269"), "H5a7")); + for url in [ + "https://discover.familytreedna.com/y-dna/R-M269", + "https://www.yfull.com/tree/R-M269/", + "https://discover.familytreedna.com/mtdna/H5a7", + "https://www.yfull.com/mtree/H5a7/", + ] { + assert!(txt.contains(url), "{url} in the text report:\n{txt}"); + assert!( + page.contains(&format!("")), + "{url} as a link in the HTML report" + ); + } + assert!(txt.contains("docs/37-y-haplogroup.md#explore-your-branch-further")); + assert!(txt.contains("docs/12-mito-haplogroup.md#explore-your-branch-further")); + assert!(txt.contains("Nothing is uploaded by this pipeline")); + assert!(page.contains("Nothing is uploaded by this pipeline")); +} + +#[test] +fn a_paragroup_links_its_named_branch() { + let (txt, _) = reports(&sample("para", Some("R-M269*(xR-L23)"), "H5a7")); + assert!(txt.contains("https://www.yfull.com/tree/R-M269/"), "{txt}"); + assert!(!txt.contains("y-dna/R-M269*"), "{txt}"); + assert!(!txt.contains("%2A"), "{txt}"); +} + +#[test] +fn a_female_sample_has_no_y_link_and_keeps_the_mtdna_ones() { + let (txt, page) = reports(&sample("female", None, "H5a7")); + for t in [&txt, &page] { + assert!(!t.contains("y-dna/"), "{t}"); + assert!(!t.contains("yfull.com/tree/"), "{t}"); + } + assert!(txt.contains("https://discover.familytreedna.com/mtdna/H5a7")); + assert!(page.contains("https://www.yfull.com/mtree/H5a7/")); +} + +#[test] +fn no_call_has_no_link() { + let (txt, page) = reports(&sample("nocall", Some("NA"), "")); + assert!(!txt.contains("discover.familytreedna.com"), "{txt}"); + assert!(!txt.contains("yfull.com"), "{txt}"); + assert!(!page.contains("discover.familytreedna.com")); + assert!( + txt.contains("insufficient markers"), + "the Y section still says why:\n{txt}" + ); +} + +#[test] +fn a_plus_in_the_branch_is_encoded() { + let (txt, _) = reports(&sample("esc", None, "L3e2b+152")); + assert!(txt.contains("mtdna/L3e2b%2B152"), "{txt}"); + assert!(txt.contains("mtree/L3e2b%2B152/"), "{txt}"); +} + +#[test] +fn a_planted_address_cannot_open_a_tag_in_the_html_report() { + let summary = collect(&sample("planted", Some("R-M269"), "H5a7")); + let planted = r#"https://x.example/\"><"), "it ended the attribute"); + assert_eq!( + page.matches(""><script>alert(1)</script>") + .count(), + 4, + "escaped in the address and the text of both links" + ); + assert!( + txt.contains("https://x.example/\">" + | .sections.haplogroup.data.explore[1].name = "YFull & co" + | .sections.haplogroup.data.contamination_status = "YES" | .sections.haplogroup.data.contamination_level = "0.4<" + | .sections.y_haplogroup.data.explore[1].url = "https://www.yfull.com/tree/\"x\"&y/" + | .sections.y_tree.data.path[0].Branch = "" | .sections.y_tree.data.path[0].Ancestral_SNPs = "M1&M2" + | .sections.y_str.data.markers[0].marker = "DYS<393>" + | .sections.ancient.data.nearest[0].group = "Spain_" | .sections.ancient.data.nearest[0].locality = "A & B" + | .sections.ancestry_fine.data.continent[0].group = "West " + | .sections.ancestry_fine.data.legend[0].colour = "#AbCdEf" | .sections.ancestry_fine.data.legend[1].group = "W" + | .sections.archaic.data.longest = ["chr1:1-2 "] + | .sections.prs.data.scores[0].condition = "CD & \"x\"" | .sections.prs.data.scores[0].effect = "HR <1.5>"' +amut painting '.sections.ancestry_fine.data.painting = [{"code": "AFR", "group": "Africa", "fraction": 0.3}, + {"code": "WEA", "group": "West Eurasia", "fraction": 0.7}] + | .sections.ancestry_fine.data.not_painted = [] + | .sections.ancestry_fine.data.left_out = [{"group": "Oceania (continental group)"}, {"group": "Oceania (regional group)"}, + {"group": "Papua (regional group)"}, {"group": "Odd (name) (x group)"}, {"group": "Plain"}] + | .sections.ancestry_fine.data.min_references = 12 + | .sections.ancestry_fine.data.continent += [{"group": "Small", "fraction": 0.009, "low": 0.001, "high": 0.02}] + | .sections.ancestry_fine.data.region += [{"group": "Wide", "fraction": 0.005, "low": 0.0, "high": 0.06}, + {"group": "Narrow", "fraction": 0.005, "low": 0.0, "high": 0.02}, {"group": "Unfit", "fraction": null, "low": null, "high": null}]' +amut no-painting '.sections.ancestry_fine.data.painting = [] | .sections.ancestry_fine.data.not_painted = [] + | .sections.ancestry_fine.data.left_out = [] | .sections.ancestry_fine.data.painting_png = ""' +amut prs-raw '.sections.prs.data.adjusted = false | del(.sections.prs.data.ancestry) + | .sections.prs.data.scores |= map(.percentile = "" | .evaluated_in = null)' +amut prs-sex '.sections.prs.data.ancestry = {"population": "", "panel": "", "low_confidence": "True"} + | .sections.prs.data.scores = [ + {"condition": "Prostate", "pgs_id": "PGS2", "score": "1", "matched": "5", "total": "6", "chrx": "0", + "applies_to": "male", "sex_check": "not applicable to the declared sex", "percentile": "", "group": "", + "evaluated_in": ["EUR"], "effect": ""}, + {"condition": "Breast", "pgs_id": "PGS3", "score": "2", "matched": "7", "total": "9", "chrx": "3 left out", + "applies_to": "female", "sex_check": "no sex declared", "percentile": "40", "group": "AFR", + "evaluated_in": [], "effect": ""}, + {"condition": "Height", "pgs_id": "PGS4", "score": "3", "matched": "1", "total": "2", "chrx": "", + "applies_to": "", "sex_check": "", "percentile": "60", "group": "AFR", "evaluated_in": ["EUR", "AFR"], + "effect": ""}]' +amut y-calls '.sections.y_tree.data.ancestral_in_terminal = "M1,M2" | .sections.y_tree.data.outvoted_ancestral = null + | .sections.y_tree.data.path = [] | .sections.y_str.data.markers |= map(.count = "") + | .sections.ancient.data.nearest = [] | .sections.ancient.data.plot_svg = "" + | .sections.archaic.data.longest = [] + | .sections.haplogroup.data.explore = [] | del(.sections.haplogroup.data.contamination_status) + | .sections.y_haplogroup.data.explore = []' +amut since '.new_since = {"previous_utc": "2026-01-01T00:00:00Z", "sections": ["y_str", "prs", "haplogroup"]}' +amut numbers '.sections.y_str.data.markers[0].count = 13 | .sections.y_str.data.markers[1].panel = true + | .sections.ancient.data.nearest[0].rank = 1 | .sections.ancient.data.nearest[1].n = 5.5 + | .sections.y_tree.data.path[0].Level = 1 | .sections.y_tree.data.path[1].Derived = 2.0 + | .sections.archaic.data.neanderthal_bp = 51234567.0 | .sections.archaic.data.denisovan_bp = true + | .sections.prs.data.scores[0].condition = 7 | .sections.prs.data.scores[1].score = 0.25 + | .sections.ancestry_fine.data.continent[1].fraction = 1' +rm -f "$abase" + # --- the oracle ----------------------------------------------------------------------- cat > oracle.sh << 'ORACLE' #!/bin/sh @@ -227,8 +535,12 @@ PYTHONPATH=$d python3 "$d/render_report.py" "$@" rm -rf "$d" # The sections og report render writes; the others are left out. PORTED=" qc variants not-run not-assessed " +PORTED="$PORTED pharmcat cpic cyp2d6 hla mito " # port-report-optin PORTED="${PORTED}exomiser mei vntyper chip mosaic-mca locityper roh telomere viral " +PORTED="${PORTED}callability clinvar clinical-filter cpsr slivar carrier-list couple " +PORTED="${PORTED}ancestry-fine mt-haplogroup y-chromosome archaic ancient prs " +PORTED="${PORTED}expansions lpa paralogs traits-panel bloodtype " while [ $# -gt 0 ]; do case "$1" in -o) @@ -245,8 +557,9 @@ done ORACLE # html.awk: drops each section not in keep (its line in the contents and its -# frame), then each chapter with no section left (its contents entry and its -# body), and the gene and rsID appendix. +# frame) and every chart (og report charts will write them), then each +# chapter with no section left (its contents entry and its body), and the +# gene and rsID appendix. cat > html.awk << 'AWK' function wanted(line, id) { if (!match(line, /id="[^"]*"/) && !match(line, /href="#[^"]*"/)) return 1 @@ -288,19 +601,41 @@ AWK # text.awk: drops each section block not in keep, then each chapter left # with no block. A block starts at its "## " title; the titles of the kept -# sections are listed here, as render_report.py writes them. +# sections are listed here, as render_report.py writes them. A PRS row +# whose counts the summary has as null reads "not available", as og writes +# it; render_report.py writes "None" there (and stops its HTML report on +# that row). cat > text.awk << 'AWK' BEGIN { rule = sprintf("%80s", ""); gsub(/ /, "#", rule) title["## Quality control"] = "qc"; title["## Variant Calling (DeepVariant)"] = "variants" + title["## Pharmacogenomics (PharmCAT)"] = "pharmcat"; title["## CPIC Drug-Gene Recommendations"] = "cpic" + title["## pypgx Pharmacogenomics"] = "cyp2d6"; title["## PGx copy number"] = "cyp2d6" + title["## CYP2D6 across callers"] = "cyp2d6"; title["## HLA Typing (T1K)"] = "hla" + title["## Mitochondrial Variants (Mutect2)"] = "mito" # port-report-optin title["## Symptom-Driven Ranking (Exomiser, step 48)"] = "exomiser"; title["## Mobile Element Insertions (step 50)"] = "mei" title["## MUC1 VNTR Frameshifts (ADTKD-MUC1, VNtyper)"] = "vntyper"; title["## Clonal Haematopoiesis (CHIP, step 49)"] = "chip" title["## Mosaic Chromosomal Alterations (MoChA, step 51)"] = "mosaic-mca"; title["## Runs of Homozygosity"] = "roh" title["## Locityper Second Opinion at Hard Loci (step 53)"] = "locityper" title["## Telomere content (relative, TelomereHunter)"] = "telomere"; title["## Viral DNA in the Unmapped Reads (step 47)"] = "viral" + title["## Genes This Genome Could Not Read Reliably"] = "callability" + title["## ClinVar Pathogenic Screen"] = "clinvar"; title["## Clinical Variant Filter"] = "clinical-filter" + title["## Cancer Predisposition Screening (CPSR)"] = "cpsr"; title["## Variant Prioritization (slivar)"] = "slivar" + title["## Carrier List (this sample)"] = "carrier-list"; title["## Couple Carrier Screen (step 39)"] = "couple" + title["## Fine-Grained Ancestry (ADMIXTURE, RFMix)"] = "ancestry-fine"; title["## Mitochondrial Haplogroup"] = "mt-haplogroup" + title["## Y-Chromosome Haplogroup (Yleaf)"] = "y-chromosome"; title["## Y Placement on the Current Public Tree"] = "y-chromosome" + title["## Y-STR Markers (HipSTR)"] = "y-chromosome"; title["## Archaic Ancestry (IBDmix, step 43)"] = "archaic" + title["## Ancient Populations (AADR PCA, step 44)"] = "ancient"; title["## Polygenic Risk Scores"] = "prs" + title["## Repeat Expansions (ExpansionHunter)"] = "expansions" + title["## Lp(a): LPA KIV-2 Repeat Count and Risk Variants"] = "lpa" + title["## Paralog Genes (Paraphase, PacBio HiFi)"] = "paralogs" + title["## SMN1/SMN2 Copy Number (Parascopy, short reads)"] = "paralogs" + title["## Paralog Copy Number"] = "paralogs" + title["## Traits Panel (step 42)"] = "traits-panel" + title["## Blood Groups (predicted from the genome)"] = "bloodtype" } -{ lines[++n] = $0 } +{ sub(/ None \(None\/None variants matched\) /, " not available (not available/not available variants matched) "); lines[++n] = $0 } END { # The end part (steps not run onward) and the head (before the first # banner) stay; chapters are banner, title, banner, blank, then blocks. diff --git a/tests/parity/golden/report-render/m-optin-viral-na/m-optin-viral-na.html b/tests/parity/golden/report-render/m-optin-viral-na/m-optin-viral-na.html index 44c2117d6..13c4013f4 100644 --- a/tests/parity/golden/report-render/m-optin-viral-na/m-optin-viral-na.html +++ b/tests/parity/golden/report-render/m-optin-viral-na/m-optin-viral-na.html @@ -113,13 +113,38 @@

    Contents

  • Variant calling
  • +
  • Clinical +
      +
    1. ClinVar screen and hits
    2. +
    3. Clinical filter and secondary findings
    4. +
    5. Cancer predisposition
    6. +
    7. Variant prioritization
    8. +
    9. Repeat expansions
    10. +
    11. Mitochondrial variants
    12. +
    +
  • +
  • Carrier +
      +
    1. Paralog genes and SMN1
    2. +
    +
  • +
  • Medicines +
      +
    1. PharmCAT
    2. +
    3. CPIC recommendations
    4. +
    5. pypgx, Cyrius and CYP2D6
    6. +
    7. HLA typing
    8. +
    +
  • Ancestry
      +
    1. Mitochondrial haplogroup
    2. Runs of homozygosity
  • Traits and scores
      +
    1. Traits
    2. Telomere content
  • @@ -177,10 +202,117 @@

    Variant Calling

    +
    +

    Clinical

    +

    Is there anything to take to a doctor?

    +
    +
    +
    +

    ClinVar Screening

    +

    Step 06 · Not run · link to this section

    +
    StatusNot run
    +
    +
    +
    +
    +

    Clinical Variant Filter

    +

    Step 23 · Not run · link to this section

    +
    StatusNot run
    +
    +
    +
    +
    +

    Cancer Predisposition

    +

    Step 17 · Not run · link to this section

    +
    StatusNot run
    +
    +
    +
    +
    +

    Variant Prioritization (slivar)

    +

    Step 31 · Not run · link to this section

    +
    StatusNot run
    +
    +
    +
    +
    +

    Repeat Expansions

    +

    Step 09 · Not run · link to this section

    +
    StatusNot run
    +
    +
    +
    +
    +

    Mitochondrial Analysis

    +

    Step 20 · Not run · link to this section

    +
    StatusNot run
    +
    +
    +
    +
    +
    +

    Carrier

    +

    What could I pass on to a child?

    +
    +
    +
    +

    Paralog Copy Number

    +

    Steps 35b, 35 · Not run · link to this section

    +
    StatusNot checked
    +

    Paralog copy number: not checked (neither step 35 nor step 35b ran)

    +
    +
    +
    +
    +
    +

    Medicines

    +

    How might I respond to medicines?

    +
    +
    +
    +

    Pharmacogenomics

    +

    Step 07 · Not run · link to this section

    +
    StatusNot run
    +
    +
    +
    +
    +

    CPIC Drug Recommendations

    +

    Step 27 · Not run · link to this section

    +
    StatusNot run
    +
    +
    +
    +
    +

    CYP2D6 Across Callers

    +

    Steps 32, 21 · Not run · link to this section

    +

    CYP2D6 sits beside its look-alike CYP2D7, and short reads often cannot tell them apart, so pypgx, Cyrius and step 36 can disagree or give no call. A disagreement or a missing call means the gene was not read reliably here; a clinical CYP2D6 test settles it.

    +
    PharmCAT (step 36's call)not run
    +
    pypgxnot run
    +
    Cyriusnot run
    +
    pypgx and Cyrius agreefewer than two usable calls
    +
    +
    +
    +
    +

    HLA Typing

    +

    Step 08 · Not run · link to this section

    +
    StatusNot run
    +
    +
    +
    +

    Ancestry

    Where do my ancestors come from?

    +
    +
    +

    Mitochondrial Haplogroup

    +

    Step 12 · Not run · link to this section

    +
    StatusNot run
    +
    +

    Runs of Homozygosity

    @@ -194,6 +326,13 @@

    Runs of Homozygosity

    Traits and scores

    What do studies say about my traits and risk scores?

    +
    +
    +

    Traits

    +

    Step 42 · Not run · link to this section

    +
    StatusNot run
    +
    +

    Telomere content (relative)

    diff --git a/tests/parity/golden/report-render/m-optin-viral-na/m-optin-viral-na.txt b/tests/parity/golden/report-render/m-optin-viral-na/m-optin-viral-na.txt index 9ebd34215..4d011bee3 100644 --- a/tests/parity/golden/report-render/m-optin-viral-na/m-optin-viral-na.txt +++ b/tests/parity/golden/report-render/m-optin-viral-na/m-optin-viral-na.txt @@ -26,6 +26,14 @@ SNPs: 11111 Indels: 6667 Counts of the calls DeepVariant made against the reference. A whole genome usually holds 4 to 5 million, most of them common. PASS calls passed the caller's filter; a count says nothing about health on its own. +################################################################################ +# 3. CARRIER: What could I pass on to a child? +################################################################################ + +## Paralog Copy Number +--- + Paralog copy number: not checked (neither step 35 nor step 35b ran) + ################################################################################ # 7. OTHER AND EXPORTS: What else did the run find or write? ################################################################################ diff --git a/tests/parity/golden/summary-collect/no-folder/exit b/tests/parity/golden/summary-collect/no-folder/exit new file mode 100644 index 000000000..d00491fd7 --- /dev/null +++ b/tests/parity/golden/summary-collect/no-folder/exit @@ -0,0 +1 @@ +1 diff --git a/tests/parity/golden/summary-collect/no-folder/no-folder.json.absent b/tests/parity/golden/summary-collect/no-folder/no-folder.json.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/summary-collect/no-folder/stderr b/tests/parity/golden/summary-collect/no-folder/stderr new file mode 100644 index 000000000..5dcf7c827 --- /dev/null +++ b/tests/parity/golden/summary-collect/no-folder/stderr @@ -0,0 +1 @@ +ERROR: no-such-folder: not a readable folder: No such file or directory (os error 2) diff --git a/tests/parity/golden/summary-collect/no-folder/stdout b/tests/parity/golden/summary-collect/no-folder/stdout new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/oracle.ref b/tests/parity/oracle.ref index ee523144d..a53cf25c9 100644 --- a/tests/parity/oracle.ref +++ b/tests/parity/oracle.ref @@ -97,6 +97,6 @@ traits 669a8196cf4c377200f9b4af9c235ef85fb774a1 file:bin/traits_report.py bash . roh-summary 7c1470bc50933f48a1a0dde3c0e0b2b87bc48718 file:scripts/11-roh-analysis.sh sh ./oracle.sh roh summary couple-screen d4d3d4b629fc77a058bbbbd302c75bbfae1a01c2 file:bin/couple_report.py sh -c 'exec sh ./oracle.sh "$0" screen "$@"' couple screen couple-genes d4d3d4b629fc77a058bbbbd302c75bbfae1a01c2 file:bin/couple_report.py sh -c 'exec sh ./oracle.sh "$0" genes "$@"' couple genes -report-render 1ec630690e63796b4410d23ec65768200adb8dc0 file:bin/render_report.py sh ./oracle.sh report render +report-render 65095e975bea01c74ed92a4f1b8c1bd00d05caaa file:bin/render_report.py sh ./oracle.sh report render archaic-compare 65095e975bea01c74ed92a4f1b8c1bd00d05caaa file:bin/archaic_compare.py sh -c 'exec python3 "$0" compare "$@"' archaic compare archaic-genomes 65095e975bea01c74ed92a4f1b8c1bd00d05caaa file:bin/archaic_compare.py sh -c 'exec python3 "$0" genomes "$@"' archaic genomes diff --git a/tests/parity/remnants.tsv b/tests/parity/remnants.tsv index c4f39fc85..4b6b6f55f 100644 --- a/tests/parity/remnants.tsv +++ b/tests/parity/remnants.tsv @@ -5,4 +5,4 @@ # filenamesimporterslane bin/family_report.py COLUMNS,MISSING,SEX,NAME,BINS,missing,read_pedigree,build_members,bad_name,kinship_table,declared_relationship,degree bin/pedigree_svg.py,tests/test_family_report.py R4 bin/genome_lookup.py GENE_KEYS,RSID_RE,norm_chrom,parse_query,names_of,walk_mentions,flat bin/results_index.py,tests/test_results_index.py R5c -bin/collect_summary.py HERE,ASSETS_DIR,SECTIONS,og,collect,STARS,COMMON_AF,RUN_ALL_STEPS,SEVERITY_KINDS,open_text,info_map,read_tsv,hla_quality,HLA_MAX_CANDIDATES,EXPLORE_DOCS,csq_rank,allele_key,parse_alleles bin/owngenome_live.py,bin/render_report.py,bin/results_index.py,tests/demo/make_demo_sample.py,tests/test_acmg_sf_rules.py,tests/test_ancestry_fine_report.py,tests/test_archaic_report.py,tests/test_callability.py,tests/test_collect_summary.py,tests/test_copy_number.py,tests/test_cross_engine_biallelic.py,tests/test_haplogroup_explore.py,tests/test_hla_report.py,tests/test_live_journal.py,tests/test_live_results.py,tests/test_lpa_kiv2.py,tests/test_mito_extended.py,tests/test_render_report.py,tests/test_results_index.py,tests/test_run_state.py,tests/test_traits_report.py,tests/test_viewer.py,tests/test_ystr_report.py R6b +bin/collect_summary.py HERE,ASSETS_DIR,SECTIONS,og,collect,STARS,COMMON_AF,RUN_ALL_STEPS,open_text,info_map,read_tsv,hla_quality,HLA_MAX_CANDIDATES,EXPLORE_DOCS,csq_rank bin/owngenome_live.py,bin/render_report.py,bin/results_index.py,tests/demo/make_demo_sample.py,tests/test_collect_summary.py,tests/test_live_journal.py,tests/test_live_results.py,tests/test_render_report.py,tests/test_results_index.py,tests/test_run_state.py,tests/test_viewer.py R6b diff --git a/tests/test_acmg_sf_rules.py b/tests/test_acmg_sf_rules.py deleted file mode 100644 index 00d738a8d..000000000 --- a/tests/test_acmg_sf_rules.py +++ /dev/null @@ -1,316 +0,0 @@ -#!/usr/bin/env python3 -"""The ACMG SF v3.3 reporting rules (assets/acmg_sf.tsv, og summary collect's -ACMG SF tier, rust/crates/og-summary) and the one-person carrier list (assets/carrier_genes.tsv), on -synthetic sample folders. - -Checks: - 1. the catalog: exactly the 84 genes of SF v3.3 Table 1 (Lee et al., Genet - Med 2025;27(8):101454), one row each; the X-linked rows are GLA, OTC and - ABCD1 with Table 1's rule each (GLA and OTC: hemi, het and homozygous; - ABCD1: hemi, homozygous or two het); the recessive genes need two - variants; HFE counts p.C282Y homozygotes only; - 2. MUTYH: one heterozygous ClinVar hit is not a secondary finding (it is - listed apart with the reason), two are, a homozygote is; - 3. HFE: p.C282Y heterozygous is not an SF, homozygous is, a homozygous - p.His63Asp is not; - 4. X-linked: a female heterozygote (0/1) in GLA and in OTC is reported, in - ABCD1 it is not; a hemizygous ABCD1 call (GT 1) is, and so are two ABCD1 - heterozygous variants; - 5. structural: an AnnotSV class 5 exon deletion in BRCA1 is in the SF - section, a class 5 deletion of the 3' end of EPCAM counts for MSH2, and a - class 5 deletion in MUTYH (no SV rule) is not; - 6. severity: BTD p.Asp444His homozygous is reported as partial, not - profound; a BTD homozygote without it carries no partial note; - 7. the carrier list of one sample, with no partner: a heterozygous CFTR hit - is a carrier line, a HEXA pseudodeficiency allele (c.739C>T) is set - aside and makes no carrier line, another HEXA hit does; the - couple screen sets the same allele aside, and a whole couple screen run - with it writes its table and names it as a gap; a severity entry with - a kind and no variant is a ValueError; - 8. both reports print the SF verdicts, the reasons and the carrier list; - an SF deletion without step 22's consensus says its caller support is - not known; a failed carrier section still shows its HTML card; - 9. without the assets folder the ClinVar hits and the clinical counts stay, - and both lists say why they were not made. -""" -import csv -import gzip -import os -import shutil -import subprocess -import sys -import tempfile - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -import collect_summary # noqa: E402 -import render_report # noqa: E402 -sys.path.insert(0, os.path.join(REPO, "tests")) -from og_binary import og_binary # noqa: E402 - -FAILS = [] - - -def check(name, ok, detail=""): - print(("[PASS] " if ok else "[FAIL] ") + name + ("" if ok else " -- " + str(detail)[:1500])) - if not ok: - FAILS.append(name) - - -def put(path, text, gz=False): - os.makedirs(os.path.dirname(path), exist_ok=True) - with (gzip.open(path, "wt") if gz else open(path, "w")) as f: - f.write(text) - return path - - -# ACMG SF v3.3 Table 1, gene by gene. -SF_V33 = """APC RET BRCA1 BRCA2 PALB2 SDHD SDHAF2 SDHC SDHB MAX TMEM127 BMPR1A SMAD4 TP53 MLH1 MSH2 MSH6 PMS2 -MEN1 MUTYH NF2 STK11 PTEN RB1 TSC1 TSC2 VHL WT1 FBN1 TGFBR1 TGFBR2 SMAD3 ACTA2 MYH11 PKP2 DSP DSC2 TMEM43 -DSG2 RYR2 CASQ2 TRDN TNNT2 LMNA FLNC TTN BAG3 DES RBM20 TNNC1 PLN COL3A1 LDLR APOB PCSK9 MYH7 MYBPC3 -TNNI3 TPM1 MYL3 ACTC1 PRKAG2 MYL2 KCNQ1 KCNH2 SCN5A CALM1 CALM2 CALM3 BTD CYP27A1 GLA HFE OTC GAA ABCD1 -ACVRL1 ENG TTR RYR1 CACNA1S HNF1A RPE65 ATP7B""".split() -# Table 1 "Variants to report" of the X-linked genes, and the rule each must be. -XL_RULES = {"GLA": ("All hemi, het, homozygous P and LP", "het_female_too"), - "OTC": ("All hemi, het, homozygous P and LP", "het_female_too"), - "ABCD1": ("All hemi, homozygous or two het. P and/or LP", "hemizygous")} -AR_TWO = {"MUTYH", "CASQ2", "TRDN", "BTD", "CYP27A1", "GAA", "RPE65", "ATP7B"} - -VCF_HEAD = "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" - - -def hit(chrom, pos, ref, alt, gene, gt, sig="Pathogenic"): - return (f"{chrom}\t{pos}\t.\t{ref}\t{alt}\t50\tPASS\tGENEINFO={gene}:1;CLNSIG={sig};" - f"CLNREVSTAT=criteria_provided,_multiple_submitters,_no_conflicts\tGT:GQ:DP\t{gt}:50:30\n") - - -def sample(work, name, hits, annotsv=None): - d = os.path.join(work, name, "S") - put(f"{d}/clinvar/S_clinvar_hits.vcf", VCF_HEAD + "".join(hits)) - if annotsv is not None: - put(f"{d}/annotsv/S_sv_class4_5_checked.tsv", ANNOTSV_COLS + annotsv) - return d - - -ANNOTSV_COLS = ("SV_chrom\tSV_start\tSV_end\tSV_length\tSV_type\tGene_name\tACMG_class\tCaller\tExon_edge\t" - "Exon_edge_detail\tPopulation_AF\tPopulation_source\tCommon\n") - - -def sv(chrom, a, b, genes, cls="5", kind="DEL"): - return f"{chrom}\t{a}\t{b}\t-{b - a}\t{kind}\t{genes}\t{cls}\tmanta\tno\t.\t\t\tno\n" - - -def sf_of(d): - return collect_summary.collect("S", d)["sections"]["clinical"]["data"].get("acmg_sf") or {} - - -def sf_of_clinvar(d): - """The SF tier of a folder with ClinVar hits and a clinical VCF with no - record (the Nextflow layout: the clinical section is the VCF's count, and - the tier comes from the hits alone).""" - v = f"{d}/clinical/S_clinical.vcf.gz" - if not os.path.exists(v): - put(v, VCF_HEAD, gz=True) - return sf_of(d) - - -def catalog(folder): - """The rows of FOLDER/acmg_sf.tsv, and {gene: rule} with its rule cells - read (the variants as collect_summary.parse_alleles reads them).""" - with open(os.path.join(folder, "acmg_sf.tsv"), encoding="utf-8") as f: - rows = list(csv.DictReader((x for x in f if not x.startswith("#")), delimiter="\t", quoting=csv.QUOTE_NONE)) - return rows, {r["gene"]: {"inheritance": r["inheritance"], "report_if": r["report_if"], - "table_text": r["table_text"], "sv_cn": r["sv_cn"] == "yes", - "variants": collect_summary.parse_alleles(r["variants"])} for r in rows} - - -def reported(sf, gene): - return [x for x in sf.get("clinvar_hits", []) if x["gene"] == gene] - - -def apart(sf, gene): - return [x for x in sf.get("not_reported", []) if x["gene"] == gene] - - -def main(): - work = tempfile.mkdtemp(prefix="acmg_sf_") - try: - # 1. the catalog - p = os.path.join(REPO, "assets", "acmg_sf.tsv") - try: - rows, rules = catalog(os.path.join(REPO, "assets")) - except Exception as e: # the pre-fix state: no catalog, no loader - rows, rules = [], {} - check("catalog: assets/acmg_sf.tsv loads", False, f"{type(e).__name__}: {e}") - genes = [r["gene"] for r in rows] - check("catalog: exactly 84 rows, one per gene", len(genes) == 84 and len(set(genes)) == 84, len(genes)) - check("catalog: the genes are SF v3.3 Table 1's", set(genes) == set(SF_V33) and len(SF_V33) == 84, - sorted(set(genes) ^ set(SF_V33))) - check("catalog: every row has an HGNC id", all(r.get("hgnc_id", "").startswith("HGNC:") for r in rows)) - xl = {g for g, r in rules.items() if r["inheritance"] == "XL"} - check("catalog: the X-linked rows are GLA, OTC and ABCD1", xl == set(XL_RULES), sorted(xl)) - for g, (text, rule) in XL_RULES.items(): - r = rules.get(g, {}) - check(f"catalog: {g} is '{text}', rule {rule}", - (r.get("table_text"), r.get("report_if")) == (text, rule), r) - two = {g for g, r in rules.items() if r["report_if"] == "biallelic"} - check("catalog: the recessive two-variant genes", two == AR_TWO, sorted(two ^ AR_TWO)) - hfe = rules.get("HFE", {}) - check("catalog: HFE counts p.C282Y (chr6:26092913 G>A) homozygotes only", - hfe.get("report_if") == "hom_variant" and list(hfe.get("variants", {})) == [("chr6", 26092913, "G", "A")], - hfe) - check("catalog: BRCA1, BRCA2, MLH1, MSH2 and LDLR take class 4/5 SVs", - {g for g, r in rules.items() if r["sv_cn"]} == {"BRCA1", "BRCA2", "MLH1", "MSH2", "LDLR"}, - sorted(g for g, r in rules.items() if r["sv_cn"])) - # A dropped gene makes the count check fail: the check can go red. - dropped = os.path.join(work, "dropped") - os.makedirs(dropped) - if os.path.isfile(p): - with open(p) as f, open(os.path.join(dropped, "acmg_sf.tsv"), "w") as g: - g.write("".join(x for x in f if not x.startswith("TTN\t"))) - n = len(catalog(dropped)[1]) - check("catalog control: a copy without TTN has 83 genes, not 84", n == 83, n) - - # 2. MUTYH: chr1 MUTYH p.Gly396Asp and p.Tyr179Cys (GRCh38) - g396 = hit("chr1", 45332803, "C", "T", "MUTYH", "0/1") - y179 = hit("chr1", 45334483, "T", "C", "MUTYH", "0/1") - sf = sf_of_clinvar(sample(work, "mutyh_het", [g396])) - check("MUTYH: one heterozygous variant is not a secondary finding", reported(sf, "MUTYH") == [], sf) - check("MUTYH: it is listed apart with the reason", - [x["reason"] for x in apart(sf, "MUTYH")] == ["one heterozygous variant; MUTYH is reported with " - "P and LP (2 variants)"], apart(sf, "MUTYH")) - sf = sf_of_clinvar(sample(work, "mutyh_two", [g396, y179])) - check("MUTYH: two heterozygous variants are a secondary finding (phase unknown)", - len(reported(sf, "MUTYH")) == 2 and not apart(sf, "MUTYH"), sf) - sf = sf_of_clinvar(sample(work, "mutyh_hom", [hit("chr1", 45332803, "C", "T", "MUTYH", "1/1")])) - check("MUTYH: a homozygote is a secondary finding", len(reported(sf, "MUTYH")) == 1, sf) - - # 3. HFE - c282y_het = hit("chr6", 26092913, "G", "A", "HFE", "0/1") - sf = sf_of_clinvar(sample(work, "hfe_het", [c282y_het])) - check("HFE: p.C282Y heterozygous is not an SF", reported(sf, "HFE") == [] and len(apart(sf, "HFE")) == 1, sf) - sf = sf_of_clinvar(sample(work, "hfe_hom", [hit("chr6", 26092913, "G", "A", "HFE", "1/1")])) - check("HFE: p.C282Y homozygous is an SF", len(reported(sf, "HFE")) == 1, sf) - sf = sf_of_clinvar(sample(work, "hfe_h63d", [hit("chr6", 26090951, "C", "G", "HFE", "1/1")])) - check("HFE: p.His63Asp homozygous is not an SF", reported(sf, "HFE") == [], sf) - - # 4. X-linked, a female heterozygote (diploid 0/1) and a hemizygous call (GT 1) - sf = sf_of_clinvar(sample(work, "xl_female", [ - hit("chrX", 101398000, "C", "T", "GLA", "0/1"), hit("chrX", 38400000, "G", "A", "OTC", "0/1"), - hit("chrX", 153725000, "C", "T", "ABCD1", "0/1")])) - check("X-linked: a female heterozygote in GLA is reported (Table 1: hemi, het, homozygous)", - len(reported(sf, "GLA")) == 1, sf) - check("X-linked: a female heterozygote in OTC is reported (Table 1: hemi, het, homozygous)", - len(reported(sf, "OTC")) == 1, sf) - check("X-linked: a female heterozygote in ABCD1 is not (Table 1: hemi, homozygous or two het)", - reported(sf, "ABCD1") == [] and len(apart(sf, "ABCD1")) == 1, sf) - sf = sf_of_clinvar(sample(work, "xl_male", [hit("chrX", 153725000, "C", "T", "ABCD1", "1")])) - check("X-linked: a hemizygous ABCD1 variant is reported", len(reported(sf, "ABCD1")) == 1, sf) - sf = sf_of_clinvar(sample(work, "xl_two", [hit("chrX", 153725000, "C", "T", "ABCD1", "0/1"), - hit("chrX", 153726000, "G", "A", "ABCD1", "0/1")])) - check("X-linked: two heterozygous ABCD1 variants are reported", len(reported(sf, "ABCD1")) == 2, sf) - - # 5. structural: BRCA1 exon deletion, EPCAM 3' end, MUTYH (no SV rule) - d = sample(work, "sv", [], sv("17", 43070000, 43080000, "BRCA1") + sv("2", 47380000, 47390000, "EPCAM") - + sv("1", 45330000, 45336000, "MUTYH")) - sf = sf_of_clinvar(d) - st = {(x["gene"], x.get("via")) for x in sf.get("structural", [])} - check("structural: a class 5 BRCA1 exon deletion is in the SF section", ("BRCA1", None) in st, sf) - check("structural: a class 5 deletion of EPCAM's 3' end counts for MSH2", - any(g == "MSH2" and v and "EPCAM" in v for g, v in st), st) - check("structural: a class 5 deletion in MUTYH (no SV rule) is not", not any(g == "MUTYH" for g, _ in st), st) - - # 6. severity: BTD p.Asp444His (chr3:15645186 G>C) - sf = sf_of_clinvar(sample(work, "btd_d444h", [hit("chr3", 15645186, "G", "C", "BTD", "1/1")])) - btd = reported(sf, "BTD") - check("BTD: D444H/D444H is reported as partial, not profound", - len(btd) == 1 and (btd[0].get("severity") or "").startswith("partial, not profound"), sf) - sf = sf_of_clinvar(sample(work, "btd_profound", [hit("chr3", 15647000, "G", "A", "BTD", "1/1")])) - btd = reported(sf, "BTD") - check("BTD: a homozygote without D444H carries no partial note", - len(btd) == 1 and not btd[0].get("severity"), sf) - - # 7. the carrier list of one sample, no partner; HEXA c.739C>T pseudodeficiency - d = sample(work, "carrier", [hit("chr7", 117559590, "ATCT", "A", "CFTR", "0/1"), - hit("chr15", 72350584, "G", "A", "HEXA", "0/1"), - hit("chr15", 72346579, "G", "GGATA", "HEXA", "0/1")]) - summ = collect_summary.collect("S", d) - car = summ["sections"].get("carrier", {}).get("data") or {} - lines = {(x["gene"], x["variant"]) for x in car.get("carriers", [])} - check("carrier list: one sample, no partner, a CFTR carrier line", ("CFTR", "chr7:117559590 ATCT>A") in lines, car) - check("carrier list: the HEXA pseudodeficiency allele makes no carrier line", - ("HEXA", "chr15:72350584 G>A") not in lines - and [x["kind"] for x in car.get("set_aside", [])] == ["pseudodeficiency"], car) - check("carrier list: the other HEXA hit does", ("HEXA", "chr15:72346579 G>GGATA") in lines, car) - # The whole couple screen (og couple screen), the sample in both partners. - ped = put(f"{work}/couple/pedigree.tsv", "sample\tsex\tfather\tmother\tpartner\nA\tfemale\t.\t.\tB\n" - "B\tmale\t.\t.\tA\n") - r = subprocess.run([og_binary(), "couple", "screen", - "--pedigree", ped, "--genes", os.path.join(REPO, "assets", "carrier_genes.tsv"), - "--clinvar", f"A={d}/clinvar/S_clinvar_hits.vcf", - "--clinvar", f"B={d}/clinvar/S_clinvar_hits.vcf", "--out-dir", f"{work}/couple/out"], - capture_output=True, text=True) - md = f"{work}/couple/out/couple_A_B.md" - md_text = open(md).read() if os.path.isfile(md) else "" - tab = f"{work}/couple/out/couple_A_B.tsv" - with open(tab) if os.path.isfile(tab) else open(os.devnull) as f: - found = sorted(line.split("\t")[0] for line in list(f)[1:]) - aside = [x for x in md_text.splitlines() if "Set aside for A" in x] - check("couple screen: the same pseudodeficiency allele is no carrier finding", - found == ["CFTR", "HEXA"] and len(aside) == 1 and "pseudodeficiency" in aside[0], (found, aside)) - check("couple screen run: a set-aside allele is a gap line, the screen still writes its table", - r.returncode == 0 and "Set aside for A, not a carrier finding on its own" in md_text - and os.path.isfile(tab), (r.returncode, r.stderr[-800:])) - try: - collect_summary.parse_alleles("hypomorph", kinds=True) - err = None - except Exception as e: # noqa: BLE001 - err = e - check("a severity entry with a kind and no variant is a ValueError", isinstance(err, ValueError), repr(err)) - - # 8. both reports - d = sample(work, "report", [g396, c282y_het, hit("chr3", 15645186, "G", "C", "BTD", "1/1"), - hit("chr7", 117559590, "ATCT", "A", "CFTR", "0/1")], - sv("17", 43070000, 43080000, "BRCA1")) - put(f"{d}/clinical/S_clinical.vcf.gz", "##fileformat=VCFv4.2\n" + VCF_HEAD.split("\n", 1)[1], gz=True) - summ = collect_summary.collect("S", d) - txt, html = render_report.text_report(summ), render_report.html_report(summ) - for what, needle in (("MUTYH not reported, with the reason", "MUTYH chr1:45332803 C>T het (ClinVar): not " - "reported, one heterozygous variant"), - ("HFE not reported", "HFE chr6:26092913 G>A het (ClinVar): not reported"), - ("BTD partial", "partial, not profound"), - ("the BRCA1 deletion", "DEL chr17:43070000-43080000"), - ("the carrier list", "Carrier list (")): - check(f"text report: {what}", needle in txt, [x for x in txt.splitlines() if "ACMG" in x or "Carrier" in x]) - check(f"HTML report: {what}", needle in html.replace(">", ">")) - check("without step 22 the SF deletion says its caller support is not known", - "caller support not known" in txt and "None caller" not in txt, - [x for x in txt.splitlines() if "AnnotSV class" in x]) - summ["sections"]["carrier"] = dict(summ["sections"]["carrier"], state="failed", data={}) - check("HTML report: a failed carrier section still shows its card", - "Carrier List (this sample)" in render_report.html_report(summ)) - - # 9. without the assets folder - keep = collect_summary.ASSETS_DIR - collect_summary.ASSETS_DIR = os.path.join(work, "no-assets") - try: - summ = collect_summary.collect("S", d) - finally: - collect_summary.ASSETS_DIR = keep - cv, cl = summ["sections"]["clinvar"], summ["sections"]["clinical"] - check("no assets: the ClinVar hits stay", cv["state"] == "ok" and cv["data"]["count"] == 4, cv["state"]) - check("no assets: both lists say why they were not made", - "acmg_sf.tsv" in (cl["data"].get("acmg_sf") or {}).get("unavailable", "") - and "carrier_genes.tsv" in summ["sections"]["carrier"]["data"].get("unavailable", ""), - (cl, summ["sections"]["carrier"])) - txt = render_report.text_report(summ) - check("no assets: the text report says so", "Secondary findings: not assessed" in txt - and "Carrier list: not made" in txt) - finally: - shutil.rmtree(work, ignore_errors=True) - print("\nRESULT:", "ALL PASS" if not FAILS else "%d FAILED" % len(FAILS)) - return 1 if FAILS else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/test_ancestry_fine_report.py b/tests/test_ancestry_fine_report.py index 196c713bb..20d25a903 100755 --- a/tests/test_ancestry_fine_report.py +++ b/tests/test_ancestry_fine_report.py @@ -15,11 +15,12 @@ chroms a chromosome is painted only with 100 SNVs spanning 5 cM report fractions with the bootstrap range, merged painting rows, the painted shares, a PNG of the right size, the left-out group - and the chromosome RFMix failed on, in the table, the text and - both pipeline reports (og summary collect, render_report.py) + and the chromosome RFMix failed on, in the table and the text The Rust unit tests (rust/crates/og-ancestry) and tests/parity check each command in detail; this test runs them as the pipeline does, through bin/og. +Both pipeline reports of the table are checked by +rust/crates/og-render/tests/ancestry_fine.rs. Run: OG=bin/og python3 tests/test_ancestry_fine_report.py """ @@ -31,7 +32,6 @@ import zlib REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) OG = os.environ.get("OG", "") FAILS = [] @@ -198,20 +198,6 @@ def run_all(tmp): expect("report: the text says chr19 was not painted, and chr20 still was", ("Not painted: RFMix failed on chr19" in text, "of chr20:" in text), (True, True)) - # Both pipeline reports (og summary collect, then render_report.py) carry them. - import collect_summary # noqa: E402 - import render_report # noqa: E402 - sd = os.path.join(tmp, "S") - os.makedirs(os.path.join(sd, "ancestry_fine")) - for name in ("S_ancestry_fine.tsv", "S_painting.png"): - with open(os.path.join(rep, name), "rb") as src, open(os.path.join(sd, "ancestry_fine", name), "wb") as dst: - dst.write(src.read()) - summ = collect_summary.collect("S", sd) - txt, html = render_report.text_report(summ), render_report.html_report(summ) - for kind, doc in (("text", txt), ("HTML", html)): - expect(f"report: the {kind} report names the group left out", - "Left out of the model: Oceania (continental and regional group). The panel has fewer than 10" in doc, True) - expect(f"report: the {kind} report names the chromosome not painted", "not painted: RFMix failed on chr19" in doc, True) os.remove(os.path.join(paint, "failed.txt")) os.remove(os.path.join(paint, "chr20.msp.tsv")) r = og("report", "--sample", "S", "--model", model, "--admixture", radm, "--paint", paint, "--panel", "test", diff --git a/tests/test_ancient_report.py b/tests/test_ancient_report.py deleted file mode 100644 index aacc2535b..000000000 --- a/tests/test_ancient_report.py +++ /dev/null @@ -1,287 +0,0 @@ -#!/usr/bin/env python3 -# og-leg: OG_REQUIRED -"""og ancient (step 44) on synthetic files, run as the pipeline runs it. - -Checks: - 1. ranking: on a synthetic .evec the nearest ancient group is the one whose - mean position is closest on the weighted PCs, a group far on PC1 but - with a small eigenvalue on PC3 is ranked by the weighted distance, and - each row carries its dates (oldest first, AADR style BCE), country and - member count; - 2. axes: a run whose PC2 has the opposite sign is turned back before the - person is placed, and a run whose present-day coordinates differ stops - with the reason instead of placing the person on other axes; - 3. liftover: lift keeps a site inside a block, a site on a minus-strand - chain with its alleles complemented, and drops a site in a gap, one - whose alleles miss the GRCh38 base, chrX and a monomorphic marker; - 4. genotypes: person-geno appends one column, 9 where the person has no - usable genotype, and stops on a genotype file shorter than the .snp file; - 5. select: the present-day reference people come from MODERN_GROUPS, the - ancient ones need a date, a passing assessment, enough sites and a place - in West Eurasia, one genetic id per individual; - 6. the report command writes the table, the summary, the text and an SVG - with the person marked; - 7. bin/render_report.py shows them: an HTML card with the table and the - plot, and a section of the text report. - -The Rust unit tests (rust/crates/og-ancestry, og-intervals chain) check the -date labels, the dosages and the single-position lift in detail, and -tests/parity checks each command against the Python it replaced. Without an -og binary the test is skipped with a note; with OG_REQUIRED set the skip is a -failure. - -Run: OG=bin/og python3 tests/test_ancient_report.py -""" -import os -import shutil -import subprocess -import sys -import tempfile - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -OG = os.environ.get("OG", "") -FAILS = 0 -CAVEAT = "Closeness on a PCA is genetic similarity, not descent" - -if not OG: - print("skip: OG is not set (OG=bin/og)") - sys.exit(1 if os.environ.get("OG_REQUIRED") else 0) - - -def check(desc, ok, detail=""): - global FAILS - print(f"[{'PASS' if ok else 'FAIL'}] {desc}{'' if ok else ' -- ' + str(detail)[:600]}") - if not ok: - FAILS += 1 - - -def write(path, text): - with open(path, "w") as f: - f.write(text) - return path - - -def og(*args): - return subprocess.run([OG, "ancient", *args], capture_output=True, text=True) - - -# --- 1. ranking ------------------------------------------------------------------- -# Present-day people M0..M19 build the axes; three ancient groups. -EVALS = [9.0, 4.0, 0.01, 0.01] - - -def evec(rows, evals=EVALS): - out = ["#eigvals: " + " ".join(f"{v:.3f}" for v in evals)] - for i, (label, pcs) in rows.items(): - out.append(f"{i} " + " ".join(f"{v:.6f}" for v in pcs) + f" {label}") - return "\n".join(out) + "\n" - - -MODERNS = {f"M{i}": ("French" if i % 2 else "Sardinian", - [i * 0.01 - 0.1, (i % 5) * 0.02 - 0.04, (i % 3) * 0.01 - 0.01, (i % 4) * 0.01 - 0.015]) - for i in range(20)} -PANEL = dict(MODERNS) -PANEL.update({ - "A1": ("Ancient", [0.10, 0.10, 0.0, 0.0]), "A2": ("Ancient", [0.12, 0.10, 0.0, 0.0]), # Steppe - "B1": ("Ancient", [-0.20, 0.00, 0.0, 0.0]), # Farmer - "C1": ("Ancient", [0.00, 0.00, 0.9, 0.0]), # close on PC1/PC2, far on PC3 (eigenvalue 0.01) -}) -ANCIENTS = { - "A1": {"id": "A1", "group": "Steppe_EBA", "date_bp": "4800", "country": "Russia", "locality": "Samara"}, - "A2": {"id": "A2", "group": "Steppe_EBA", "date_bp": "4600", "country": "Russia", "locality": "Samara"}, - "B1": {"id": "B1", "group": "Anatolia_N", "date_bp": "8200", "country": "Turkey", "locality": "Barcin"}, - "C1": {"id": "C1", "group": "Odd_C", "date_bp": "3000", "country": "Italy", "locality": "Somewhere"}, -} -# The person's run: the present-day people with PC2 flipped, as smartpca may -# write it, and the person. -RUN = {k: (lab, [pcs[0], -pcs[1], pcs[2], pcs[3]]) for k, (lab, pcs) in MODERNS.items()} -RUN["S"] = ("Person", [0.08, -0.09, 0.0, 0.0]) - -tmp = tempfile.mkdtemp() -try: - stats = write(os.path.join(tmp, "stats.tsv"), "panel_sites\t4\nsites_genotyped\t3\nother_allele\t1\nsites_used\t2\n") - at = os.path.join(tmp, "anc.tsv") - with open(at, "w") as f: - f.write("id\tgroup\tdate_bp\tdate_sd\tfull_date\tlocality\tcountry\tlat\tlon\tsnps\n") - for r in ANCIENTS.values(): - f.write(f"{r['id']}\t{r['group']}\t{r['date_bp']}\t50\tx\t{r['locality']}\t{r['country']}\t50\t40\t500000\n") - outs = {k: os.path.join(tmp, f"S_{k}") for k in ("ancient.tsv", "info.tsv", "ancient.txt", "pca.svg")} - - def report(panel_evec, run_evec, with_stats=True): - return og("report", "--panel-evec", panel_evec, "--run-evec", run_evec, "--ancients", at, - "--sample", "S", "--release", "test", *(["--stats", stats] if with_stats else []), - "--out-tsv", outs["ancient.tsv"], "--out-info", outs["info.tsv"], - "--out-txt", outs["ancient.txt"], "--out-svg", outs["pca.svg"]) - - def table(): - with open(outs["ancient.tsv"]) as f: - head, *rows = [line.split("\t") for line in f.read().splitlines()] - return [dict(zip(head, r)) for r in rows] - - pe = write(os.path.join(tmp, "panel.evec"), evec(PANEL)) - re_ = write(os.path.join(tmp, "run.evec"), evec(RUN)) - p = report(pe, re_) - check("report exits 0", p.returncode == 0, p.stderr) - rows = table() if p.returncode == 0 else [] - order = [r["group"] for r in rows] - check("three ancient groups ranked", len(rows) == 3, rows) - check("the nearest group is first (Steppe_EBA), from the flipped run turned back", - order[:1] == ["Steppe_EBA"], order) - check("a group far only on a PC with a tiny eigenvalue is second, not last (Odd_C)", order[1:2] == ["Odd_C"], order) - check("the farthest group is last (Anatolia_N)", order[-1:] == ["Anatolia_N"], order) - st = rows[0] if rows else {} - check("the row has the member count, dates and place", st.get("n") == "2" and st.get("dates") == "2850-2650 BCE" - and st.get("country") == "Russia" and st.get("locality") == "Samara", st) - check("ranks are 1..n", [r["rank"] for r in rows] == ["1", "2", "3"], rows) - flat_pe = write(os.path.join(tmp, "flat_panel.evec"), evec(PANEL, [1.0, 1.0, 1.0, 1.0])) - flat_re = write(os.path.join(tmp, "flat_run.evec"), evec(RUN, [1.0, 1.0, 1.0, 1.0])) - p = report(flat_pe, flat_re) - check("control: without the eigenvalue weights Odd_C falls behind Anatolia_N", - p.returncode == 0 and [r["group"] for r in table()] == ["Steppe_EBA", "Anatolia_N", "Odd_C"], p.stderr) - - # --- 2. axes ---------------------------------------------------------------------- - p = og("align", "--panel-evec", pe, "--run-evec", re_, "--pcs", "2") - check("align: a flipped PC2 is found", p.returncode == 0 and "PC1+ PC2-" in p.stdout, (p.stdout, p.stderr)) - bad = {k: (lab, [pcs[0] * (1 if int(k[1:]) % 3 else -1)] + pcs[1:]) for k, (lab, pcs) in MODERNS.items()} - bad_re = write(os.path.join(tmp, "bad.evec"), evec(bad)) - p = og("align", "--panel-evec", pe, "--run-evec", bad_re, "--pcs", "2") - check("align: other axes stop, naming the PC", p.returncode == 1 and "PC1" in p.stderr, (p.returncode, p.stderr)) - - # --- 3. liftover -------------------------------------------------------------------- - chain = write(os.path.join(tmp, "test.chain"), "\n".join([ - # chr1 source 1000-1100 -> target 2000-2100 (+), gap of 50, then 1150-1200 -> 2150-2200 - "chain 100 chr1 100000 + 1000 1200 chr1 100000 + 2000 2200 1", - "100 50 50", "50", "", - # chr2 source 500-600 -> target on the minus strand of a 10,000 bp chr2, reverse start 300 - "chain 90 chr2 50000 + 500 600 chr2 10000 - 300 400 2", - "100", "", - # chr3: two chains over 100-200 - "chain 80 chr3 50000 + 100 200 chr3 50000 + 100 200 3", "100", "", - "chain 70 chr3 50000 + 150 250 chr3 50000 + 5150 5250 4", "100", ""]) + "\n") - seq1 = ["N"] * 3000 - seq1[2050] = "G" # 1-based 2051 - seq1[2159] = "T" # 1-based 2160 - seq2 = ["N"] * 10000 - seq2[9689] = "C" # 1-based 9690: the complement of the source G - fa = os.path.join(tmp, "ref.fa") - with open(fa, "w") as f, open(fa + ".fai", "w") as fai: - off = 0 - for name, seq in (("chr1", seq1), ("chr2", seq2)): - head = f">{name}\n" - f.write(head) - off += len(head) - s = "".join(seq) - for i in range(0, len(s), 60): - f.write(s[i:i + 60] + "\n") - fai.write(f"{name}\t{len(s)}\t{off}\t60\t61\n") - off += len(s) + (len(s) + 59) // 60 - snp = write(os.path.join(tmp, "src.snp"), "\n".join([ - "rs1 1 0.0 1051 G A", # ref G at target: kept - "rs2 1 0.0 1160 C T", # second allele is the target base: kept - "rs3 1 0.0 1120 A G", # gap: dropped - "rs4 2 0.0 511 G T", # minus strand: alleles C/A, target C: kept - "rs5 1 0.0 1051 A C", # neither allele is G: dropped - "rs6 23 0.0 1051 G A", # X: dropped - "rs7 1 0.0 1051 G X", # monomorphic marker: dropped - ]) + "\n") - out = os.path.join(tmp, "sites.tsv") - p = og("lift", "--snp", snp, "--chain", chain, "--fasta", fa, "--out", out) - got = open(out).read().splitlines() if os.path.exists(out) else [] - check("lift keeps rs1, rs2 and the complemented rs4, in GRCh38 order", - p.returncode == 0 and got == ["snp\tchrom\tpos\tallele1\tallele2", "rs1\tchr1\t2051\tG\tA", - "rs2\tchr1\t2160\tC\tT", "rs4\tchr2\t9690\tC\tA"], (p.returncode, got, p.stderr)) - - # --- 4. genotypes ----------------------------------------------------------------- - msnp = write(os.path.join(tmp, "moderns.snp"), - "rs1 1 0.0 1051 G A\nrs2 1 0.0 1160 C T\nrsX 23 0.0 5 A G\nrs4 2 0.0 511 G T\n") - mgeno = write(os.path.join(tmp, "moderns.geno"), "012\n220\n999\n101\n") - gts = write(os.path.join(tmp, "gts.tsv"), "chr1\t2051\tG\tA\tG/A\nchr1\t2160\tT\t.\tT/T\nchr2\t9690\tC\tG\tC/G\n") - merged = os.path.join(tmp, "merged.geno") - pstats = os.path.join(tmp, "pstats.tsv") - p = og("person-geno", "--sites", out, "--genotypes", gts, "--snp", msnp, "--moderns-geno", mgeno, - "--out", merged, "--stats", pstats) - got = open(merged).read().splitlines() if os.path.exists(merged) else [] - check("person-geno appends one column: 1, 0, 9 for X, 9 for an allele outside the pair", - p.returncode == 0 and got == ["0121", "2200", "9999", "1019"], (p.returncode, got, p.stderr)) - st = dict(line.split("\t") for line in open(pstats).read().splitlines()) if os.path.exists(pstats) else {} - check("its counts", st == {"panel_sites": "4", "sites_genotyped": "3", "other_allele": "1", "sites_used": "2"}, st) - short = write(os.path.join(tmp, "short.geno"), "012\n220\n") - p = og("person-geno", "--sites", out, "--genotypes", gts, "--snp", msnp, "--moderns-geno", short, - "--out", merged, "--stats", pstats) - check("a genotype file shorter than the .snp file stops", p.returncode == 1, (p.returncode, p.stderr)) - - # --- 5. select ------------------------------------------------------------------------ - head = ["Genetic ID (long text)", "Persistent Genetic ID", "Individual ID", '"Date mean in BP in years before 1950"', - "Date standard deviation in BP [x]", "Full Date One of two formats", "Group ID", "Locality", - "Political Entity", "Latitude", "Longitude", "Data type", - "SNPs hit on autosomal targets (Computed using easystats on 1240k snpset)", - "SNPs hit on autosomal targets (Computed using easystats on HO snpset)", "ASSESSMENT", - "ASSESSMENT WARNINGS (long)"] - - def arow(gid, ind, bp, group, lat, lon, dtype, snps, ass): - return [gid, gid, ind, str(bp), "50", "x", group, "Loc", "Land", str(lat), str(lon), dtype, str(snps), "0", ass, ""] - - rows_ = [arow(f"F{i}.DG", f"F{i}", 0, "French", 46, 2, "Shotgun.diploid", 1100000, "PROVISIONAL_PASS") - for i in range(60)] - rows_ += [ - arow("F0.SG", "F0", 0, "French", 46, 2, "Shotgun.diploid", 900000, "PASS"), # same individual, fewer sites - arow("Y1.DG", "Y1", 0, "Yakut", 62, 129, "Shotgun.diploid", 1100000, "PASS"), # not a reference group - arow("H1.HO", "H1", 0, "French", 46, 2, "HumanOrigins", 600000, "PASS"), # not shotgun diploid - arow("A1.AG", "A1", 4800, "Russia_Steppe", 53, 50, "1240k", 500000, "PASS"), - arow("A2.AG", "A2", 4800, "Russia_Steppe", 53, 50, "1240k", 50000, "PASS"), # too few sites - arow("A3.AG", "A3", 4800, "Russia_Steppe", 53, 50, "1240k", 500000, "Questionable"), - arow("A4.AG", "A4", 4800, "China_N", 40, 116, "1240k", 500000, "PASS"), # outside West Eurasia - arow("A5.AG", "A5", 4800, "Unknown", "..", "..", "1240k", 500000, "PASS"), # no place - ] - anno = write(os.path.join(tmp, "x.anno"), "\n".join("\t".join(r) for r in [head] + rows_) + "\n") - ind = write(os.path.join(tmp, "x.ind"), "".join(f"{r[0]} U {r[6]}\n" for r in rows_)) - oind, plist, anc = (os.path.join(tmp, n) for n in ("panel.ind", "poplist.txt", "ancients.tsv")) - p = og("select", "--anno", anno, "--ind", ind, "--out-ind", oind, "--poplist", plist, "--ancients", anc) - rc = p.returncode - labels = dict(line.split("\t")[0::2] for line in open(oind).read().splitlines()) if rc == 0 else {} - check("select: the 60 French shotgun genomes are the reference, one id per person (F0.DG)", - rc == 0 and sum(1 for v in labels.values() if v == "French") == 60 and labels.get("F0.SG") == "Ignore", - (labels, p.stderr)) - check("select: Yakut and a Human Origins genome are ignored", - labels.get("Y1.DG") == "Ignore" and labels.get("H1.HO") == "Ignore", labels) - check("select: only A1 is projected", [k for k, v in labels.items() if v == "Ancient"] == ["A1.AG"], labels) - check("select: the population list", open(plist).read() == "French\n" if rc == 0 else False) - check("select: ancients.tsv holds A1 with its group", - rc == 0 and open(anc).read().splitlines()[1].startswith("A1.AG\tRussia_Steppe\t4800"), - open(anc).read() if rc == 0 else "") - - # --- 6. the report command ---------------------------------------------------------- - p = report(pe, re_) - check("report exits 0", p.returncode == 0, p.stderr) - info = dict(line.split("\t") for line in open(outs["info.tsv"]).read().splitlines()[1:]) if p.returncode == 0 else {} - check("report: the person's PC2 in the panel's sign", info.get("pc2") == "0.09000", info) - check("report: the sites used", info.get("sites_used") == "2", info) - svg = open(outs["pca.svg"]).read() if p.returncode == 0 else "" - check("report: the SVG is an SVG with a dark background and the person's label", - svg.startswith("S" in svg and svg.rstrip().endswith(""), svg[:200]) - txt = open(outs["ancient.txt"]).read() if p.returncode == 0 else "" - check("report: the text carries the caveat and says qpAdm is not run", CAVEAT in txt and "qpAdm" in txt, txt) - # --- 7. both reports read the outputs ---------------------------------------------- - sd = os.path.join(tmp, "S") - os.makedirs(os.path.join(sd, "ancient")) - for k, f in outs.items(): - shutil.copy(f, os.path.join(sd, "ancient", f"S_{k.replace('pca.svg', 'ancient_pca.svg').replace('info.tsv', 'ancient_info.tsv')}")) - html_out, txt_out = os.path.join(tmp, "S.html"), os.path.join(tmp, "S.txt") - p = subprocess.run([sys.executable, os.path.join(REPO, "bin", "render_report.py"), "--sample", "S", "--sample-dir", sd, - "--json", os.path.join(tmp, "summary.json"), "-o", html_out, "-o", txt_out], - capture_output=True, text=True) - check("render_report exits 0", p.returncode == 0, p.stderr) - page = open(html_out).read() if os.path.exists(html_out) else "" - check("the HTML report has the card, its first group and the plot as an image", - "

    Ancient Populations

    " in page and "Steppe_EBA" in page and "data:image/svg+xml;base64," in page, page[-400:]) - text = open(txt_out).read() if os.path.exists(txt_out) else "" - check("the text report has the section with the nearest group and the caveat", - "Ancient Populations (AADR PCA, step 44)" in text and "1. Steppe_EBA (" in text and "not descent" in text, text[-600:]) - - p = report(pe, pe, with_stats=False) - check("report: a run without the person stops", p.returncode == 1 and "no row for S" in p.stderr, p.stderr) -finally: - shutil.rmtree(tmp) - -print(f"\n{FAILS} failure(s)") -sys.exit(1 if FAILS else 0) diff --git a/tests/test_archaic_report.py b/tests/test_archaic_report.py deleted file mode 100644 index 030f74e97..000000000 --- a/tests/test_archaic_report.py +++ /dev/null @@ -1,592 +0,0 @@ -#!/usr/bin/env python3 -# og-leg: OG_REQUIRED -"""og archaic (step 43) on a synthetic archaic panel, with the real IBDmix image. - -Checks: - 1. table: a 1000 Genomes site keeps its frequencies and gets each archaic - genome's ALT count, 10 plus it outside that genome's mask or inside an - excluded region; an archaic record with another ALT or another REF is - unknown (9); a site only an archaic genome varies at gets frequency 0; a - site no archaic genome was called at, or only called as reference with - no modern variation, is left out; a position given twice by 1000 Genomes - is left out; - 1b. strict: a lifted row outside the GRCh38 strict mask is kept once per - position with every archaic call marked masked (9 stays 9), not dropped; - 2. genotypes: a masked archaic call (10 to 12) goes to IBDmix as its ALT - count and its position into that genome's mask file; the stand-in - columns carry the population's ALT count; your column is your ALT - count, 9 for a non-PASS record, and where your VCF has no record 0 - inside a reference block of the gVCF and 9 (unknown) outside every - one; the span runs over the sites read; a site where your VCF has another ALT is left out; a - chromosome your VCF has no record on is not analysed; in a split VCF an - SNV keeps its call beside an insertion, a multi-allelic record or the - 0/0 half of a split site, in either order, a usable record of the same - SNV wins over a filtered one, and two carried SNVs at one position are - unknown (9); - 3. report: Altai and Vindija segments merge into Neanderthal ones, a - Denisova segment that overlaps one is not counted as Denisovan, segments - under 50 kb or LOD 4 are dropped, the share is of the analysed span, and - the genes of the longest segments are named; og summary collect - reads the summary and stops on a damaged one, and bin/render_report.py - prints both shares; - 4. with IBDmix itself (IBDMIX_IMAGE, docker): a planted 600 kb stretch where - the synthetic person carries the Neanderthal haplotype comes out as one - Neanderthal segment over it, and a planted Denisovan one as Denisovan; - the same person without the planted stretches gets no segment at all; - where the person contradicts both Neanderthals inside the planted - stretch and their calls there are masked, the segment stops at the - masked mismatches, while the same calls written as unknown (9, the - table before the fix) let it run through; a stretch where the person - has no call matches archaic genomes that carry the common reference - allele, and gives a segment only when a gVCF reference block says the - stretch was read (the step before that fix read it as the reference - either way). - Set ARCHAIC_TEST_NO_DOCKER=1 to skip part 4 on a machine without docker; - CI never sets it. - -The Rust unit tests (rust/crates/og-ancestry) check pick_call's symmetry and -which genes touch a segment's ends, and tests/parity checks each command -against the Python it replaced. Without an og binary the test is skipped with -a note; with OG_REQUIRED set the skip is a failure. - -Run: OG=bin/og python3 tests/test_archaic_report.py -""" -import gzip -import os -import random -import shutil -import subprocess -import sys -import tempfile - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -OG = os.environ.get("OG", "") -FAILS = 0 -TABLE_HEADER = ["chrom", "pos", "ref", "alt", "Altai", "Vindija", "Denisova", - "AF_ALL", "AF_AFR", "AF_AMR", "AF_EAS", "AF_EUR", "AF_SAS"] -SEGMENT_HEADER = ["chrom", "start", "end", "length_bp", "ancestry", "matched", "lod", "genes"] -MASKED = 10 -ARCHAICS = ("Altai", "Vindija", "Denisova") - -if not OG: - print("skip: OG is not set (OG=bin/og)") - sys.exit(1 if os.environ.get("OG_REQUIRED") else 0) - - -def check(desc, ok, detail=""): - global FAILS - print(f"[{'PASS' if ok else 'FAIL'}] {desc}{'' if ok else ' -- ' + str(detail)[:600]}") - if not ok: - FAILS += 1 - - -def write(path, text): - with open(path, "w") as f: - f.write(text) - - -def run(*args): - return subprocess.run([OG, "archaic", *args], capture_output=True, text=True) - - -def image(var): - for line in open(os.path.join(REPO, "versions.env")): - if line.startswith(f"{var}="): - return line.split("=", 1)[1].split("#")[0].strip().strip('"') - raise SystemExit(f"{var} not in versions.env") - - -TMP = tempfile.mkdtemp() - -# --- 1. table ------------------------------------------------------------------ -afs = "0.1\t0.2\t0.05\t0\t0.15\t0.1" -write(f"{TMP}/modern.tsv", - f"100\tA\tG\t{afs}\n" # every archaic genome called - f"200\tC\tT\t{afs}\n" # Vindija has another ALT, Denisova another REF - f"300\tG\tA\t{afs}\n" # no archaic record - f"400\tT\tC\t{afs}\n400\tT\tG\t{afs}\n") # a split multi-allelic site -write(f"{TMP}/altai.tsv", "100\tA\tG\t1/1\n200\tC\t.\t0/0\n500\tG\tC\t0/1\n600\tA\t.\t0/0\n") -write(f"{TMP}/vindija.tsv", "100\tA\t.\t0/0\n200\tC\tA\t0/1\n500\tG\t.\t0/0\n600\tA\t.\t0/0\n") -write(f"{TMP}/denisova.tsv", "100\tA\tG\t0|1\n200\tT\tC\t0/1\n400\tT\tC\t1/1\n500\tG\tT\t1/1\n") -r = run("table", "--chrom", "22", "--modern", f"{TMP}/modern.tsv", "--archaic", f"Altai={TMP}/altai.tsv", - "--archaic", f"Vindija={TMP}/vindija.tsv", "--archaic", f"Denisova={TMP}/denisova.tsv", "--out", f"{TMP}/t.vcf") -check("table exits 0", r.returncode == 0, r.stderr) -recs = {int(l.split("\t")[1]): l.rstrip("\n").split("\t") for l in open(f"{TMP}/t.vcf") if not l.startswith("#")} -info = {p: dict(kv.split("=") for kv in c[7].split(";")) for p, c in recs.items()} -check("rows kept: the modern sites with an archaic call and the archaic-only site", sorted(recs) == [100, 200, 500], sorted(recs)) -check("a 1000 Genomes site: each genome's ALT count", (info[100]["Altai"], info[100]["Vindija"], info[100]["Denisova"]) == ("2", "0", "1"), info.get(100)) -check("its frequencies are kept", info[100]["AF_ALL"] == "0.1" and info[100]["AF_EUR"] == "0.15", info.get(100)) -check("another ALT or another REF is unknown (9)", (info[200]["Altai"], info[200]["Vindija"], info[200]["Denisova"]) == ("0", "9", "9"), info.get(200)) -check("an archaic-only site takes the first non-reference ALT and frequency 0", - recs[500][4] == "C" and info[500]["Altai"] == "1" and info[500]["Vindija"] == "0" and info[500]["Denisova"] == "9" - and info[500]["AF_ALL"] == "0", (recs.get(500), info.get(500))) -check("chrom gets its chr prefix for the liftover", recs[100][0] == "chr22", recs[100][0]) - -# The masks: each archaic genome is read only inside its own FilterBed. The -# first run is the control: without --mask the same inputs keep every call, -# so the codes below come from the mask and nothing else. -write(f"{TMP}/altai.bed", "22\t50\t150\n22\t450\t550\n") # 1-based 51-150 and 451-550: 100 and 500, not 200 -write(f"{TMP}/vindija.bed", "22\t150\t250\n") # holds 200, not 100 -write(f"{TMP}/denisova.bed.gz", "") -with gzip.open(f"{TMP}/denisova.bed.gz", "wt") as f: - f.write("chr22\t99\t100\n") # 1-based 100 only, and named chr22 -common = ["table", "--chrom", "22", "--modern", f"{TMP}/modern.tsv", "--archaic", f"Altai={TMP}/altai.tsv", - "--archaic", f"Vindija={TMP}/vindija.tsv", "--archaic", f"Denisova={TMP}/denisova.tsv"] - - -def table_info(out, extra=()): - r = run(*common, *extra, "--out", out) - check(f"table exits 0 ({'masked' if extra else 'no mask'})", r.returncode == 0, r.stderr) - rows = {int(l.split("\t")[1]): l.rstrip("\n").split("\t") for l in open(out) if not l.startswith("#")} - return rows, {p: dict(kv.split("=") for kv in c[7].split(";")) for p, c in rows.items()} - - -nm_rows, nm = table_info(f"{TMP}/nomask.vcf") -check("control, no mask: the Altai call at 100 is read, and 200 and 500 are rows", - nm[100]["Altai"] == "2" and sorted(nm_rows) == [100, 200, 500], (sorted(nm_rows), nm.get(100))) -m_rows, mi = table_info(f"{TMP}/masked.vcf", ("--mask", f"Altai={TMP}/altai.bed", - "--mask", f"Vindija={TMP}/vindija.bed", - "--mask", f"Denisova={TMP}/denisova.bed.gz")) -check("inside its mask a genome keeps its call", mi[100]["Altai"] == "2" and mi[100]["Denisova"] == "1", mi.get(100)) -# Outside its mask a call is kept as masked (10 plus its ALT count), as Chen -# et al. keep the site in IBDmix's input and mask it there; the old table -# coded it 9 (unknown), which let IBDmix's segments run through it. -check("outside its mask a genome's call is masked (10 for 0/0), where the control read 0", - mi[100]["Vindija"] == "10" and nm[100]["Vindija"] == "0", (mi.get(100), nm.get(100))) -check("a row whose only call is masked stays, with that call masked (200: Altai 0/0 is 10)", - sorted(m_rows) == [100, 200, 500] and mi[200]["Altai"] == "10" and nm[200]["Altai"] == "0", - (sorted(m_rows), mi.get(200))) -check("an archaic-only site inside its mask keeps that genome's ALT, and the other genome's call is masked", - m_rows[500][4] == "C" and mi[500]["Altai"] == "1" and mi[500]["Vindija"] == "10", (m_rows.get(500), mi.get(500))) -check("unknown stays 9 inside or outside a mask", - mi.get(200, {}).get("Vindija") == "9" and mi.get(500, {}).get("Denisova") == "9", (mi.get(200), mi.get(500))) -check("a mask file is read with and without the chr prefix", mi[100]["Denisova"] == "1", mi.get(100)) -# The excluded regions: every call at a site inside one is masked, whatever -# the genome's own mask says; the sites outside are untouched. -write(f"{TMP}/excl.bed", "22\t90\t120\tsegdup\n") -ex_rows, ex = table_info(f"{TMP}/excl.vcf", ("--exclude", f"{TMP}/excl.bed")) -check("every call at a site inside an excluded region is masked (100: 12, 10, 11), where the control read 2, 0, 1", - (ex[100]["Altai"], ex[100]["Vindija"], ex[100]["Denisova"]) == ("12", "10", "11") - and (nm[100]["Altai"], nm[100]["Vindija"], nm[100]["Denisova"]) == ("2", "0", "1"), (ex.get(100), nm.get(100))) -check("and the sites outside it keep their calls", sorted(ex_rows) == [100, 200, 500] - and (ex[200]["Altai"], ex[500]["Altai"]) == ("0", "1"), (sorted(ex_rows), ex.get(200), ex.get(500))) -write(f"{TMP}/excl_other.bed", "7\t90\t120\tsegdup\n") -r = run(*common, "--exclude", f"{TMP}/excl_other.bed", "--out", f"{TMP}/x.vcf") -check("an exclusion list with no interval on the chromosome stops the build", - r.returncode == 1 and "no interval" in r.stderr, r.stderr) -r = run(*common, "--mask", f"Altai={TMP}/altai.bed", "--mask", f"Altai={TMP}/altai.bed", "--out", f"{TMP}/x.vcf") -check("--mask twice for one genome stops the build", r.returncode == 2 and "twice" in r.stderr, r.stderr) -r = run(*common, "--mask", f"Nobody={TMP}/altai.bed", "--out", f"{TMP}/x.vcf") -check("--mask with an unknown genome stops the build", r.returncode == 2, r.stderr) -write(f"{TMP}/other.bed", "7\t50\t150\n") -r = run(*common, "--mask", f"Altai={TMP}/other.bed", "--out", f"{TMP}/x.vcf") -check("a mask with no interval on the chromosome stops the build, instead of emptying the table", - r.returncode == 1 and "no interval" in r.stderr, r.stderr) - -# --- 1b. strict: rows outside the GRCh38 strict mask are marked, not dropped ------- -# Lifted rows (13 columns, no header), sorted by position as Picard writes -# them. Strict mask 1-based 101-300 on chr22: 100 is outside, 150 and 300 -# inside, 301 outside. A duplicate position keeps the first row. Codes 0, 1, 2 -# outside become 10, 11, 12; 9 stays 9; a code already masked is not marked -# twice. -rows = ["chr22\t100\tA\tG\t2\t0\t9\t0.5\t0\t0\t0\t0\t0", - "chr22\t150\tA\tG\t1\t1\t1\t0.5\t0\t0\t0\t0\t0", - "chr22\t300\tG\tA\t0\t0\t0\t0.5\t0\t0\t0\t0\t0", - "chr22\t301\tC\tT\t0\t10\t12\t0.5\t0\t0\t0\t0\t0", - "chr22\t301\tC\tT\t2\t2\t2\t0.5\t0\t0\t0\t0\t0"] -write(f"{TMP}/lifted.tsv", "\n".join(rows) + "\n") -write(f"{TMP}/strict.bed", "chr22\t100\t300\n") -r = run("strict", "--chrom", "22", "--strict", f"{TMP}/strict.bed", "--rows", f"{TMP}/lifted.tsv", "--out", f"{TMP}/marked.tsv") -check("strict exits 0", r.returncode == 0, r.stderr) -got = [l.rstrip("\n").split("\t") for l in open(f"{TMP}/marked.tsv")] -by = {int(c[1]): c[4:7] for c in got} -check("every position is kept, once", [int(c[1]) for c in got] == [100, 150, 300, 301], [c[1] for c in got]) -check("outside the strict mask 2, 0 and unknown become 12, 10 and 9", by[100] == ["12", "10", "9"], by.get(100)) -check("inside the strict mask the calls are untouched (last base 300 too)", by[150] == ["1", "1", "1"] and by[300] == ["0", "0", "0"], (by.get(150), by.get(300))) -check("a masked call stays as it is, and the first of a duplicate wins", by[301] == ["10", "10", "12"], by.get(301)) -write(f"{TMP}/unsorted.tsv", "\n".join([rows[0], rows[3], rows[2]]) + "\n") -r = run("strict", "--chrom", "22", "--strict", f"{TMP}/strict.bed", "--rows", f"{TMP}/unsorted.tsv", "--out", f"{TMP}/marked3.tsv") -check("rows out of position order stop the step (a duplicate is found only next to its twin)", - r.returncode == 2 and "sorted" in r.stderr and not os.path.exists(f"{TMP}/marked3.tsv"), (r.returncode, r.stderr)) -write(f"{TMP}/strict_other.bed", "chr21\t0\t10\n") -r = run("strict", "--chrom", "22", "--strict", f"{TMP}/strict_other.bed", "--rows", f"{TMP}/lifted.tsv", "--out", f"{TMP}/marked2.tsv") -check("a strict mask with no interval on the chromosome stops the step", r.returncode != 0 and "no interval" in r.stderr, r.stderr) - -# --- 2. genotypes ------------------------------------------------------------------ -tdir = f"{TMP}/table" -os.makedirs(tdir) -hdr = "\t".join(TABLE_HEADER) + "\n" -rows = ["chr22\t1000\tA\tG\t2\t2\t0\t0.5\t0.5\t0.5\t0.5\t0.5\t0.5", - "chr22\t2000\tC\tT\t0\t0\t0\t0.01\t0\t0\t0\t0.02\t0", - "chr22\t3000\tG\tA\t1\t1\t2\t0\t0\t0\t0\t0\t0", - "chr22\t4000\tT\tC\t2\t9\t0\t0.2\t0\t0\t0\t0\t0", - "chr22\t5000\tA\tC\t0\t0\t0\t0.2\t0\t0\t0\t0\t0"] -with gzip.open(f"{tdir}/chr22.tsv.gz", "wt") as f: - f.write(hdr + "\n".join(rows) + "\n") -with gzip.open(f"{tdir}/chr21.tsv.gz", "wt") as f: - f.write(hdr + "chr21\t1000\tA\tG\t2\t2\t0\t0.5\t0.5\t0.5\t0.5\t0.5\t0.5\n") -vcf = ("##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" - "chr22\t1000\t.\tA\tG\t50\tPASS\t.\tGT:GQ\t0/1:30\n" - "chr22\t3000\t.\tG\tA\t50\tRefCall\t.\tGT\t1/1\n" - "chr22\t5000\t.\tA\tT\t50\tPASS\t.\tGT\t1/1\n") -write(f"{TMP}/s.vcf", vcf) -GVCF_HEAD = "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" - - -def ref_block(chrom, start, end): - return f"{chrom}\t{start}\t.\tN\t<*>\t0\t.\tEND={end}\tGT:GQ:MIN_DP\t0/0:50:30\n" - - -# A reference block over 1-2500: 2000 was read as the reference; 4000 was not read. -write(f"{TMP}/s.g.vcf", GVCF_HEAD + ref_block("chr22", 1, 2500)) -r = run("genotypes", "--table-dir", tdir, "--vcf", f"{TMP}/s.vcf", "--gvcf", f"{TMP}/s.g.vcf", - "--pop", "EUR", "--pseudo", "10", "--out-dir", f"{TMP}/gt") -check("genotypes exits 0", r.returncode == 0, r.stderr) -lines = gzip.open(f"{TMP}/gt/chr22.gt.gz", "rt").read().splitlines() -g = {int(l.split("\t")[1]): l.split("\t") for l in lines[1:]} -check("header: four columns, three archaic, SAMPLE, ten stand-ins", - lines[0].split("\t")[:8] == ["chrom", "pos", "ref", "alt", "Altai", "Vindija", "Denisova", "SAMPLE"] - and len(lines[0].split("\t")) == 18, lines[0]) -check("the chromosome is an integer for IBDmix", g[1000][0] == "22", g[1000][0]) -check("your heterozygous call is 1", g[1000][7] == "1", g[1000]) -check("no record inside a gVCF reference block is 0 (reference)", g[2000][7] == "0", g[2000]) -check("no record outside every reference block is 9 (unknown)", g[4000][7] == "9", g[4000]) -check("a non-PASS record is 9", g[3000][7] == "9", g[3000]) -check("a site where you carry another ALT is left out", 5000 not in g, sorted(g)) -check("stand-ins carry EUR's ALT count (0.5 of 20 alleles = 10)", sum(int(x) for x in g[1000][8:]) == 10, g[1000][8:]) -check("and 0.02 of 20 rounds to 0", sum(int(x) for x in g[2000][8:]) == 0, g[2000][8:]) -check("each stand-in column is one character", all(len(x) == 1 for x in g[1000][4:]), g[1000]) -check("a chromosome the VCF has no record on is not analysed", not os.path.exists(f"{TMP}/gt/chr21.gt.gz")) -spans = open(f"{TMP}/gt/spans.tsv").read().splitlines() -check("spans: the sites read, chr22 from 1000 to 2000, two sites", spans[1].split("\t")[:4] == ["chr22", "1000", "2000", "2"], spans) -samples = open(f"{TMP}/gt/samples.txt").read().split() -check("samples.txt: SAMPLE and the stand-ins", samples[0] == "SAMPLE" and len(samples) == 11, samples) -write(f"{TMP}/two.vcf", "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tA\tB\n") -r = run("genotypes", "--table-dir", tdir, "--vcf", f"{TMP}/two.vcf", "--out-dir", f"{TMP}/gt2") -check("a two-sample VCF stops the step", r.returncode != 0 and "one-sample" in r.stderr, r.stderr) -# One position twice, as bcftools norm -m- writes it. Rows: REF>ALT of the table. -sdir2 = f"{TMP}/table_split" -os.makedirs(sdir2) -split_rows = {1000: "A\tG", 2000: "C\tT", 3000: "G\tA", 4000: "T\tC", 5000: "A\tG", 6000: "C\tT", - 7000: "G\tA", 8000: "T\tC", 9000: "A\tC"} -with gzip.open(f"{sdir2}/chr22.tsv.gz", "wt") as f: - f.write(hdr + "".join(f"chr22\t{p}\t{ra}\t0\t0\t0\t0.2\t0\t0\t0\t0\t0\n" for p, ra in split_rows.items())) -write(f"{TMP}/split.vcf", "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" - "chr22\t1000\t.\tA\tAT\t50\tPASS\t.\tGT\t0/1\n" # insertion, then the SNV - "chr22\t1000\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\n" - "chr22\t2000\t.\tC\tT\t50\tPASS\t.\tGT\t1/1\n" # the SNV, then an insertion - "chr22\t2000\t.\tC\tCA\t50\tPASS\t.\tGT\t0/1\n" - "chr22\t3000\t.\tG\tA,GT\t50\tPASS\t.\tGT\t1/2\n" # a multi-allelic record, then the SNV - "chr22\t3000\t.\tG\tA\t50\tPASS\t.\tGT\t0/1\n" - "chr22\t4000\t.\tT\tC\t50\tPASS\t.\tGT\t0/1\n" # two SNVs, both carried - "chr22\t4000\t.\tT\tG\t50\tPASS\t.\tGT\t0/1\n" - "chr22\t4000\t.\tT\tC\t50\tPASS\t.\tGT\t0/1\n" - "chr22\t5000\t.\tA\tT\t50\tPASS\t.\tGT\t0/0\n" # a split 0/1 at A>G,T: 0/0, then 0/1 - "chr22\t5000\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\n" - "chr22\t6000\t.\tC\tT\t50\tPASS\t.\tGT\t0/1\n" # 0/1, then the 0/0 half - "chr22\t6000\t.\tC\tG\t50\tPASS\t.\tGT\t0/0\n" - "chr22\t7000\t.\tG\tA\t50\tLowQual\t.\tGT\t0/1\n" # the same SNV, filtered, then usable - "chr22\t7000\t.\tG\tA\t50\tPASS\t.\tGT\t1/1\n" - "chr22\t8000\t.\tT\tA\t50\tPASS\t.\tGT\t0/0\n" # two 0/0 halves: the reference - "chr22\t8000\t.\tT\tG\t50\tPASS\t.\tGT\t0/0\n") -r = run("genotypes", "--table-dir", sdir2, "--vcf", f"{TMP}/split.vcf", "--pseudo", "10", "--out-dir", f"{TMP}/gt3") -check("genotypes on a split VCF exits 0", r.returncode == 0, r.stderr) -g3 = {int(l.split("\t")[1]): l.split("\t") for l in gzip.open(f"{TMP}/gt3/chr22.gt.gz", "rt").read().splitlines()[1:]} -code3 = lambda p: g3.get(p, ["left out"] * 8)[7] -check("an insertion before the SNV at one position keeps the SNV call", code3(1000) == "1", g3.get(1000)) -check("an insertion after the SNV at one position keeps the SNV call", code3(2000) == "2", g3.get(2000)) -check("a multi-allelic record beside the SNV keeps the SNV call", code3(3000) == "1", g3.get(3000)) -check("two carried SNVs at one position are unknown (9), also when a third record follows", code3(4000) == "9", g3.get(4000)) -check("a 0/0 half of a split site before the carried SNV keeps the SNV call", code3(5000) == "1", g3.get(5000)) -check("a 0/0 half after the carried SNV keeps the SNV call", code3(6000) == "1", g3.get(6000)) -check("a usable record of the same SNV wins over a filtered one", code3(7000) == "2", g3.get(7000)) -check("two 0/0 halves are the reference (0)", code3(8000) == "0", g3.get(8000)) - -# Masked calls: IBDmix gets the call itself and a mask per genome of the -# positions where it is masked, the runs of written rows closed by the next -# unmasked one. Codes per row: Altai, Vindija, Denisova. -mdir = f"{TMP}/table_masked" -os.makedirs(mdir) -mrows = {100: ("A\tG", "12\t0\t9"), 200: ("C\tT", "10\t11\t0"), 300: ("G\tA", "2\t0\t0"), - 400: ("T\tC", "10\t0\t0"), 500: ("A\tC", "0\t0\t0"), 600: ("C\tT", "11\t0\t0")} -with gzip.open(f"{mdir}/chr22.tsv.gz", "wt") as f: - f.write(hdr + "".join(f"chr22\t{p}\t{ra}\t{c}\t0.2\t0\t0\t0\t0\t0\n" for p, (ra, c) in mrows.items())) -write(f"{TMP}/m.vcf", "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" - "chr22\t500\t.\tA\tT\t50\tPASS\t.\tGT\t0/1\n") # another ALT: row 500 is left out -write(f"{TMP}/m.g.vcf", GVCF_HEAD + ref_block("chr22", 1, 1000)) -r = run("genotypes", "--table-dir", mdir, "--vcf", f"{TMP}/m.vcf", "--gvcf", f"{TMP}/m.g.vcf", "--pseudo", "10", - "--out-dir", f"{TMP}/gtm") -check("genotypes on a table with masked calls exits 0", r.returncode == 0, r.stderr) -gm = {int(l.split("\t")[1]): l.split("\t")[4:7] for l in gzip.open(f"{TMP}/gtm/chr22.gt.gz", "rt").read().splitlines()[1:]} -check("IBDmix gets each masked call as its ALT count (12, 10, 11 are 2, 0, 1)", - gm.get(100) == ["2", "0", "9"] and gm.get(200) == ["0", "1", "0"] and gm.get(600) == ["1", "0", "0"], gm) -masks = {g: open(f"{TMP}/gtm/chr22.{g}.mask.bed").read() if os.path.exists(f"{TMP}/gtm/chr22.{g}.mask.bed") else None - for g in ARCHAICS} -check("Altai's mask: 100-200, then 400-600 (the left-out row 500 does not close it)", - masks["Altai"] == "22\t99\t200\n22\t399\t600\n", masks["Altai"]) -check("Vindija's mask: 200 alone", masks["Vindija"] == "22\t199\t200\n", masks["Vindija"]) -check("Denisova's mask is written and empty", masks["Denisova"] == "", masks["Denisova"]) -check("an unmasked table writes empty masks", open(f"{TMP}/gt/chr22.Altai.mask.bed").read() == "") -bdir = f"{TMP}/table_bad" -os.makedirs(bdir) -with gzip.open(f"{bdir}/chr22.tsv.gz", "wt") as f: - f.write(hdr + "chr22\t100\tA\tG\t7\t0\t0\t0.2\t0\t0\t0\t0\t0\n") -r = run("genotypes", "--table-dir", bdir, "--vcf", f"{TMP}/m.vcf", "--out-dir", f"{TMP}/gtb") -check("a code the table never holds (7) stops the step", r.returncode == 1 and "code '7'" in r.stderr, r.stderr) - -# A streaming join must refuse backwards positions, including records past -# the table's end and records whose genotype it would otherwise ignore. -vcf_header = "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" -for label, records in ( - ("within-table", [(3000, "G", "A", "GT"), (1000, "A", "G", "GT")]), - ("past-table", [(9000, "A", "G", "GT"), (8000, "A", "G", "GT")]), - ("ignored-call", [(3000, "G", "A", "GT"), (1000, "A", ".", "GQ")])): - bad_vcf = f"{TMP}/{label}.vcf" - write(bad_vcf, vcf_header + "".join( - f"chr22\t{p}\t.\t{ref}\t{alt}\t.\tPASS\t.\t{fmt}\t0/1\n" - for p, ref, alt, fmt in records)) - out = f"{TMP}/order-{label}" - os.makedirs(out) - protected = ["chr22.gt.gz", *(f"chr22.{g}.mask.bed" for g in ARCHAICS)] - for name in protected: - write(f"{out}/{name}", "previous output\n") - r = run("genotypes", "--table-dir", tdir, "--vcf", bad_vcf, "--out-dir", out) - check(f"unsorted VCF {label} exits 2", r.returncode == 2 and "not sorted" in r.stderr, r.stderr) - check(f"unsorted VCF {label} keeps prior genotype and masks", - all(open(f"{out}/{name}", "rb").read() == b"previous output\n" for name in protected)) - -unordered_table = f"{TMP}/table_unordered" -os.makedirs(unordered_table) -with gzip.open(f"{unordered_table}/chr22.tsv.gz", "wt") as f: - f.write(hdr + "\n".join([rows[1], rows[0], *rows[2:]]) + "\n") -r = run("genotypes", "--table-dir", unordered_table, "--vcf", f"{TMP}/s.vcf", - "--out-dir", f"{TMP}/gt_unordered") -check("an unsorted table exits 2", r.returncode == 2 and "not sorted" in r.stderr, r.stderr) -check("an unsorted table leaves no partial genotype or mask", - not any(name.endswith((".gt.gz", ".mask.bed")) for name in os.listdir(f"{TMP}/gt_unordered"))) - -# --- 3. report --------------------------------------------------------------------- -idir = f"{TMP}/ibd" -os.makedirs(idir) -head = "ID\tchrom\tstart\tend\tslod\n" -write(f"{idir}/chr22.Altai.txt", head + "SAMPLE\t22\t1000000\t1300000\t40\nP001\t22\t5000000\t9000000\t99\n" - "SAMPLE\t22\t2000000\t2040000\t30\nSAMPLE\t22\t3000000\t3100000\t3.5\n") -write(f"{idir}/chr22.Vindija.txt", head + "SAMPLE\t22\t1200000\t1500000\t50\n") -write(f"{idir}/chr22.Denisova.txt", head + "SAMPLE\t22\t1400000\t1450000\t12\nSAMPLE\t22\t6000000\t6200000\t20\n") -write(f"{TMP}/spans.tsv", "chrom\tfirst\tlast\tsites\nchr22\t1\t10000000\t5000\n") -write(f"{TMP}/genes.gtf", - 'chr22\tHAVANA\tgene\t1100000\t1110000\t.\t+\t.\tgene_id "G1"; gene_type "protein_coding"; gene_name "GENEA";\n' - 'chr22\tHAVANA\tgene\t1120000\t1130000\t.\t+\t.\tgene_id "G2"; gene_type "lncRNA"; gene_name "LNC1";\n' - 'chr22\tHAVANA\tgene\t6100000\t6300000\t.\t-\t.\tgene_id "G3"; gene_type "protein_coding"; gene_name "GENEB";\n') -r = run("report", "--sample", "S1", "--ibdmix-dir", idir, "--spans", f"{TMP}/spans.tsv", "--genes", f"{TMP}/genes.gtf", - "--out-dir", f"{TMP}/out") -check("report exits 0", r.returncode == 0, r.stderr) -segs = [l.split("\t") for l in open(f"{TMP}/out/S1_archaic_segments.tsv").read().splitlines()] -check("segments header", segs[0] == SEGMENT_HEADER, segs[0]) -body = segs[1:] -check("two segments: the merged Neanderthal one and the Denisovan one", len(body) == 2, body) -check("Altai 1.0-1.3 and Vindija 1.2-1.5 Mb merge into Neanderthal 1.0-1.5 Mb", - body and body[0][:6] == ["chr22", "1000000", "1500000", "500000", "Neanderthal", "Altai,Vindija,Denisova"], body[:1]) -check("the merged segment keeps the higher LOD", body and body[0][6] == "50.0", body[:1]) -check("a Denisova stretch inside a Neanderthal one is not Denisovan; one apart is", len(body) > 1 and body[1][:5] == ["chr22", "6000000", "6200000", "200000", "Denisovan"], body) -check("protein-coding genes only", body and body[0][7] == "GENEA", body[:1]) -check("a stand-in's segment is not yours, nor one under 50 kb or LOD 4", all(r[1] not in ("5000000", "2000000", "3000000") for r in body), body) -summ = dict(l.split("\t", 1) for l in open(f"{TMP}/out/S1_archaic.tsv").read().splitlines()[1:]) -check("Neanderthal share of the 10 Mb span: 5.00%", summ.get("neanderthal_percent") == "5.00", summ) -check("Denisovan share: 2.00%", summ.get("denisovan_percent") == "2.00", summ) -check("segment counts", summ.get("neanderthal_segments") == "1" and summ.get("denisovan_segments") == "1", summ) -check("the longest segment is listed first with its genes", summ.get("longest_1", "").startswith("chr22:1000000-1500000 Neanderthal") - and "GENEA" in summ.get("longest_1", ""), summ.get("longest_1")) -txt = open(f"{TMP}/out/S1_archaic.txt").read() -check("the text names the caveat", "An estimate, not a measurement." in txt, txt[-300:]) -r = run("report", "--sample", "S1", "--ibdmix-dir", f"{TMP}/nothing", "--spans", f"{TMP}/spans.tsv", "--out-dir", f"{TMP}/out2") -check("a missing IBDmix output stops the report", r.returncode != 0 and "no IBDmix output" in r.stderr, r.stderr) - -# --- 3b. the reports read it ------------------------------------------------------ -import collect_summary as cs # noqa: E402 -import render_report as rr # noqa: E402 - - -def archaic(d): - """The archaic section og summary collect writes for folder D.""" - return cs.collect("S1", d)["sections"]["archaic"] - - -sdir = f"{TMP}/sample" -os.makedirs(f"{sdir}/archaic") -shutil.copy(f"{TMP}/out/S1_archaic.tsv", f"{sdir}/archaic/S1_archaic.tsv") -sec = archaic(sdir) -data = sec["data"] -check("og summary collect reads the summary", sec["state"] == "ok" and data.get("neanderthal_percent") == "5.00" - and data.get("denisovan_segments") == 1 and len(data.get("longest", [])) == 2, sec) -check("the archaic section is in SECTIONS as step 43", any(x[0] == "archaic" and x[2] == "43" for x in cs.SECTIONS) - and sec["step"] == "43") -lines = rr.archaic_lines(data) -check("the report lines give both shares", lines[0].startswith("Neanderthal: 5.00% (0.5 Mb in 1 segments)") - and lines[1].startswith("Denisovan: 2.00%"), lines) -write(f"{sdir}/archaic/S1_archaic.tsv", "key\tvalue\nneanderthal_percent\tlots\n") -check("a damaged summary is unreadable, not a share", archaic(sdir)["state"] == "unreadable", archaic(sdir)) -write(f"{sdir}/archaic/S1_archaic.tsv", "key\tvalue\nneanderthal_segments\n") -check("a row cut after its key is unreadable, not a crash", archaic(sdir)["state"] == "unreadable", archaic(sdir)) -nowhere = archaic(f"{TMP}/nowhere") -check("no file: the section is missing", (nowhere["state"], nowhere["data"]) == ("missing", {}), nowhere) - -# --- 4. the real IBDmix ------------------------------------------------------------- -NEA = (2000000, 2600000) -DEN = (3200000, 3600000) - - -MID = (2200000, 2400000) - - -def synthetic(d, plant, mid=None): - """A 4 Mb chr22 table and one person. A third of the sites carry an allele - the Neanderthals share and modern people rarely do, a third one only the - Denisovan carries, a third are common. The person draws both copies from - the frequencies, except inside the planted stretches, where one copy - carries the archaic allele. - - mid: the middle of the planted Neanderthal stretch (MID) holds sites the - person contradicts (0/0 where both Neanderthals are 1/1), and the two - Neanderthals' calls there are outside their masks. "masked" writes them - as this table does (10 plus the call); "unknown" as the table before the - fix did (9).""" - rnd = random.Random(11) - os.makedirs(f"{d}/table") - rows, calls = [], [] - for i, pos in enumerate(range(1000000, 5000000, 1000)): - kind = i % 3 - if kind == 0: - f, al, vi, de = rnd.uniform(0.01, 0.04), 2, 2, 0 - elif kind == 1: - f, al, vi, de = rnd.uniform(0.01, 0.04), 0, 0, 2 - else: - f = rnd.uniform(0.1, 0.9) - al = vi = de = sum(rnd.random() < f for _ in range(2)) - ta, tv = al, vi # the codes the table writes - g = [rnd.random() < f, rnd.random() < f] - if plant and NEA[0] <= pos < NEA[1]: - g[0] = al == 2 - if mid and MID[0] <= pos < MID[1]: - if kind == 0: - g = [False, False] - ta, tv = (al + MASKED, vi + MASKED) if mid == "masked" else (9, 9) - rows.append(f"chr22\t{pos}\tA\tG\t{ta}\t{tv}\t{de}\t" + "\t".join([f"{f:.4f}"] * 6)) - if plant and DEN[0] <= pos < DEN[1]: - g[0] = de == 2 - if any(g): - calls.append(f"chr22\t{pos}\t.\tA\tG\t50\tPASS\t.\tGT\t{'1/1' if all(g) else '0/1'}") - with gzip.open(f"{d}/table/chr22.tsv.gz", "wt") as f: - f.write("\t".join(TABLE_HEADER) + "\n" + "\n".join(rows) + "\n") - write(f"{d}/s.vcf", "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" + "\n".join(calls) + "\n") - - -def ibdmix_run(d, gvcf=None): - if gvcf is None: - gvcf = GVCF_HEAD + ref_block("chr22", 1, 6000000) # every site read - write(f"{d}/s.g.vcf", gvcf) - r = run("genotypes", "--table-dir", f"{d}/table", "--vcf", f"{d}/s.vcf", "--gvcf", f"{d}/s.g.vcf", - "--out-dir", f"{d}/gt") - if r.returncode: - return r - os.makedirs(f"{d}/ibd") - # The flags of scripts/43-archaic.sh. - sh = ("set -e; gzip -dc /w/gt/chr22.gt.gz > /w/gt/chr22.gt; for a in Altai Vindija Denisova; do " - "ibdmix -g /w/gt/chr22.gt -s /w/gt/samples.txt -n $a -r /w/gt/chr22.$a.mask.bed -o /w/ibd/chr22.$a.txt " - "-d 4 -m 1 -a 0.01 -e 0.002 -c 2; done") - r = subprocess.run(["docker", "run", "--rm", "--network", "none", "-u", f"{os.getuid()}:{os.getgid()}", - "-v", f"{d}:/w", image("IBDMIX_IMAGE"), "sh", "-c", sh], capture_output=True, text=True) - if r.returncode: - return r - return run("report", "--sample", "S", "--ibdmix-dir", f"{d}/ibd", "--spans", f"{d}/gt/spans.tsv", "--out-dir", f"{d}/out") - - -if os.environ.get("ARCHAIC_TEST_NO_DOCKER") == "1": - print("[SKIP] part 4: ARCHAIC_TEST_NO_DOCKER=1") -else: - for plant in (True, False): - d = f"{TMP}/real_{plant}" - os.makedirs(d) - os.chmod(d, 0o777) - synthetic(d, plant) - r = ibdmix_run(d) - check(f"IBDmix chain exits 0 ({'planted' if plant else 'control'})", r.returncode == 0, r.stderr + r.stdout) - if r.returncode: - continue - body = [l.split("\t") for l in open(f"{d}/out/S_archaic_segments.tsv").read().splitlines()[1:]] - print(f" segments ({'planted' if plant else 'control'}): {[(b[1], b[2], b[4]) for b in body]}") - if plant: - nea = [b for b in body if b[4] == "Neanderthal"] - den = [b for b in body if b[4] == "Denisovan"] - cover = lambda segs, lo, hi: sum(max(0, min(int(b[2]), hi) - max(int(b[1]), lo)) for b in segs) - check("the planted Neanderthal stretch: one Neanderthal segment covering 90% of it", - len(nea) == 1 and cover(nea, *NEA) >= 0.9 * (NEA[1] - NEA[0]), nea) - check("and no more than 100 kb beyond it", nea and int(nea[0][3]) <= NEA[1] - NEA[0] + 100000, nea) - check("the planted Denisovan stretch: one Denisovan segment covering 90% of it", - len(den) == 1 and cover(den, *DEN) >= 0.9 * (DEN[1] - DEN[0]), den) - else: - check("without the planted stretches: no segment", body == [], body) - - # The masked middle. Chen et al. keep a masked site in IBDmix's input and - # mask it there, so a homozygous mismatch still counts against a segment; - # the table before the fix wrote such calls as unknown (9), and a segment - # ran through them unopposed. "unknown" is that control: the planted - # segment crosses the contradicted middle. "masked" is this table: it - # must not. - cover_mid = {} - for mid in ("unknown", "masked"): - d = f"{TMP}/real_mid_{mid}" - os.makedirs(d) - os.chmod(d, 0o777) - synthetic(d, True, mid) - r = ibdmix_run(d) - check(f"IBDmix chain exits 0 (middle {mid})", r.returncode == 0, r.stderr + r.stdout) - if r.returncode: - continue - body = [l.split("\t") for l in open(f"{d}/out/S_archaic_segments.tsv").read().splitlines()[1:]] - print(f" segments (middle {mid}): {[(b[1], b[2], b[4]) for b in body]}") - cover_mid[mid] = sum(max(0, min(int(b[2]), MID[1]) - max(int(b[1]), MID[0])) - for b in body if b[4] == "Neanderthal") - span = MID[1] - MID[0] - check("control, calls written as unknown (9): the Neanderthal segment runs through the contradicted middle", - cover_mid.get("unknown", 0) >= 0.9 * span, cover_mid) - check("calls written as masked: IBDmix's mask keeps the mismatches, and no segment covers the middle", - "masked" in cover_mid and cover_mid["masked"] <= 0.1 * span, cover_mid) - - # A stretch with no call (lin-01 of the 1.0.0 review). 150 sites where - # all three archaic genomes carry the reference allele and modern people - # carry the ALT at 0.9, and a person with one call far away: read as the - # reference, the stretch matches the archaic genomes. Control: with a - # reference block over it (the person was read and matches the - # reference) IBDmix calls a segment there. With no block over it, - # nothing was read, and no segment may come out. - gap_lo, gap_hi = 200000, 349000 - gap_segment = {} - for covered in (True, False): - d = f"{TMP}/real_gap_{covered}" - os.makedirs(f"{d}/table") - os.chmod(d, 0o777) - with gzip.open(f"{d}/table/chr22.tsv.gz", "wt") as f: - f.write("\t".join(TABLE_HEADER) + "\n" + "".join( - f"chr22\t{p}\tC\tT\t0\t0\t0\t" + "\t".join(["0.9"] * 6) + "\n" - for p in range(gap_lo, gap_hi + 1, 1000))) - write(f"{d}/s.vcf", GVCF_HEAD + "chr22\t10000000\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\n") - # Without the block the person has one read site left, at the start. - block = ref_block("chr22", 1, 400000) if covered else ref_block("chr22", 1, gap_lo) - r = ibdmix_run(d, GVCF_HEAD + block) - check(f"IBDmix chain exits 0 (stretch {'read' if covered else 'with no call'})", r.returncode == 0, - r.stderr + r.stdout) - if r.returncode: - continue - body = [l.split("\t") for l in open(f"{d}/out/S_archaic_segments.tsv").read().splitlines()[1:]] - print(f" segments (stretch {'read' if covered else 'with no call'}): {[(b[1], b[2], b[4]) for b in body]}") - gap_segment[covered] = sum(int(b[3]) for b in body if b[4] == "Neanderthal") - check("control: a stretch read as the reference gives a Neanderthal segment of 100 kb or more", - gap_segment.get(True, 0) >= 100000, gap_segment) - check("a stretch with no call and no reference block gives no segment (it scored 149 kb when read as the reference)", - gap_segment.get(False, 1) == 0, gap_segment) - -shutil.rmtree(TMP, ignore_errors=True) -print(f"{FAILS} check(s) failed" if FAILS else "all checks passed") -sys.exit(1 if FAILS else 0) diff --git a/tests/test_callability.py b/tests/test_callability.py deleted file mode 100644 index f085d245f..000000000 --- a/tests/test_callability.py +++ /dev/null @@ -1,268 +0,0 @@ -#!/usr/bin/env python3 -# og-leg: OG_REQUIRED -"""The gene callability of step 16b and the ALT contig refusal. - -assets/callability_genes.tsv lists the genes short reads often cannot place -(scripts/ci/build-callability.sh writes it; the e2e case -tests/e2e/callability-1-catalog.sh rebuilds it and compares). Step 16b counts, -per gene, the bases with 10 or more reads of MAPQ 20 or more -(_callability.bed.gz, mosdepth's thresholds table), and -og summary collect (rust/crates/og-summary) turns that into the report's section. - 1. the catalog: its header, one row per gene, known reasons, an HGNC id per - row, a position on chr1-22, X or Y unless ALT-only, and owners as the - build script's rule gives them (assets/paralog_loci.tsv, - assets/pgx_genes.tsv, bin/cn_loci.tsv); - 2. a gene whose reads all had MAPQ 0 (no base at 10x of MAPQ >= 20) is not - read; a unique-sequence gene with every base at 10x is read; an ALT-only - gene is never read, whatever the table holds. Control: a reader that - takes an ALT-only gene as read (the planted bug) fails this check; - 3. the count: the report's list of genes not read and with no owner has as - many genes as a query of the catalog gives (rows with owner '.', minus - the ones the table reads), in the summary and in both reports. Control: - a reader that drops the ALT-only genes from that list fails it; - 4. a table from another catalog (a gene missing) makes the section - unreadable, not a reading; a chromosome with no row at all (a BAM - whose reference lacks it, as the e2e fixture's) puts its genes under - "not on this reference"; - 5. og cn check-bam and call --bam-header refuse a header with an ALT or - an HLA contig (exit 4, a REFUSED line) and pass a header without one - (the positive control). They need og: OG names the binary; without it - part 5 is skipped, and with OG_REQUIRED set too the test fails. - -Run: OG=rust/target/release/og python3 tests/test_callability.py -""" -import csv -import gzip -import os -import shutil -import subprocess -import sys -import tempfile - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -import collect_summary # noqa: E402 -import render_report # noqa: E402 - -CATALOG = os.path.join(REPO, "assets", "callability_genes.tsv") -COLUMNS = ["gene", "hgnc_id", "chrom", "start", "end", "reasons", "owner"] -REASONS = {"CMRG", "segdup", "false_duplication", "collapsed", "ALT-only"} -CHROMS = {f"chr{c}" for c in list(range(1, 23)) + ["X", "Y"]} -FAILS = 0 - - -def check(desc, ok, detail=""): - global FAILS - print(f"[{'PASS' if ok else 'FAIL'}] {desc}" + ("" if ok else f" -- {str(detail)[:400]}")) - if not ok: - FAILS += 1 - - -def rows_of(path): - with open(path) as f: - return list(csv.DictReader((x for x in f if not x.startswith("#")), delimiter="\t", quoting=csv.QUOTE_NONE)) - - -def table_of(path): - with open(path) as f: - return list(csv.DictReader((x for x in f if x.strip() and not x.startswith("#")), delimiter="\t", - quoting=csv.QUOTE_NONE)) - - -def write_thresholds(path, catalog, zero=(), drop=()): - """mosdepth's thresholds table over the catalog's genes: every base at - 10x, except the genes in zero (none) and the ones in drop (no row).""" - with gzip.open(path, "wt") as f: - f.write("#chrom\tstart\tend\tregion\t10X\n") - for r in catalog: - if r["start"] == "." or r["gene"] in drop: - continue - n = int(r["end"]) - int(r["start"]) - f.write(f"{r['chrom']}\t{r['start']}\t{r['end']}\t{r['gene']}\t{0 if r['gene'] in zero else n}\n") - - -# --- 1. the catalog -------------------------------------------------------------------- -with open(CATALOG) as f: - header = next(x for x in f if not x.startswith("#")).rstrip("\n").split("\t") -check("1. the catalog's columns", header == COLUMNS, header) -cat = rows_of(CATALOG) -genes = [r["gene"] for r in cat] -check("1. one row per gene", len(genes) == len(set(genes)) and len(genes) > 1000, len(genes)) -check("1. every row has an HGNC id", all(r["hgnc_id"].startswith("HGNC:") for r in cat)) -check("1. every reason is known", all(set(r["reasons"].split(",")) <= REASONS for r in cat)) -bad = [r["gene"] for r in cat if ("ALT-only" in r["reasons"]) != (r["start"] == ".") - or (r["start"] == "." and not r["chrom"].endswith("_alt")) - or (r["start"] != "." and (r["chrom"] not in CHROMS or not int(r["start"]) < int(r["end"])))] -check("1. a position on chr1-22, X or Y, or an ALT scaffold with no position for an ALT-only gene", not bad, bad) -check("1. 273 CMRG genes", sum("CMRG" in r["reasons"] for r in cat) == 273) -by = {r["gene"]: r for r in cat} -# GSTT1 is a bam gene of step 32, but step 32 skips it on the no-ALT reference and refuses a BAM -# with ALT contigs, so no step calls it: it has no owner. -check("1. GSTT1 and HLA-DRB3 are ALT-only with no owner", - by.get("GSTT1", {}).get("reasons") == "ALT-only" and by["GSTT1"]["owner"] == "." - and by.get("HLA-DRB3", {}).get("reasons") == "ALT-only" and by["HLA-DRB3"]["owner"] == ".") -check("1. KCNE1, CBS and CRYAA are in a false duplication of GRCh38", - all("false_duplication" in by.get(g, {}).get("reasons", "") for g in ("KCNE1", "CBS", "CRYAA"))) -parascopy = {g for r in table_of(os.path.join(REPO, "assets", "paralog_loci.tsv")) for g in r["genes"].split(",")} -pypgx = {r["gene"] for r in table_of(os.path.join(REPO, "assets", "pgx_genes.tsv")) - if r["mode"] == "bam" and r["callable_without_alt"] != "no"} -wrong = [r["gene"] for r in cat if (r["gene"] in parascopy and r["owner"] != "parascopy") - or (r["gene"] not in parascopy and r["gene"] in pypgx and r["owner"] != "pypgx") - or (r["owner"] not in (".", "parascopy", "pypgx") and not r["owner"].startswith("cn_loci:"))] -check("1. owners: parascopy for assets/paralog_loci.tsv's genes, then pypgx for the bam genes step 32 can call", - not wrong, wrong) -check("1. HBA1 and HBA2 belong to the depth engine's HBA row", - by.get("HBA1", {}).get("owner") == "cn_loci:HBA" and by.get("HBA2", {}).get("owner") == "cn_loci:HBA") -check("1. a gene that only holds a depth target is not the depth engine's (RSRP1 holds RHD, RHD is)", - by.get("RSRP1", {}).get("owner") == "." and by.get("RHD", {}).get("owner") == "cn_loci:RHD", - (by.get("RSRP1"), by.get("RHD"))) - - -# --- 2 to 4. the reader ---------------------------------------------------------------- -def og_callability(catalog, table): - """The callability data og summary collect reads from TABLE as step 16b's - table, in a sample folder of its own (og reads the repository's catalog, - which is CATALOG).""" - assert catalog is cat - sd = tempfile.mkdtemp(dir=os.path.dirname(table)) - os.makedirs(os.path.join(sd, "mosdepth")) - shutil.copy(table, os.path.join(sd, "mosdepth", "S_callability.bed.gz")) - sec = collect_summary.collect("S", sd)["sections"]["callability"] - return sec["data"] if sec["state"] == "ok" else {"not_read": [], "not_read_owned": [], "read": 0, - "not_on_reference": [], "state": sec} - - -def alt_read_bug(catalog, table): - """The planted bug: an ALT-only gene taken as read.""" - d = og_callability(catalog, table) - d["not_read"] = [g for g in d["not_read"] if g["status"] != "not on this reference"] - d["read"] += len(d["not_on_reference"]) - return d - - -def readings(d): - return {g["gene"]: g["status"] for g in d["not_read"] + d["not_read_owned"]} - - -def mapq0_and_alt(reader, tsv): - """Check 2 against reader: the MAPQ 0 gene and the ALT-only genes not - read, the unique gene read. [] when it holds, else what is wrong.""" - d = reader(cat, tsv) - got = readings(d) - out = [] - if got.get(MAPQ0) != "not read": - out.append(f"{MAPQ0}: {got.get(MAPQ0)}") - if UNIQUE in got: - out.append(f"{UNIQUE}: {got[UNIQUE]}") - for g in ("GSTT1", "HLA-DRB3"): - if got.get(g) != "not on this reference": - out.append(f"{g}: {got.get(g, 'read')}") - return out - - -def count_matches(reader, tsv): - """Check 3: the unowned genes not read are the catalog query's.""" - d = reader(cat, tsv) - query = [r["gene"] for r in cat if r["owner"] == "." and (r["start"] == "." or r["gene"] == MAPQ0)] - return [g["gene"] for g in d["not_read"]] == query, d - - -MAPQ0 = "KCNE1" # all its reads MAPQ 0, as the false duplication of GRCh38 makes them -UNIQUE = "CBS" # every base at 10x -OWNED = "HBA1" # not read, but step 35's HBA count calls it -with tempfile.TemporaryDirectory() as work: - tsv = os.path.join(work, "S_callability.bed.gz") - write_thresholds(tsv, cat, zero={MAPQ0, OWNED}) - check("2. control: a reader that takes an ALT-only gene as read fails check 2", - bool(mapq0_and_alt(alt_read_bug, tsv)), "the planted bug passed") - problems = mapq0_and_alt(og_callability, tsv) - check("2. a gene of MAPQ 0 reads is not read, a unique one is read, an ALT-only one is never read", - not problems, problems) - check("3. control: the planted bug's list differs from the catalog query", - not count_matches(alt_read_bug, tsv)[0]) - ok, d = count_matches(og_callability, tsv) - check("3. the genes not read with no owner are the catalog query's", ok, d["not_read"][:5]) - check("3. owned genes not read are listed apart (HBA1 by the HBA count), GSTT1 is not one of them", - any(g["gene"] == OWNED and g["owner"] == "cn_loci:HBA" for g in d["not_read_owned"]) - and all(g["gene"] != "GSTT1" for g in d["not_read_owned"]) - and any(g["gene"] == "GSTT1" for g in d["not_read"]), (d["not_read_owned"], d["not_read"][:5])) - - # the section and both reports, from a sample folder (the Nextflow layout) - sd = os.path.join(work, "S") - os.makedirs(os.path.join(sd, "coverage")) - os.replace(tsv, os.path.join(sd, "coverage", "S_callability.bed.gz")) - s = collect_summary.collect("S", sd) - sec = s["sections"]["callability"] - check("3. the section reads the Nextflow folder (coverage/)", sec["state"] == "ok", sec) - n = len(d["not_read"]) - txt, html = render_report.text_report(s), render_report.html_report(s) - line = f"Not read, and no other step calls them ({n}): " - check("3. the text report lists them with the count", line in txt and f"{MAPQ0} (0%)" in txt - and "HLA-DRB3 (not on this reference)" in txt, txt[:600]) - check("3. the HTML report lists them with the count", render_report.E(line) in html - and "Genes This Genome Could Not Read Reliably" in html) - - # 4. a table written from another catalog - os.makedirs(os.path.join(sd, "mosdepth")) - write_thresholds(os.path.join(sd, "mosdepth", "S_callability.bed.gz"), cat, drop={UNIQUE}) - sec = collect_summary.collect("S", sd)["sections"]["callability"] - check("4. a table missing a catalog gene is unreadable", sec["state"] == "unreadable" - and UNIQUE in (sec["note"] or ""), sec) - - # 4b. a BAM whose reference lacks a whole chromosome (as the e2e fixture's): no row there - chr3 = {r["gene"] for r in cat if r["chrom"] == "chr3"} - write_thresholds(os.path.join(sd, "mosdepth", "S_callability.bed.gz"), cat, drop=chr3) - sec = collect_summary.collect("S", sd)["sections"]["callability"] - nr = {g["gene"]: g["status"] for g in (sec.get("data") or {}).get("not_read", []) + (sec.get("data") or {}).get("not_read_owned", [])} - check("4. genes on a chromosome with no row at all are not on this reference, not unreadable", - sec["state"] == "ok" and chr3 and all(nr.get(g) == "not on this reference" for g in chr3), sec.get("state")) - - -# --- 5. the ALT contig refusal --------------------------------------------------------- -OG = os.environ.get("OG", "") -if OG and not os.path.realpath(OG).startswith(os.path.realpath(REPO) + os.sep): - # og cn finds bin/cn_loci.tsv beside its own binary. Guard's unit-tests job - # builds og in a temporary folder, so the tests run a copy that sits next - # to the catalog, as bin/.og/og-* does in a checkout. - import atexit # noqa: E401 - _og_dir = tempfile.mkdtemp(prefix="og-cn-") - atexit.register(shutil.rmtree, _og_dir, True) - shutil.copy2(OG, os.path.join(_og_dir, "og")) - shutil.copy2(os.path.join(REPO, "bin", "cn_loci.tsv"), os.path.join(_og_dir, "cn_loci.tsv")) - OG = os.environ["OG"] = os.path.join(_og_dir, "og") - - -def copy_number(*args, stdin=None): - p = subprocess.run([OG, "cn", *args], capture_output=True, text=True, input=stdin) - return p.returncode, p.stderr - - -if not OG: - print("skip: OG is not set (OG=rust/target/release/og): part 5 did not run") - if os.environ.get("OG_REQUIRED"): - FAILS += 1 -else: - with tempfile.TemporaryDirectory() as work: - noalt = os.path.join(work, "noalt.txt") - with open(noalt, "w") as f: - f.write("@HD\tVN:1.6\tSO:coordinate\n@SQ\tSN:chr16\tLN:90338345\n@SQ\tSN:chrM\tLN:16569\n") - alt = os.path.join(work, "alt.txt") - with open(alt, "w") as f: - f.write("@SQ\tSN:chr16\tLN:90338345\n@SQ\tSN:chr22_KI270879v1_alt\tLN:304135\n@RG\tID:x\tSM:y\n") - rc, err = copy_number("check-bam", "--header", noalt) - check("5. positive control: a header without ALT contigs passes", rc == 0 and not err, (rc, err)) - rc, err = copy_number("check-bam", "--header", alt) - check("5. a header with an ALT contig is refused (exit 4, a REFUSED line naming it)", - rc == 4 and err.startswith("REFUSED:") and "chr22_KI270879v1_alt" in err, (rc, err)) - rc, err = copy_number("check-bam", "--header", "-", stdin="chr1\nHLA-A*01:01:01:01\n") - check("5. contig names on stdin with an HLA contig are refused", rc == 4 and "HLA-A" in err, (rc, err)) - bed = os.path.join(work, "bedcov.tsv") - open(bed, "w").close() - rc, err = copy_number("call", "--locus", "HBA", "--bedcov", bed, "--bedcov-mapq1", bed, "--bam-header", alt) - check("5. call --bam-header refuses before it reads any depth", rc == 4 and err.startswith("REFUSED:"), - (rc, err)) - rc, err = copy_number("call", "--locus", "HBA", "--bedcov", bed, "--bedcov-mapq1", bed, "--bam-header", noalt) - check("5. call --bam-header without ALT contigs goes on (two empty tables: not assessed)", rc == 0, (rc, err)) - -print("\nRESULT:", "ALL PASS" if FAILS == 0 else f"{FAILS} FAILED") -sys.exit(1 if FAILS else 0) diff --git a/tests/test_collect_summary.py b/tests/test_collect_summary.py index 487bff8a4..dd5552867 100644 --- a/tests/test_collect_summary.py +++ b/tests/test_collect_summary.py @@ -62,13 +62,7 @@ counted, the variant rows, SMN1 beside step 35's Parascopy call (agree, disagree, or no line without that table) in both reports, and a folder without the step lists it under Steps Not Run. - 18. step 42's traits table (bin/traits_report.py's own output): the counts - per status leave the APOE rows out, the rows and the APOE lines reach - both reports (the APOE lines last, after the warning, folded away in - the HTML), a row with ancestry yes is marked, a not-covered row is - listed as not read; a table without the reading column, or with a row - cut short, is unreadable; - without the file the section is missing and listed as not run. + 18. step 42's traits table: rust/og/tests/genome_traits_report.rs. 19. step 40's summary table: its counts and '--' reasons are read; a table without a whole-number 'called' row, or with a '--' reason without a whole-number count, is unreadable, not a zero or a reason left out; an @@ -233,13 +227,6 @@ def cdna_table(): "note", "pmids"] -def traits_rows(): - """The rows of assets/traits.tsv, step 42's panel, as dicts.""" - with open(os.path.join(REPO, "assets", "traits.tsv"), encoding="utf-8") as f: - return list(csv.DictReader((x for x in f if x.strip() and not x.startswith("#")), delimiter="\t", - quoting=csv.QUOTE_NONE)) - - def og_tool(*args): """og's exit code for a step tool that once lived in collect_summary.py (og qc sample, og prs table, og mito annotate): they write the tables @@ -1241,54 +1228,10 @@ def run(name, gt): def report_followups_sections(work, schema): - """26. step 42's genotype rules, step 9b's motif table of the other - flagged motif loci and the loci the repeat catalog leaves without - thresholds reach both reports.""" + """26. step 9b's motif table of the other flagged motif loci and the loci + the repeat catalog leaves without thresholds reach both reports (step + 42's genotype rules: rust/og/tests/genome_traits_report.rs).""" import html as htmlmod - # The genotype rules: TAS2R38 homozygous for the taster haplotype PAV - # (G, G, C on the plus strand; rs1726866's G is the reference base). - t = os.path.join(work, "rules", "S") - rows = traits_rows() - g = os.path.join(work, "rules_g.tsv") - put(g, "chr7\t141972804\tT\tC\tC/C\t40\nchr7\t141972905\tG\t.\tG/G\t40\nchr7\t141973545\tC\tG\tG/G\t40\n" - "chr19\t44908684\tT\t.\tT/T\t40\nchr19\t44908822\tC\t.\tC/C\t40\n") - from test_traits_report import traits_tables - os.makedirs(f"{t}/traits", exist_ok=True) - saved_rules = os.path.join(work, "rules_saved.tsv") - traits_tables(g, f"{t}/traits/S_traits.tsv", saved_rules) - summ, txt, html_ = render(t) - check("rules: without the rules table the section has no rule lines and the reports none", - summ["sections"]["traits"]["data"].get("rules") is None and "Genotype rules (several sites" not in txt + html_, - summ["sections"]["traits"]["data"].get("rules")) - shutil.copyfile(saved_rules, f"{t}/traits/S_genotype_rules.tsv") - summ, txt, html_ = render(t) - html_ = htmlmod.unescape(html_) - rl = summ["sections"]["traits"]["data"].get("rules") or [] - check("rules: the TAS2R38 line is read, PAV/PAV, and APOE (in the traits table) is left out", - [(r["rule"], r["result"], r["status"], r["phase"]) for r in rl if r["rule"] in ("TAS2R38", "APOE")] - == [("TAS2R38", "PAV/PAV", "called", "not_needed")], - rl) - import csv - with open(saved_rules) as f: - trait = next(r["trait"] for r in csv.DictReader(f, delimiter="\t") if r["rule"] == "TAS2R38") - check("rules: the text report prints the rule line with its result and sites", - f"{trait} (TAS2R38 rs713598, rs1726866, rs10246939): PAV/PAV (no phase needed). " in txt - and "traits/S_genotype_rules.tsv" in txt, txt) - card = html_.split("

    Traits

    ", 1)[-1].split("

    ", 1)[0] - check("rules: the HTML Traits card has the rules table with the genotypes and the result", - "

    Genotype rules (several sites read together)

    " in card - and "G/G, G/G, C/CPAV/PAV (no phase needed)" in card, card) - check("rules: the APOE lines stay folded away only", "rs429358+rs7412" in card.split("
    ", 1)[-1] - and "APOE" not in card.split("

    Genotype rules", 1)[-1].split("
    ", 1)[0], card) - check("rules: the TAS2R38 rows of the table point at the rule line", - all("genotype rule" in r["note"] for r in rows if r["gene"] == "TAS2R38") - and not any("is not known here" in r["note"] for r in rows if r["gene"] == "TAS2R38"), [r["note"] for r in rows if r["gene"] == "TAS2R38"]) - check("rules: the summary validates", not validate.validate(schema, json.loads(json.dumps(summ))), - validate.validate(schema, json.loads(json.dumps(summ)))) - put(f"{t}/traits/S_genotype_rules.tsv", "rule\tgene\tresult\nTAS2R38\tTAS2R38\tPAV/PAV\n") - check("rules: a rules table without the reading column is unreadable", - collect_summary.collect("S", t)["sections"]["traits"]["state"] == "unreadable") - # The repeats: the catalog step 09 ran on gives MUC1 no thresholds, so # Stranger gives it no status; SAMD12 is flagged and step 9b counted its # motif; NOTINCAT is outside the catalog. @@ -2865,50 +2808,7 @@ def strec(rep, cn, so, ci, adsp, st, nmax="20", pmin="40"): sec_m = collect_summary.collect("S", mo)["sections"]["mosaic_sex_loss"] check("mosaic: an unknown mosaic_call is unreadable", sec_m["state"] == "unreadable" and "mosaic_call" in (sec_m["note"] or ""), sec_m) - # 18. the traits panel (step 42) - t = os.path.join(work, "traits", "S") - rows = traits_rows() - by_id = {r["rsid"]: r for r in rows} - lct, ear = by_id["rs4988235"], by_id["rs17822931"] - g = os.path.join(work, "traits_g.tsv") - put(g, f"{lct['chrom']}\t{lct['pos']}\t{lct['ref']}\t{lct['alt']}\t{lct['ref']}/{lct['alt']}\n" - f"{ear['chrom']}\t{ear['pos']}\t{ear['ref']}\t.\t{ear['ref']}/{ear['ref']}\n" - "chr19\t44908684\tT\t.\tT/T\nchr19\t44908822\tC\t.\tC/C\n") - os.makedirs(f"{t}/traits", exist_ok=True) - from test_traits_report import traits_tables - traits_tables(g, f"{t}/traits/S_traits.tsv", os.path.join(work, "traits_rules.tsv")) - summ_t, txt_t, html_t = render(t) - td = summ_t["sections"]["traits"] - check("traits: section ok, counts without the APOE rows", - td["state"] == "ok" and (td["data"]["called"], td["data"]["not_covered"]) == (2, len(rows) - 4), td) - check("traits: the APOE rows and the APOE line are kept apart", - len(td["data"]["apoe"]) == 3 and not any(r["gene"] == "APOE" for r in td["data"]["rows"]), td["data"]["apoe"]) - check("traits: the text report has the read row with its genotype, reading and PMID, marked ancestry", - f"(MCM6 rs4988235): {lct['ref']}/{lct['alt']} [ancestry]. {lct['het']} (PMID {lct['pmids']})" in txt_t, txt_t) - check("traits: the text report lists a not-covered row as not read", - "Not read (no genotype in the gVCF): " in txt_t and "rs671" in txt_t.split("Not read (no genotype in the gVCF): ", 1)[-1]) - apoe_at = txt_t.find(render_report.APOE_WARNING) - check("traits: the APOE line comes after the warning in the text report, e3/e3", - apoe_at > 0 and "(rs429358+rs7412): e3/e3." in txt_t[apoe_at:] and "e3/e3" not in txt_t[:apoe_at], txt_t) - card_tr = html_t.split("

    Traits

    ", 1)[-1].split("

    ", 1)[0] - check("traits: the HTML card has the row and the ancestry badge", - f"rs4988235{lct['ref']}/{lct['alt']} " in card_tr and 'badge-yellow">ancestry<' in card_tr, card_tr) - folded = card_tr.split("
    ", 1) - check("traits: in the HTML the APOE lines are inside the folded block only", - len(folded) == 2 and "rs429358+rs7412" in folded[1] and "rs429358" not in folded[0], card_tr) - check("traits: the summary validates", not validate.validate(schema, json.loads(json.dumps(summ_t)))) - put(f"{t}/traits/S_traits.tsv", "rsid\tgene\tstatus\nrs1\tX\tcalled\n") - check("traits: a table without the reading column is unreadable", - collect_summary.collect("S", t)["sections"]["traits"]["state"] == "unreadable") - put(f"{t}/traits/S_traits.tsv", "rsid\tgene\ttrait\tgenotype\tstatus\treading\tancestry\tnote\tpmids\n" - "rs1\tX\tT\tA/A\tcalled\n") - check("traits: a row cut short after its status is unreadable, never a called trait without a reading", - collect_summary.collect("S", t)["sections"]["traits"]["state"] == "unreadable") - os.remove(f"{t}/traits/S_traits.tsv") - summ_m, _, html_m = render(t) - check("traits: no file -> missing, listed under Steps Not Run, the card says Not run", - summ_m["sections"]["traits"]["state"] == "missing" and "Traits panel" in summ_m["not_run"] - and ">Not run<" in html_m.split("

    Traits

    ", 1)[-1].split("

    ", 1)[0], summ_m["not_run"]) + # 18. step 42's traits panel: rust/og/tests/genome_traits_report.rs # 19. step 40, the raw-data export e_ = os.path.join(work, "export", "S") put(f"{e_}/consumer_export/S_consumer_export.tsv", "key\tvalue\nsites\t10\ncalled\t7\nno_call\t3\n" diff --git a/tests/test_copy_number.py b/tests/test_copy_number.py deleted file mode 100644 index 92870a44b..000000000 --- a/tests/test_copy_number.py +++ /dev/null @@ -1,1284 +0,0 @@ -#!/usr/bin/env python3 -# og-leg: OG_REQUIRED -"""og cn and bin/cn_loci.tsv: one depth-ratio engine for every copy-number -locus, compared with the pinned historical catalog reader and assess() oracle. - -Without og (OG unset) the checks of the catalog and of assess() run, and the -wiring; with OG naming the og binary (OG=rust/target/release/og, or bin/og) -also those of og cn: the bedcov reader, parity, the CLI, and the blocks of -step 35 cut out and run. With OG_REQUIRED set, a run without og fails. - - 1. the catalog: every row loads with its columns checked; a broken row - (a bad kind, a segment without a span, a class call without an - interval, a reused segment name) is refused; - 2. one owner per locus: no row's targets or controls overlap any region - (main or paralog copy) of a model the pinned Parascopy data holds - (assets/parascopy_models_v1.7.txt), except the pairs EXEMPT lists; a - planted AMY1C row and a planted row on C4A's copy (C4B) are caught; - 3. parity: made-up HBA and LPA depths give, through og cn call, the very - tables the code before the engine wrote (GOLD, frozen from it); - 4. synthetic bedcov tables at 0, 1, 2, 3 and 4 copies of a made-up gene - row are each called right; a MAPQ 0 share above the row's ceiling is - not assessed; a kind=interval row with pooled depths works the same; - 5. a control segment with no read while the other controls are covered - raises BrokenControl, and `call` exits 3 naming BROKEN_CONTROL; with no - read anywhere it is 'not assessed', not an error; - 6. the CLI: `bed` prints every bedcov locus, `call --table` writes step - 46's table; og's catalog reader refuses the broken rows the Python - refuses, with the same message; - 7. the wiring: steps 35, 41 and 46 and their Nextflow modules call og cn, - never the old Python, with the bedcov options of the catalog row. UGT2B17 has no row: pypgx calls it - in BAM mode (assets/pgx_genes.tsv), and tests/test_pypgx_genes.sh - checks that no catalog row names a gene pypgx calls from the BAM; - 8. the gene-dosage rows (HP, LCE3BC, CCL3L1): made-up depths at 0 - to 4 copies are each called right; the block of step 35 that counts - them, cut out and run on tests/fixtures/cn/, writes the known count of - each, and a row shifted by 1 kb no longer matches its fixture; the - PARASCOPY module runs the same list; the CCL3L1 reading names no - disease (a planted 'HIV' is caught); - 9. the interval rows (og dev build cn-intervals, with og): a small ClinGen - BED with a 'chrx' row converts to rows on chrX that load, the same rows - left as 'chrx' are refused, and an unknown contig stops the build; the - committed rows are the ClinGen regions plus the named hand rows, with - no target in a breakpoint row; made-up depths call a 22q11.2 A-D - deletion (1 copy), a 17p12 duplication (3) and 2 for both unaltered; - AZFc on a male at half its DAZ depth is a deletion, AZFa a male's one - copy, every chrY row on a female not applicable and without a sex not - assessed; `intervals` over one bedcov table writes one line per row and - a row with a broken control does not stop the others; the interval - block of step 35, cut out and run, writes that table, and the PARASCOPY - module runs the same commands. - 10. the CYP21A2 row (call psv): its six PSV sites hold the CYP21A2 base of - the reference with the CYP21A1P base at the matching position; the - pileup parser skips read starts, ends and indels; the fixtures of - tests/fixtures/cn/CYP21A2_ (bedcov and mpileup tables, made by - cyp_fixture here) give 2 copies and no finding when unmodified, 1 copy - with the module total down by one for a planted chimeric deletion, and - a ratio of 0.5 at In2G, a possible conversion, for a planted In2G - conversion on one copy; the same depths with the pileup ignored (every - site even) give no finding, so the call rests on the PSV reads; the - block of step 35, cut out and run on the fixtures, writes that table, - and writes not assessed when mpileup fails; the PARASCOPY module runs - the same commands; the couple screen counts the deletion and the - conversion as CYP21A2 findings and the unmodified table as none. - -Run: OG=rust/target/release/og python3 tests/test_copy_number.py -""" -import json -import os -import re -import subprocess -import sys -import tempfile - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -OG = os.environ.get("OG", "") -if OG and not os.path.realpath(OG).startswith(os.path.realpath(REPO) + os.sep): - # og cn finds bin/cn_loci.tsv beside its own binary. Guard's unit-tests job - # builds og in a temporary folder, so the tests run a copy that sits next - # to the catalog, as bin/.og/og-* does in a checkout. - import atexit, shutil # noqa: E401 - _og_dir = tempfile.mkdtemp(prefix="og-cn-") - atexit.register(shutil.rmtree, _og_dir, True) - shutil.copy2(OG, os.path.join(_og_dir, "og")) - shutil.copy2(os.path.join(REPO, "bin", "cn_loci.tsv"), os.path.join(_og_dir, "cn_loci.tsv")) - OG = os.environ["OG"] = os.path.join(_og_dir, "og") -CATALOG = os.path.join(REPO, "bin", "cn_loci.tsv") -MODELS = os.path.join(REPO, "assets", "parascopy_models_v1.7.txt") -sys.path.insert(0, os.path.join(REPO, "bin")) -# The old engine is a test oracle pinned to the traits port's parent. It is -# materialised only for these historical differential assertions. No pipeline -# step imports or ships a second engine. -import importlib.util # noqa: E402 - - -def catalog_oracle(): - ref = "669a8196cf4c377200f9b4af9c235ef85fb774a1" - if subprocess.run(["git", "-C", REPO, "cat-file", "-e", ref + "^{commit}"], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode: - subprocess.run(["git", "-C", REPO, "fetch", "--depth=1", "origin", ref], check=True) - source = subprocess.check_output(["git", "-C", REPO, "show", ref + ":bin/copy_number.py"]) - with tempfile.TemporaryDirectory(prefix="cn-oracle-") as work: - path = os.path.join(work, "copy_number.py") - with open(path, "wb") as f: - f.write(source) - spec = importlib.util.spec_from_file_location("cn_catalog_oracle", path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - module.CATALOG = os.path.join(REPO, "bin", "cn_loci.tsv") - return module - - -cn = catalog_oracle() - -FAILS = 0 -# Frozen from bin/hba_copy_number.py and bin/lpa_kiv2.py as they were before the -# engine (main at 6056452): inputs [depths..., MAPQ 0 share] and the table rows they wrote. -GOLD = json.loads(''.join(( - '{"hba":[[[30,30,30,30,0.25],{"hba2_depth":30,"hba1_depth":30,"flank_left_depth":30,"flank_right_depth":30,"map' - 'q0_share":"0.250","estimate":"4.00","alpha_globin_genes":"4","status":"called","message":"four alpha-globin ge' - 'nes (aa/aa): no deletion seen"}],[[15,30,30,30,0.25],{"hba2_depth":15,"hba1_depth":30,"flank_left_depth":30,"f' - 'lank_right_depth":30,"mapq0_share":"0.250","estimate":"3.00","alpha_globin_genes":"3","status":"called","messa' - 'ge":"three alpha-globin genes: one gene lost on one chromosome (-a/aa, as -a3.7 or -a4.2), a silent carrier; u' - 'nconfirmed: this short-read count has been checked on one real genome only (HG002, four genes), so a count und' - 'er four needs a clinical test before it means anything"}],[[15,15,30,30,0.25],{"hba2_depth":15,"hba1_depth":15' - ',"flank_left_depth":30,"flank_right_depth":30,"mapq0_share":"0.250","estimate":"2.00","alpha_globin_genes":"2"' - ',"status":"called","message":"two alpha-globin genes: alpha-thalassaemia trait, either both genes lost on one ' - "chromosome (--/aa, as --SEA; a child with a partner who carries the same can have Hb Bart's) or one on each (-" - 'a/-a); depth cannot tell these apart; unconfirmed: this short-read count has been checked on one real genome o' - 'nly (HG002, four genes), so a count under four needs a clinical test before it means anything"}],[[45,30,30,30' - ',0.25],{"hba2_depth":45,"hba1_depth":30,"flank_left_depth":30,"flank_right_depth":30,"mapq0_share":"0.250","es' - 'timate":"5.00","alpha_globin_genes":"5","status":"called","message":"5 alpha-globin genes: more than four, a t' - 'riplication (as aaa/aa)"}],[[22.5,30,30,30,0.25],{"hba2_depth":22.5,"hba1_depth":30,"flank_left_depth":30,"fla' - 'nk_right_depth":30,"mapq0_share":"0.250","estimate":"3.50","alpha_globin_genes":".","status":"not assessed","m' - 'essage":"not assessed: the estimate 3.50 lies between 3 and 4 alpha-globin genes"}],[[5,5,6,6,0.25],{"hba2_dep' - 'th":5,"hba1_depth":5,"flank_left_depth":6,"flank_right_depth":6,"mapq0_share":"0.250","estimate":".","alpha_gl' - 'obin_genes":".","status":"not assessed","message":"not assessed: flank depth under 8x (6.0x and 6.0x), too low' - ' to compare"}],[[30,30,30,15,0.25],{"hba2_depth":30,"hba1_depth":30,"flank_left_depth":30,"flank_right_depth":' - '15,"mapq0_share":"0.250","estimate":".","alpha_globin_genes":".","status":"not assessed","message":"not assess' - 'ed: the two flanks disagree (30.0x and 15.0x); a deletion larger than --SEA, or uneven coverage, would skew th' - 'e count"}],[[15.5,25.68,30.11,31.25,0],{"hba2_depth":15.5,"hba1_depth":25.68,"flank_left_depth":30.11,"flank_r' - 'ight_depth":31.25,"mapq0_share":"0.000","estimate":"2.68","alpha_globin_genes":".","status":"not assessed","me' - 'ssage":"not assessed: reads of mapping quality 0 make up 0.0% of the depth over HBA1 and HBA2, so the aligner ' - 'left out the reads that fit both genes (as novoalign does by default) and the count would come out low; realig' - 'n with minimap2 (step 02) or BWA-MEM"}],[[30,30,30,30,0.01],{"hba2_depth":30,"hba1_depth":30,"flank_left_depth' - '":30,"flank_right_depth":30,"mapq0_share":"0.010","estimate":"4.00","alpha_globin_genes":".","status":"not ass' - 'essed","message":"not assessed: reads of mapping quality 0 make up 1.0% of the depth over HBA1 and HBA2, so th' - 'e aligner left out the reads that fit both genes (as novoalign does by default) and the count would come out l' - 'ow; realign with minimap2 (step 02) or BWA-MEM"}],[[0,0,30,30,0],{"hba2_depth":0,"hba1_depth":0,"flank_left_de' - 'pth":30,"flank_right_depth":30,"mapq0_share":"0.000","estimate":"0.00","alpha_globin_genes":"0","status":"call' - 'ed","message":"no alpha-globin gene: the pattern of Hb Bart\'s hydrops fetalis (--/--); unconfirmed: this short' - '-read count has been checked on one real genome only (HG002, four genes), so a count under four needs a clinic' - 'al test before it means anything"}],[[7.4,7.6,30,30,0.25],{"hba2_depth":7.4,"hba1_depth":7.6,"flank_left_depth' - '":30,"flank_right_depth":30,"mapq0_share":"0.250","estimate":"1.00","alpha_globin_genes":"1","status":"called"' - ',"message":"one alpha-globin gene: the pattern of HbH disease (--/-a); unconfirmed: this short-read count has ' - 'been checked on one real genome only (HG002, four genes), so a count under four needs a clinical test before i' - 't means anything"}],[[30.2,29.1,29.4,31.9,0.2],{"hba2_depth":30.2,"hba1_depth":29.1,"flank_left_depth":29.4,"f' - 'lank_right_depth":31.9,"mapq0_share":"0.200","estimate":"3.87","alpha_globin_genes":"4","status":"called","mes' - 'sage":"four alpha-globin genes (aa/aa): no deletion seen"}]],"lpa":[[[12,30,0,0,0.9],{"status":"estimated","ki' - 'v2_copies":"12.0","ci95_low":"12.0","ci95_high":"12.0","class":"low","kiv2_depth":"30.00","flank_depth":"30.00' - '","mapq0_share":"0.900","message":"about 12 KIV-2 repeats in total (95% interval 12.0 to 12.0): fewer KIV-2 re' - 'peats than most people (the whole 95% interval is under 30; about 70% of the 1000 Genomes samples had 30 to 45' - '): a low count goes with small apo(a) isoforms and a higher Lp(a) level"}],[[20,30,0.1,0.15,0.9],{"status":"es' - 'timated","kiv2_copies":"19.8","ci95_low":"17.9","ci95_high":"22.0","class":"low","kiv2_depth":"47.45","flank_d' - 'epth":"30.43","mapq0_share":"0.900","message":"about 20 KIV-2 repeats in total (95% interval 17.9 to 22.0): fe' - 'wer KIV-2 repeats than most people (the whole 95% interval is under 30; about 70% of the 1000 Genomes samples ' - 'had 30 to 45): a low count goes with small apo(a) isoforms and a higher Lp(a) level"}],[[29,30,0.03,0.03,0.9],' - '{"status":"estimated","kiv2_copies":"29.0","ci95_low":"28.3","ci95_high":"29.7","class":"low","kiv2_depth":"66' - '.56","flank_depth":"30.09","mapq0_share":"0.900","message":"about 29 KIV-2 repeats in total (95% interval 28.3' - ' to 29.7): fewer KIV-2 repeats than most people (the whole 95% interval is under 30; about 70% of the 1000 Gen' - 'omes samples had 30 to 45): a low count goes with small apo(a) isoforms and a higher Lp(a) level"}],[[29,30,0.' - '25,0.25,0.9],{"status":"estimated","kiv2_copies":"28.8","ci95_low":"23.5","ci95_high":"35.1","class":"typical"' - ',"kiv2_depth":"67.50","flank_depth":"30.72","mapq0_share":"0.900","message":"about 29 KIV-2 repeats in total (' - '95% interval 23.5 to 35.1): within or overlapping the range most people have (30 to 45 for about 70% of the 10' - '00 Genomes samples)"}],[[47,30,0.25,0.25,0.9],{"status":"estimated","kiv2_copies":"46.6","ci95_low":"38.3","ci' - '95_high":"56.6","class":"typical","kiv2_depth":"106.69","flank_depth":"30.72","mapq0_share":"0.900","message":' - '"about 47 KIV-2 repeats in total (95% interval 38.3 to 56.6): within or overlapping the range most people have' - ' (30 to 45 for about 70% of the 1000 Genomes samples)"}],[[60,30,0.03,0.03,0.9],{"status":"estimated","kiv2_co' - 'pies":"59.9","ci95_low":"58.6","ci95_high":"61.4","class":"high","kiv2_depth":"133.11","flank_depth":"30.09","' - 'mapq0_share":"0.900","message":"about 60 KIV-2 repeats in total (95% interval 58.6 to 61.4): more KIV-2 repeat' - 's than most people (the whole 95% interval is over 45): a high count goes with large apo(a) isoforms and, on a' - 'verage, a lower Lp(a) level"}],[[20,8,0,0,0.9],{"status":"too little depth","kiv2_copies":".","ci95_low":".","' - 'ci95_high":".","class":".","kiv2_depth":"12.57","flank_depth":"8.00","mapq0_share":"0.900","message":"too litt' - 'le depth: 8.0x over the single-copy LPA exons, under the 10x the count needs"}],[[20,30,0,0,0],{"status":"not ' - 'assessed","kiv2_copies":".","ci95_low":".","ci95_high":".","class":".","kiv2_depth":"47.14","flank_depth":"30.' - '00","mapq0_share":"0.000","message":"not assessed: reads of mapping quality 0 make up 0.0% of the depth over t' - 'he KIV-2 exons, so the aligner left out the reads that fit several KIV-2 copies (as novoalign does by default)' - ' and the count would come out low; realign with minimap2 (step 02) or BWA-MEM"}],[[-1.5,30,0,0,0.9],{"status":' - '"not assessed","kiv2_copies":".","ci95_low":".","ci95_high":".","class":".","kiv2_depth":"1.07","flank_depth":' - '"30.00","mapq0_share":"0.900","message":"not assessed: the KIV-2 exons have 1.1x against 30.0x over the single' - '-copy exons, too little for any repeat count (a deletion or a mapping problem)"}],[[38,12,0.3,0.2,0.5],{"statu' - 's":"estimated","kiv2_copies":"38.0","ci95_low":"31.0","ci95_high":"46.5","class":"typical","kiv2_depth":"34.95' - '","flank_depth":"12.23","mapq0_share":"0.500","message":"about 38 KIV-2 repeats in total (95% interval 31.0 to' - ' 46.5): within or overlapping the range most people have (30 to 45 for about 70% of the 1000 Genomes samples)"' - '}],[[-2,30,0,0,0.9],{"status":"not assessed","kiv2_copies":".","ci95_low":".","ci95_high":".","class":".","kiv' - '2_depth":"0.00","flank_depth":"30.00","mapq0_share":"0.000","message":"not assessed: reads of mapping quality ' - '0 make up 0.0% of the depth over the KIV-2 exons, so the aligner left out the reads that fit several KIV-2 cop' - 'ies (as novoalign does by default) and the count would come out low; realign with minimap2 (step 02) or BWA-ME' - 'M"}]]}' -))) - - -def check(desc, ok, detail=""): - global FAILS - print(f"[{'PASS' if ok else 'FAIL'}] {desc}" + ("" if ok else f" -- {str(detail)[:800]}")) - if not ok: - FAILS += 1 - - -HEADER = "\t".join(cn.COLUMNS) - - -def row(**kw): - """A made-up bedcov row: one 1 kb gene between two 10 kb flanks, called - by rounding, with a MAPQ 0 ceiling of 0.5.""" - r = {"locus": "TESTG", "kind": "gene", "chrom": "chr3", "targets": "TESTG:50000-51000", - "controls": "fl_a:30000-40000;fl_b:60000-70000", "details": ".", "target_depth": "sum", - "control_depth": "flanks", "scale": "2", "offset": "0", "bedcov": ".,-Q 1", "mapq0_min": ".", - "mapq0_max": "0.5", "min_control_depth": "8", "max_control_ratio": "1.5", "call": "round:0.3", - "interval": ".", "readings": "0=none|1=one|2=two|more={n} copies", "input": "bedcov", - "owner": "cn-depth", "cross_check": "none", "pmids": "."} - r.update(kw) - return r - - -def write_catalog(path, rows): - with open(path, "w") as f: - f.write(HEADER + "\n") - for r in rows: - f.write("\t".join(r[c] for c in cn.COLUMNS) + "\n") - - -def bedcov(path, prow, depth, exact=False): - """samtools bedcov over a row's segments at these mean depths: whole sums, - or with exact the sums themselves, so each mean comes back as given.""" - with open(path, "w") as f: - for c, s, e, n in cn.regions(prow): - v = depth[n] * (e - s) - f.write(f"{c}\t{s}\t{e}\t{n}\t{v!r}\n" if exact else f"{c}\t{s}\t{e}\t{n}\t{round(v)}\n") - - -def og_cn(*args, **kw): - """og cn with these arguments.""" - return subprocess.run([OG, "cn", *args], capture_output=True, text=True, **kw) - - -def table(path): - """{metric: value} of a metricvalue table, {} when it is not there.""" - if not os.path.exists(path): - return {} - return dict(x.split("\t", 1) for x in open(path).read().splitlines()[1:]) - - -# Step 46's table, in its order (what bin/lpa_kiv2.py wrote). -LPA_ORDER = ["status", "kiv2_copies", "ci95_low", "ci95_high", "class", "kiv2_depth", "flank_depth", "mapq0_share", - "message", "advice"] -# Step 35's interval table (what copy_number.py intervals wrote). -INTERVAL_COLUMNS = ["locus", "name", "chrom", "start", "end", "status", "copies", "expected", "change", "estimate", - "reason"] - - -def change(b): - """The change column of an interval row for an assess() block.""" - if b["copies"] is None or b["expected"] is None: - return "." - return "deletion" if b["copies"] < b["expected"] else "duplication" if b["copies"] > b["expected"] else "no change" - - -# The only rows allowed to overlap a Parascopy model, each for a stated reason. -# A new pair here needs the same: a cross_check in the catalog does not exempt a row. -EXEMPT = { - # Step 41 runs by default and Parascopy is opt-in, so the RhD deletion stays a - # depth row; RHCE's model holds RHD as a paralog copy and is its cross-check. - ("RHD", "RHCE"), -} -DOSAGE = ["HP", "LCE3BC", "CCL3L1"] -# The counts tests/fixtures/cn/.bedcov.tsv was made for. -FIXTURE_COPIES = {"HP": 3, "LCE3BC": 1, "CCL3L1": 5} -# Words a reading of CCL3L1 must not carry: its disease links are disputed. -DENY = re.compile(r"hiv|aids|infect|disease|risk|suscept|progress|malaria|hepatitis|arthritis|lupus|kawasaki|" - r"psoria|diabet|crohn|vaccin", re.I) - - -def overlaps(loc_row, models): - """[(model, segment)] of a parsed row's targets and controls that overlap any - region of a model (1-based, both ends included, against 0-based segments).""" - hits = [] - for n, s, e, _ in loc_row["_targets"] + loc_row["_controls"]: - for name, regs in models.items(): - if any(c == loc_row["chrom"] and s < me and ms - 1 < e for c, ms, me, _ in regs): - hits.append((name, n)) - return hits - - -def owner_violations(cat, models): - return [(loc, m, seg) for loc, r in cat.items() for m, seg in overlaps(r, models) if (loc, m) not in EXEMPT] - - -def main(): - # 1. the catalog - cat = cn.load_catalog(CATALOG) - check("catalog: the six gene and dosage loci and CYP21A2 first, then the interval rows", - list(cat)[:7] == ["HBA", "LPA_KIV2", "RHD"] + DOSAGE + ["CYP21A2"] - and all(r["kind"] == "interval" and r["_passes"] == ["."] for r in list(cat.values())[7:]), list(cat)[:8]) - for loc, r in cat.items(): - check(f"catalog {loc}: owner cn-depth, cross_check named, PMIDs numbers or '.'", - r["owner"] == "cn-depth" and re.fullmatch(r"none|[a-z]+:[A-Za-z0-9_]+(,[a-z]+:[A-Za-z0-9_]+)*", - r["cross_check"]) - and re.fullmatch(r"\.|\d+(,\d+)*", r["pmids"]), r) - if r["input"] == "bedcov": - passes = r["_passes"] - check(f"catalog {loc}: bedcov passes ('.' or samtools options; a second pass adds -Q 1)", - all(re.fullmatch(r"\.|(-[A-Za-z]( \d+)?)( -[A-Za-z]( \d+)?)*", p) for p in passes) - and (len(passes) == 1 or passes[1].replace(".", "").replace("-Q 1", "").strip() - == passes[0].replace(".", "").strip()), passes) - segs = r["_targets"] + r["_controls"] + r["_details"] - check(f"catalog {loc}: every segment is a span of under 1 Mb", all(0 < e - s < 1_000_000 for _, s, e, _ in segs)) - ctl = r["_controls"] - check(f"catalog {loc}: no control overlaps a target", - not any(s < te and ts < e for _, s, e, _ in ctl for _, ts, te, _ in r["_targets"])) - readings = r["_readings"] - mode, vals = r["_call"] - want = {"low", "typical", "high"} if mode == "class" else {str(i) for i in range(int(len(vals) + 1))} \ - if mode == "cut" else None - check(f"catalog {loc}: a reading for every call", want is None or want <= set(readings), readings) - broken = [] - with tempfile.TemporaryDirectory() as t: - p = os.path.join(t, "c.tsv") - for desc, kw, err in (("a bad kind", {"kind": "exon"}, "kind"), - ("a segment with no span", {"targets": "TESTG:51000-50000"}, "empty span"), - ("a class call without an interval", {"call": "class:30,45"}, "needs interval cv"), - ("a MAPQ 0 guard with one bedcov pass", {"bedcov": "."}, "two bedcov passes"), - ("a segment name used twice", {"details": "fl_a:1-2"}, "used twice"), - ("a NaN call threshold", {"call": "round:nan"}, "is not round"), - ("an infinite cut threshold", {"call": "cut:0.5,inf"}, "is not round"), - ("a NaN number column", {"mapq0_max": "nan"}, "not a number")): - write_catalog(p, [row(**kw)]) - try: - cn.load_catalog(p) - got = "loaded" - except cn.CatalogError as e: - got = str(e) - check(f"catalog: {desc} is refused", err in got, got) - broken.append((desc, [row(**kw)], got)) - write_catalog(p, [row(), row(locus="TESTH", targets="TESTG:80000-81000", controls="fl_c:1-2;fl_d:3-4")]) - try: - cn.load_catalog(p) - got = "loaded" - except cn.CatalogError as e: - got = str(e) - check("catalog: one segment name in two loci is refused", "used by two loci" in got, got) - - # 2. one owner per locus - models = {} - with open(MODELS) as f: - for line in f: - if line.startswith("#") or not line.strip(): - continue - name, reg, what = line.split() - c, span = reg.split(":") - s, e = span.split("-") - models.setdefault(name, []).append((c, int(s), int(e), what)) - check("Parascopy model list: 167 models, AMY1C, C4A, CFH, CFHR4, FCGR3A, CHRNA7, APOBEC3A, GYPA, RHCE among them", - len(models) == 167 and {"AMY1C", "C4A", "CFH", "CFHR4", "FCGR3A", "CHRNA7", "APOBEC3A", "GYPA", "RHCE"} - <= set(models), len(models)) - check("Parascopy model list: one main region per model, and the paralog copies (over 1,000 regions in all)", - all(sum(w == "main" for *_, w in r) == 1 and all(w in ("main", "copy") for *_, w in r) - for r in models.values()) and sum(len(r) for r in models.values()) > 1000, - sum(len(r) for r in models.values())) - for loc, r in cat.items(): - for x in r["cross_check"].split(","): - if x.startswith("parascopy:"): - check(f"catalog {loc}: cross_check {x} is a pinned model", x.split(":", 1)[1] in models, x) - v = owner_violations(cat, models) - check("owner: no row counts a gene a pinned Parascopy model owns", not v, v) - stale = [x for x in EXEMPT if x[0] not in cat or x[1] not in {m for m, _ in overlaps(cat[x[0]], models)}] - check("owner: every exemption is still needed (its row overlaps its model)", not stale, stale) - planted = dict(cat) - planted["AMY1"] = cn.parse_row(row(locus="AMY1", chrom="chr1", targets="AMY1C:103745500-103760000", - controls="amy_l:103000000-103010000;amy_r:104000000-104010000")) - v = owner_violations(planted, models) - check("owner: a planted AMY1C row is caught", v == [("AMY1", "AMY1C", "AMY1C")], v) - planted = dict(cat) - planted["C4B"] = cn.parse_row(row(locus="C4B", chrom="chr6", targets="C4B:32014800-32035400", - controls="c4_l:31000000-31010000;c4_r:33500000-33510000")) - v = owner_violations(planted, models) - check("owner: a planted row on C4A's paralog copy (C4B) is caught", v == [("C4B", "C4A", "C4B")], v) - planted = dict(cat) - planted["GYPA"] = cn.parse_row(row(locus="GYPA", chrom="chr4", cross_check="parascopy:GYPA", - targets="GYPA_T:144140000-144150000", - controls="gy_l:140000000-140010000;gy_r:150000000-150010000")) - v = owner_violations(planted, models) - check("owner: naming the model in cross_check does not exempt a row", ("GYPA", "GYPA", "GYPA_T") in v, v) - # DEFB4 has no row: the beta-defensin repeat (DEFB4B to DEFB103B) lies in SPAG11A's paralog copies. - planted = dict(cat) - planted["DEFB4"] = cn.parse_row(row(locus="DEFB4", chrom="chr8", targets="DEFB_UNIT_B:7414854-7430348", - controls="defb_l:7622000-7662000;defb_r:8250000-8290000")) - v = owner_violations(planted, models) - check("owner: a planted DEFB4 row on the beta-defensin repeat is caught (SPAG11A's model)", - v == [("DEFB4", "SPAG11A", "DEFB_UNIT_B")], v) - check("owner: RHD overlaps RHCE's model only (a paralog copy), the exempt pair", - {m for m, _ in overlaps(cat["RHD"], models)} == {"RHCE"}, overlaps(cat["RHD"], models)) - - # 4. synthetic copies, the MAPQ 0 ceiling, a kind=interval row - g = cn.parse_row(row()) - for copies in (0, 1, 2, 3, 4): - m = {"fl_a": 30.0, "fl_b": 30.0, "TESTG": 15.0 * copies} - b = cn.assess(g, m, {n: v * 0.9 for n, v in m.items()}) - check(f"synthetic: {copies} copies called {copies}", (b["status"], b["copies"]) == ("called", copies), b) - m = {"fl_a": 30.0, "fl_b": 30.0, "TESTG": 30.0} - b = cn.assess(g, m, {"fl_a": 30.0, "fl_b": 30.0, "TESTG": 12.0}) - check("synthetic: 60% of the target from MAPQ 0, over the 0.5 ceiling: not assessed", - (b["status"], b["reason"]) == ("not assessed", "high_mapq0_share"), b) - b = cn.assess(g, m, {"fl_a": 30.0, "fl_b": 30.0, "TESTG": 16.0}) - check("synthetic: 47% from MAPQ 0, under the ceiling: called 2", (b["status"], b["copies"]) == ("called", 2), b) - m = {"fl_a": 30.0, "fl_b": 30.0, "TESTG": 37.5} - b = cn.assess(g, m, m) - check("synthetic: 2.5 copies is between counts", b["reason"] == "between_counts", b) - z = cn.parse_row(row(min_control_depth=".")) - try: - b = cn.assess(z, {"fl_a": 0.0, "fl_b": 0.0, "TESTG": 30.0}, {"fl_a": 0.0, "fl_b": 0.0, "TESTG": 30.0}) - got = b["reason"] - except ZeroDivisionError as e: - got = f"ZeroDivisionError: {e}" - check("synthetic: no control depth with no minimum set is low_control_depth, not a crash", - got == "low_control_depth", got) - try: - b = cn.assess(z, {"fl_a": 0.0, "fl_b": 30.0, "TESTG": 30.0}, {"fl_a": 0.0, "fl_b": 30.0, "TESTG": 30.0}) - got = b["reason"] - except ZeroDivisionError as e: - got = f"ZeroDivisionError: {e}" - check("synthetic: a flank with no read, no minimum set, is controls_disagree, not a crash", - got == "controls_disagree", got) - iv = cn.parse_row(row(kind="interval", target_depth="pooled", control_depth="pooled", mapq0_max=".", - bedcov=".", targets="seg1:50000-50500;seg2:50500-51000")) - b = cn.assess(iv, {"fl_a": 20.0, "fl_b": 40.0, "seg1": 45.0, "seg2": 45.0}) - check("interval row: pooled 45x over pooled 30x controls is 3 copies", (b["status"], b["copies"]) == ("called", 3), b) - b = cn.assess(cat["RHD"], {n: 30.0 for *_, n in cn.regions(cat["RHD"])}) - check("a row without a MAPQ 0 guard, given no MAPQ >= 1 depths, reports no MAPQ 0 share", - b["status"] == "called" and b["mapq0_share"] is None, b) - try: - cn.assess(g, m) - got = "no error" - except ValueError as e: - got = str(e) - check("a row with a MAPQ 0 guard, given no MAPQ >= 1 depths, is refused", "MAPQ >= 1 depths are needed" in got, got) - - # 5. a broken control - bad = {"fl_a": 0.0, "fl_b": 30.0, "TESTG": 30.0} - try: - cn.assess(g, bad, bad) - got = "no error" - except cn.BrokenControl as e: - got = str(e) - check("broken control: fl_a with no read beside fl_b at 30x raises BROKEN_CONTROL", - got.startswith("BROKEN_CONTROL: control segment fl_a of TESTG"), got) - b = cn.assess(g, {"fl_a": 0.0, "fl_b": 0.0, "TESTG": 0.0}, {}) - check("no read anywhere: not assessed (too little depth), not an error", - (b["status"], b["reason"]) == ("not assessed", "low_control_depth"), b) - b = cn.assess(g, {"fl_a": 0.0, "fl_b": 5.0, "TESTG": 5.0}, {}) - check("one flank empty beside a thin one (5x): too little depth, not an error", b["reason"] == "low_control_depth", b) - # 7. the wiring - def read(p): - with open(os.path.join(REPO, p)) as f: - return f.read() - for path, loci in (("scripts/35-paralogs.sh", ["HBA"]), ("modules/local/parascopy/main.nf", ["HBA"]), - ("scripts/41-bloodtype.sh", ["RHD"]), ("modules/local/bloodtype/main.nf", ["RHD"]), - ("scripts/46-lpa.sh", ["LPA_KIV2"]), ("modules/local/lpa/main.nf", ["LPA_KIV2"])): - text = read(path) - for loc in loci: - check(f"{path}: og cn bed --locus {loc}", f"og cn bed --locus {loc}" in text) - if loc != "RHD": # step 41's report imports the engine - check(f"{path}: og cn call --locus {loc}", f"og cn call --locus {loc}" in text) - passes = ["" if p == "." else p for p in cat[loc]["_passes"]] - if loc == "HBA": # and the dosage loop's two passes (section 8), the interval pass (section 9) - passes += ["", "-Q 1", "", ""] # and the CYP21A2 passes (section 10) - passes += ["-Q 1"] - # the commands only: options, then a path, a quoted argument or a variable - calls = [o for o, arg in re.findall(r"samtools bedcov((?: -[A-Za-z](?: \d+)?)*) (\S+)", text) - if re.match(r'["$/]|\S+\.bed\b', arg)] - check(f"{path}: its bedcov passes are the catalog's ({' and '.join(repr(p) for p in passes)})", - sorted(c.strip() for c in calls) == sorted(passes), calls) - check(f"{path}: no call of the old Python", - not re.search(r"(copy_number|hba_copy_number|lpa_kiv2)\.py |traits_report\.py --marker", text)) - - dosage(cat, read) - intervals(cat, read) - psv(cat, read) - if OG: - og_engine(cat, broken) - else: - print("skip: OG is not set (OG=rust/target/release/og): the og cn checks did not run") - - print("\nRESULT:", "ALL PASS" if FAILS == 0 else f"{FAILS} FAILED") - return 1 if FAILS or (not OG and os.environ.get("OG_REQUIRED")) else 0 - - -def og_engine(cat, broken): - """Sections 1, 3, 5 and 6 through og cn: its catalog reader, bedcov - reader, the frozen tables, the broken control and the CLI.""" - with tempfile.TemporaryDirectory() as t: - p = os.path.join(t, "c.tsv") - for desc, rows, want in broken: - write_catalog(p, rows) - out = og_cn("bed", "--catalog", p) - check(f"og catalog: {desc} is refused with the Python's message", - out.returncode == 2 and out.stderr == f"ERROR: {want}\n", (out.returncode, out.stderr, want)) - - write_catalog(p, [row()]) - g0 = cn.parse_row(row()) - b = os.path.join(t, "dup.bedcov") - bedcov(b, g0, {"fl_a": 30.0, "fl_b": 30.0, "TESTG": 30.0}) - with open(b) as f: - first = f.readline() - with open(b, "a") as f: - f.write(first.rsplit("\t", 1)[0] + "\t0\n") - out = og_cn("call", "--catalog", p, "--locus", "TESTG", "--bedcov", b, "--bedcov-mapq1", b) - check("bedcov: a region listed twice is refused, not overwritten", - out.returncode == 2 and "appears twice" in out.stderr, out) - with open(b, "w") as f: - f.write(first.split("\t")[0] + "\t50000\t50000\tTESTG\t0\n") - out = og_cn("call", "--catalog", p, "--locus", "TESTG", "--bedcov", b, "--bedcov-mapq1", b) - check("bedcov: an empty region is refused with its name", - out.returncode == 2 and "is empty or reversed" in out.stderr, out) - - # 3. parity with the code before the engine: every row of the frozen - # table, the depths as numbers - def same(got, want): - def eq(k, v): - if isinstance(v, (int, float)): - try: - return abs(float(got.get(k)) - v) < 1e-9 - except (TypeError, ValueError): - return False - return got.get(k) == v - return all(eq(k, v) for k, v in want.items()) - - a, q, tab = (os.path.join(t, x) for x in ("a.tsv", "q.tsv", "t.tsv")) - hrow = cat["HBA"] - for (h2, h1, left, right, share), want in GOLD["hba"]: - m = {"flank_left": left, "HBA2": h2, "HBA1": h1, "flank_right": right} - bedcov(a, hrow, m, exact=True) - bedcov(q, hrow, {n: v * (1 - share) if n in ("HBA2", "HBA1") else v for n, v in m.items()}, exact=True) - out = og_cn("call", "--catalog", CATALOG, "--locus", "HBA", "--bedcov", a, "--bedcov-mapq1", q, - "--table", tab) - got = table(tab) - check(f"parity HBA {h2}/{h1} flanks {left}/{right} MAPQ0 {share}: {want['status']} " - f"{want['alpha_globin_genes']}", out.returncode == 0 and same(got, want), (out.stderr, got, want)) - lrow = cat["LPA_KIV2"] - # in catalog order, as bin/lpa_kiv2.py's REGIONS listed them - kiv = [n for n, *_ in lrow["_targets"]] - fl = [n for n, *_ in lrow["_controls"]] - for (n, flank, ks, fs, share), want in GOLD["lpa"]: - k = flank * (n + 2) / 14 - m = {x: k * (1 + (ks if i % 2 == 0 else -ks)) for i, x in enumerate(kiv)} - m.update({x: flank * (1 + (fs if i % 2 == 0 else -fs)) for i, x in enumerate(fl)}) - bedcov(a, lrow, m, exact=True) - bedcov(q, lrow, {x: v * (1 - share) if x.startswith("kiv2_") else v for x, v in m.items()}, exact=True) - out = og_cn("call", "--catalog", CATALOG, "--locus", "LPA_KIV2", "--bedcov", a, "--bedcov-mapq1", q, - "--table", tab) - got = table(tab) - check(f"parity LPA {n} copies at {flank}x: {want['status']} {want['kiv2_copies']}", - out.returncode == 0 and same(got, want), (out.stderr, got, want)) - - # 5. a broken control - os.remove(tab) - depth = {n: 30.0 for *_, n in cn.regions(lrow)} - depth["flank_e19"] = 0.0 - bedcov(a, lrow, depth) - bedcov(q, lrow, depth) - out = og_cn("call", "--locus", "LPA_KIV2", "--bedcov", a, "--bedcov-mapq1", q, "--table", tab) - check("broken control: `call` exits 3 with BROKEN_CONTROL and writes nothing", - out.returncode == 3 and "BROKEN_CONTROL: control segment flank_e19 of LPA_KIV2" in out.stderr - and not os.path.exists(tab), out) - - # 6. the CLI - out = og_cn("bed") - names = [x.split("\t")[3] for x in out.stdout.splitlines()] - want = sum(len(cn.regions(r)) for r in cat.values() if r["input"] == "bedcov") - check("bed: every segment of the bedcov loci, once (the catalog beside og)", - out.returncode == 0 and len(names) == len(set(names)) == want, (out.returncode, out.stderr, len(names), want)) - depth = {n: 30.0 for *_, n in cn.regions(lrow)} - for n in depth: - if n.startswith("kiv2_"): - depth[n] = 30.0 * 22 / 14 - bedcov(a, lrow, depth) - bedcov(q, lrow, {n: v * (0.1 if n.startswith("kiv2_") else 1) for n, v in depth.items()}) - out = og_cn("call", "--locus", "LPA_KIV2", "--bedcov", a, "--bedcov-mapq1", q, "--table", tab) - tab_rows = table(tab) - check("call --table: step 46's table, 20 KIV-2 repeats, in its order", - out.returncode == 0 and tab_rows.get("status") == "estimated" and tab_rows.get("kiv2_copies") == "20.0" - and list(tab_rows) == LPA_ORDER, (out, tab_rows)) - out = og_cn("call", "--locus", "LPA_KIV2", "--bedcov", a) - check("call: a row with a MAPQ 0 guard needs --bedcov-mapq1", out.returncode == 2 and "--bedcov-mapq1" in out.stderr, - out) - empty = os.path.join(t, "empty.tsv") - open(empty, "w").close() - for loc, key in (("HBA", "alpha_globin_genes"), ("LPA_KIV2", "kiv2_copies")): - if os.path.exists(tab): - os.remove(tab) - out = og_cn("call", "--locus", loc, "--bedcov", empty, "--bedcov-mapq1", empty, "--table", tab) - tab_rows = table(tab) - check(f"call {loc} on two empty tables (no reads on the contig): exit 0, not assessed", - out.returncode == 0 and tab_rows.get("status") == "not assessed" and tab_rows.get(key) == "." - and "no chr" in tab_rows.get("message", ""), (out, tab_rows)) - - -def dosage_depths(r, copies, q1_share=0.5): - """Mean depths of a dosage row at a copy number: controls at 30x (two - copies), the target copies spread evenly over the row's targets (the - reference copies), and the MAPQ >= 1 depths with q1_share of the targets.""" - tg = {n for n, *_ in r["_targets"]} - m = {n: 15.0 * copies / len(tg) if n in tg else 30.0 for *_, n in cn.regions(r)} - return m, {n: v * q1_share if n in tg else v for n, v in m.items()} - - -# run_in and cpath for the dosage block of step 35: og cn runs for real (OG), -# samtools bedcov prints tests/fixtures/cn/[.q1].bedcov.tsv. -STUB = r""" -run_in() { - while [ $# -gt 0 ]; do case $1 in -v|--cpus|--memory) shift 2 ;; *) break ;; esac; done - shift - case "$1 $2" in - "/pgp-bin/og cn") shift; "${OG}" "$@" ;; - "samtools bedcov") shift 2; q=""; if [ "$1" = -Q ]; then q=.q1; shift 2; fi - d=$(dirname "$1"); cat "${FIX}/${d##*/}${q}.bedcov.tsv" ;; - *) return 1 ;; - esac -} -cpath() { printf '%s' "$1"; } -""" - - -def dosage(cat, read): - step, module = read("scripts/35-paralogs.sh"), read("modules/local/parascopy/main.nf") - m = re.search(r'^DOSAGE_LOCI="([^"]*)"$', step, re.M) - check("step 35: DOSAGE_LOCI lists the dosage rows", m is not None and m.group(1).split() == DOSAGE, - m and m.group(1)) - m = re.search(r'^ *dosage_loci="([^"]*)"$', module, re.M) - check("PARASCOPY module: dosage_loci is step 35's list", m is not None and m.group(1).split() == DOSAGE, - m and m.group(1)) - for path, text, q in (("scripts/35-paralogs.sh", step, '"'), ("modules/local/parascopy/main.nf", module, '"\\')): - for cmd in ("bed", "call"): - check(f"{path}: og cn {cmd} --locus on each dosage row", - f"og cn {cmd} --locus {q}$loc" in text) - for loc in DOSAGE: - r = cat[loc] - check(f"dosage {loc}: two bedcov passes, all reads then MAPQ >= 1 (what the loop runs)", - r["_passes"] == [".", "-Q 1"], r["_passes"]) - check(f"dosage {loc}: a whole-number call with a reading for 0 and 1 and for more", - r["_call"][0] == "round" and {"0", "1", "more"} <= set(r["_readings"]), r["_readings"]) - check(f"dosage {loc}: a PMID", re.fullmatch(r"\d+(,\d+)*", r["pmids"]) is not None, r["pmids"]) - for copies in (0, 1, 2, 3, 4): - m, q = dosage_depths(r, copies) - b = cn.assess(r, m, q) - check(f"dosage {loc}: {copies} copies called {copies}, with its reading", - (b["status"], b["copies"]) == ("called", copies) and b["reading"] == cn.reading(r, copies) - and "{n}" not in b["reading"], b) - m, q = dosage_depths(r, 2.5) - check(f"dosage {loc}: 2.5 copies is between counts", cn.assess(r, m, q)["reason"] == "between_counts") - hp = cat["HP"] - m, q = dosage_depths(hp, 4) - m["HP_UNIT1"], m["HP_UNIT2"] = 60.0, 0.0 - check("dosage HP: reads of both copies on one reference copy still add up to 4", - cn.assess(hp, m, q)["copies"] == 4) - - readings = " ".join(cat["CCL3L1"]["_readings"].values()) - check("dosage CCL3L1: no reading names a disease", not DENY.search(readings), DENY.findall(readings)) - planted = readings + " more copies go with slower HIV progression" - check("dosage CCL3L1: a planted 'HIV' reading is caught", DENY.search(planted) is not None) - - # The block of step 35, cut out and run on the fixtures. - blk = re.search(r'\n(DOSAGE_LOCI=.*?\nrm -rf "\$DOSAGE_WORK"\n)', step, re.S) - check("step 35 holds the dosage block", blk is not None) - if not OG: - return - fix = os.path.join(REPO, "tests", "fixtures", "cn") - with tempfile.TemporaryDirectory() as t: - env = dict(os.environ, REPO=REPO, FIX=fix, OUTDIR=t, SAMPLE="S", PGP_ROOT=REPO, PARASCOPY_IMAGE="img", - BAM=os.path.join(t, "S.bam")) - out = subprocess.run(["bash", "-euo", "pipefail", "-c", STUB + (blk.group(1) if blk else "false\n")], - capture_output=True, text=True, env=env) - tab = os.path.join(t, "S_cn_dosage.tsv") - rows = {} - if os.path.exists(tab): - lines = open(tab).read().splitlines() - rows = {x.split("\t")[0]: x.split("\t") for x in lines[1:]} - check("step 35 dosage table: its header", lines[0] == "locus\tstatus\tcopies\testimate\treason\treading", - lines[0]) - check("step 35 dosage block: exit 0, one line per dosage row", out.returncode == 0 and list(rows) == DOSAGE, - (out.returncode, out.stderr[-600:], list(rows))) - for loc, n in FIXTURE_COPIES.items(): - got = rows.get(loc, []) - check(f"step 35 on tests/fixtures/cn/{loc}: called {n}, with its reading", - got[1:3] == ["called", str(n)] and got[5:] == [cn.reading(cat[loc], n)], got) - - # A row shifted by 1 kb no longer matches the fixture made for its place. - with tempfile.TemporaryDirectory() as t: - p = os.path.join(t, "c.tsv") - for loc in DOSAGE: - r = {k: v for k, v in cat[loc].items() if not k.startswith("_")} - r["targets"] = ";".join(f"{n}:{s + 1000}-{e + 1000}" for n, s, e, _ in cat[loc]["_targets"]) - write_catalog(p, [r]) - f = os.path.join(fix, f"{loc}.bedcov.tsv") - out = og_cn("call", "--catalog", p, "--locus", loc, "--bedcov", f, "--bedcov-mapq1", f) - check(f"dosage {loc}: shifted 1 kb, its fixture is refused", - out.returncode == 2 and "regions differ" in out.stderr, out) - - -# Section 9: the interval rows. -SMALL_BED = """track name="test" -#chrom\tstart\tend\tname\tscore\tstrand\tthickStart\tthickEnd\titemRgb -chr22\t18159724\t18924717\tBP-A\t0\t.\t18159724\t18924717\t0,0,0 -chr22\t18924718\t21111383\t22q11.2 recurrent (DGS) region (proximal, A-D) (includes TBX1)\t0\t.\t18924718\t21111383\t255,111,0 -chr22\t21111384\t21562827\tBP-D\t0\t.\t21111384\t21562827\t0,0,0 -chrx\t6527561\t6537770\tXp22.31 distal breakpoint\t0\t.\t6527561\t6537770\t0,0,0 -chrx\t6537771\t8156913\tXp22.31 recurrent region (includes STS)\t0\t.\t6537771\t8156913\t255,111,0 -""" -HAND = ["1p36_terminal", "Xq28_MECP2", "GJB6_D13S1830", "CFTR_dele2_3", "HBB_exon3_del", "HBB_619bp_del", - "HBD_HBB_Sicilian_del", "AZFa", "AZFb", "AZFc_DAZ"] -STUB_IV = r""" -run_in() { - while [ $# -gt 0 ]; do case $1 in -v|--cpus|--memory) shift 2 ;; *) break ;; esac; done - shift - case "$1 $2" in - "/pgp-bin/og cn") shift; "${OG}" "$@" ;; - "samtools bedcov") shift 2; python3 "${FAKECOV}" "$1" ;; - *) return 1 ;; - esac -} -cpath() { printf '%s' "$1"; } -""" -# fakecov BED: samtools bedcov over BED at 30x, the segments named in FAKE_HALF at 15x, in FAKE_ZERO at 0. -FAKECOV = r""" -import os, sys -half = set(os.environ.get("FAKE_HALF", "").split(",")) -zero = set(os.environ.get("FAKE_ZERO", "").split(",")) -for line in open(sys.argv[1]): - c = line.rstrip("\n").split("\t") - d = 0.0 if c[3] in zero else 15.0 if c[3] in half else 30.0 - print("\t".join(c + [str(int(d * (int(c[2]) - int(c[1]))))])) -""" - - -def depths(r, target=30.0, control=30.0): - tg = {n for n, *_ in r["_targets"]} - return {n: target if n in tg else control for *_, n in cn.regions(r)} - - -def build_rows(bed): - """og dev build cn-intervals --stdout on a ClinGen BED and no masks: (exit code, rows as dicts, stderr).""" - with tempfile.TemporaryDirectory() as t: - p = os.path.join(t, "clingen.bed") - with open(p, "w") as f: - f.write(bed) - out = subprocess.run([OG, "dev", "build", "cn-intervals", "--clingen", p, "--segdups", os.devnull, - "--gaps", os.devnull, "--satellites", os.devnull, "--parascopy", os.devnull, - "--stdout"], capture_output=True, text=True) - rows = [dict(zip(cn.COLUMNS, line.split("\t"))) for line in out.stdout.splitlines() if not line.startswith("#")] - return out.returncode, rows, out.stderr - - -def intervals(cat, read): - if not OG: - print("skip: OG is not set: the converter checks of og dev build cn-intervals did not run") - else: - code, rows, err = build_rows(SMALL_BED) - check("converter: the two region rows of the small BED, its chrx row on chrX", - code == 0 and [(r["locus"], r["chrom"]) for r in rows[:2]] - == [("22q11.2_DGS_proximal_A-D", "chr22"), ("Xp22.31", "chrX")], - (code, err, [(r["locus"], r["chrom"]) for r in rows])) - with tempfile.TemporaryDirectory() as t: - p = os.path.join(t, "c.tsv") - write_catalog(p, rows) - try: - got = len(cn.load_catalog(p)) - except cn.CatalogError as e: - got = str(e) - check("converter: its rows load", got == len(rows), got) - write_catalog(p, [dict(r, chrom="chrx") if r["chrom"] == "chrX" else r for r in rows]) - try: - cn.load_catalog(p) - got = "loaded" - except cn.CatalogError as e: - got = str(e) - check("converter: the same rows left as 'chrx' are refused", "chrom 'chrx'" in got, got) - code, _, err = build_rows(SMALL_BED.replace("chrx", "chrQ")) - check("converter: an unknown contig stops the build", - code == 2 and "'chrQ' is not a GRCh38 chromosome" in err, (code, err)) - check("converter: no target piece overlaps a breakpoint row", - all(not (s < 18924717 and 18159724 < e) and not (s < 21562827 and 21111384 < e) for r in rows[:1] - for _, s, e, _ in cn.parse_row(r)["_targets"])) - - iv = [r for r in cat.values() if r["kind"] == "interval" and r["_passes"] == ["."]] - names = [r["locus"] for r in iv] - check("interval rows: the ClinGen regions (over 70), then the hand rows", len(iv) > 80 and names[-10:] == HAND, - (len(iv), names[-10:])) - # HbVar's GRCh38 spans (979, 1035), 0-based start: one target each, never cut - spans = {loc: [(s, e) for _, s, e, _ in cat[loc]["_targets"]] if loc in cat else None - for loc in ("HBB_619bp_del", "HBD_HBB_Sicilian_del")} - check("interval rows: the HBB 619 bp and Sicilian deletions at HbVar's GRCh38 spans", - spans == {"HBB_619bp_del": [(5225388, 5226007)], "HBD_HBB_Sicilian_del": [(5219877, 5233280)]}, spans) - check("interval rows: each cross-checked by CNVpytor, step 18", - all(r["cross_check"] == "cnvpytor:cnvs" for r in iv)) - check("interval rows: the 15q13.3 rows keep CHRNA7 (a Parascopy model) out of their targets", - all(not (32154858 < e and s < 32178018) for r in iv if r["locus"].startswith("15q13.3") - for _, s, e, _ in r["_targets"]) - and any(r["locus"].startswith("15q13.3") for r in iv)) - hits = [(r["locus"], o["locus"]) for r in iv for _, cs, ce, _ in r["_controls"] for o in iv - if o is not r and o["chrom"] == r["chrom"] and any(ts < ce and cs < te for _, ts, te, _ in o["_targets"])] - check("interval rows: no row's control sits on another row's target (Xq28's left flank off Xq28_MECP2)", - not hits, hits) - - a_d, p17 = cat["22q11.2_DGS_proximal_A-D"], cat["17p12_HNPP_CMT1A"] - for r, target, want, change_ in ((a_d, 15.0, 1, "deletion"), (p17, 45.0, 3, "duplication"), - (a_d, 30.0, 2, "no change"), (p17, 30.0, 2, "no change")): - b = cn.assess(r, depths(r, target)) - check(f"interval {r['locus']}: targets at {target}x over 30x controls is {want} copies, {change_}", - (b["status"], b["copies"], change(b)) == ("called", want, change_), b) - az = cat["AZFc_DAZ"] - m = depths(az, 15.0 / 4, 15.0) - check("AZFc on a male: four DAZ copies at a single copy's depth in all is 4 copies", - cn.assess(az, {n: (15.0 if n in {x for x, *_ in az["_targets"]} else 15.0) for n in m}, None, "male") - ["copies"] == 4) - half = {n: (7.5 if n in {x for x, *_ in az["_targets"]} else 15.0) for n in m} - b = cn.assess(az, half, None, "male") - check("AZFc on a male at half its DAZ depth: 2 copies, a deletion (gr/gr or b2/b3)", - (b["copies"], change(b)) == (2, "deletion"), b) - b = cn.assess(cat["AZFa"], depths(cat["AZFa"], 15.0, 15.0), None, "male") - check("AZFa on a male: one copy, no change", (b["copies"], b["expected"]) == (1, 1), b) - b = cn.assess(cat["AZFa"], depths(cat["AZFa"], 0.0, 15.0), None, "male") - check("AZFa on a male with no read over it: 0 copies, a deletion", - (b["copies"], change(b)) == (0, "deletion"), b) - for loc in ("AZFa", "AZFb", "AZFc_DAZ"): - b = cn.assess(cat[loc], depths(cat[loc], 15.0, 15.0), None, "female") - check(f"{loc} on a female: not applicable", (b["status"], b["reason"]) == ("not applicable", "no_y"), b) - b = cn.assess(cat[loc], depths(cat[loc], 15.0, 15.0), None, "unknown") - check(f"{loc} with no sex: not assessed", (b["status"], b["reason"]) == ("not assessed", "sex_unknown"), b) - xp = cat["Xp22.31"] - b = cn.assess(xp, depths(xp, 15.0, 15.0), None, "male") - check("Xp22.31 on a male at his one X's depth: 1 copy, no change", (b["copies"], b["expected"]) == (1, 1), b) - b = cn.assess(xp, depths(xp, 0.0, 15.0), None, "male") - check("Xp22.31 on a male with no read there: 0 copies, a deletion (the STS deletion)", b["copies"] == 0, b) - - step, module = read("scripts/35-paralogs.sh"), read("modules/local/parascopy/main.nf") - for path, text in (("scripts/35-paralogs.sh", step), ("modules/local/parascopy/main.nf", module)): - for cmd in ("og cn bed --kind interval", "og cn intervals --bedcov", "og cn intervals --failed"): - check(f"{path}: {cmd}", cmd in text) - check("PARASCOPY module: the samplesheet sex goes to the interval rows", - "def sex = meta.sex in ['male', 'female'] ? meta.sex : 'unknown'" in module and "--sex ${sex}" in module) - if not OG: - return - with tempfile.TemporaryDirectory() as t: - fake = os.path.join(t, "fakecov.py") - with open(fake, "w") as f: - f.write(FAKECOV) - bed = os.path.join(t, "r.bed") - out = og_cn("bed", "--kind", "interval") - open(bed, "w").write(out.stdout) - check("bed --kind interval: every segment of every interval row, once", - out.returncode == 0 and len(out.stdout.splitlines()) == sum(len(cn.regions(r)) for r in iv), out.stderr) - halfs = ",".join(n for n, *_ in a_d["_targets"]) - env = dict(os.environ, FAKE_HALF=halfs, FAKE_ZERO=p17["_controls"][0][0]) - cov = os.path.join(t, "cov.tsv") - with open(cov, "w") as f: - subprocess.run([sys.executable, fake, bed], stdout=f, env=env, check=True) - tab = os.path.join(t, "iv.tsv") - out = og_cn("intervals", "--bedcov", cov, "--sex", "female", "--out", tab) - lines = open(tab).read().splitlines() if os.path.exists(tab) else [] - rows = {x.split("\t")[0]: x.split("\t") for x in lines[1:]} - check("intervals: exit 0, the header, one line per interval row", - out.returncode == 0 and lines[:1] == ["\t".join(INTERVAL_COLUMNS)] and list(rows) == names, - (out.returncode, out.stderr[-400:], lines[:1])) - check("intervals: the 22q11.2 A-D deletion at 1 copy", rows.get(a_d["locus"], [""] * 11)[5:9] - == ["called", "1", "2", "deletion"], rows.get(a_d["locus"])) - check("intervals: a control with no read leaves 17p12 not assessed (broken_control), the rest called", - rows.get(p17["locus"], [""] * 11)[10] == "broken_control" - and rows.get("3q29", [""] * 11)[5:9] == ["called", "2", "2", "no change"], rows.get(p17["locus"])) - check("intervals: chrY rows on a female are not applicable", - all(rows.get(x, [""] * 11)[5] == "not applicable" for x in ("AZFa", "AZFb", "AZFc_DAZ"))) - out = og_cn("intervals", "--failed", "--out", tab) - lines = open(tab).read().splitlines()[1:] - check("intervals --failed: every row not assessed, check_failed", - out.returncode == 0 and len(lines) == len(iv) and all(x.split("\t")[10] == "check_failed" for x in lines)) - - blk = re.search(r'\n(INTERVAL_OUT=.*?\nrm -rf "\$INTERVAL_WORK"\n)', step, re.S) - check("step 35 holds the interval block", blk is not None) - env = dict(os.environ, REPO=REPO, FAKECOV=fake, OUTDIR=t, SAMPLE="S", PGP_ROOT=REPO, PARASCOPY_IMAGE="img", - BAM=os.path.join(t, "S.bam"), SEX="male", FAKE_HALF=",".join(n for n, *_ in az["_targets"])) - out = subprocess.run(["bash", "-euo", "pipefail", "-c", STUB_IV + (blk.group(1) if blk else "false\n")], - capture_output=True, text=True, env=env) - tab = os.path.join(t, "S_cn_intervals.tsv") - rows = {x.split("\t")[0]: x.split("\t") for x in open(tab).read().splitlines()[1:]} if os.path.exists(tab) else {} - check("step 35 interval block: exit 0, one line per interval row", out.returncode == 0 and list(rows) == names, - (out.returncode, out.stderr[-600:])) - # AZFc's DAZ at 15x against controls at 30x on a male: 4 x 15 / 30 = 2 copies of 4. - check("step 35 interval block on a male: AZFc at half its depth is a deletion, AZFa one copy", - rows.get("AZFc_DAZ", [""] * 11)[6:9] == ["2", "4", "deletion"] - and rows.get("AZFa", [""] * 11)[6:9] == ["1", "1", "no change"], (rows.get("AZFc_DAZ"), rows.get("AZFa"))) - good = os.path.join(t, "good_cn_intervals.tsv") - if os.path.exists(tab): - os.replace(tab, good) - env["FAKECOV"] = "/nonexistent.py" - out = subprocess.run(["bash", "-euo", "pipefail", "-c", STUB_IV + (blk.group(1) if blk else "false\n")], - capture_output=True, text=True, env=env) - lines = open(tab).read().splitlines()[1:] if os.path.exists(tab) else [] - check("step 35 interval block with a failed bedcov: exit 0, every row check_failed", - out.returncode == 0 and len(lines) == len(iv) and all(x.split("\t")[10] == "check_failed" for x in lines), - (out.returncode, out.stderr[-400:])) - # og summary collect: step 35's section carries the interval rows with a change - import collect_summary as cs - sd = os.path.join(t, "sample") - os.makedirs(os.path.join(sd, "paralogs")) - with open(os.path.join(sd, "paralogs", "S_smn_copy_number.tsv"), "w") as f: - f.write("chrom\tstart\tend\tlocus\tagCN_filter\tagCN\tagCN_qual\tpsCN_filter\tpsCN\tpsCN_qual\t" - "homologous_regions\n") - if os.path.exists(good): - os.replace(good, os.path.join(sd, "paralogs", "S_cn_intervals.tsv")) - data = cs.collect("S", sd)["sections"]["parascopy"]["data"] - ci = data.get("cn_intervals") or {} - check("summary: step 35's section lists the interval rows with a change (AZFc) and counts the rest", - [c["locus"] for c in ci.get("changes", [])] == ["AZFc_DAZ"] and ci.get("rows") == len(iv), ci) - os.remove(os.path.join(sd, "paralogs", "S_smn_copy_number.tsv")) - data = cs.collect("S", sd)["sections"]["parascopy"]["data"] - ci = (data or {}).get("cn_intervals") or {} - check("summary: the interval rows stay when step 35 wrote no other table", - [c["locus"] for c in ci.get("changes", [])] == ["AZFc_DAZ"] and ci.get("rows") == len(iv), ci) - - -# Section 10: the CYP21A2 row, call psv. -CYP_CASES = { - # case: (module total, {site: CYP21A2 copies there}); the sites not named carry 2 - "normal": (4, {}), - # the 30 kb chimeric deletion: one module fewer, its chimera carries CYP21A1P's sequence - # at the 5' sites (In2G, I172N) and CYP21A2's at the 3' ones - "deletion": (3, {"PSV_In2G": 1, "PSV_I172N": 1}), - # In2G converted to the pseudogene base on one of the two CYP21A2 copies - "conversion": (4, {"PSV_In2G": 1}), -} -CYP_FIX = os.path.join(REPO, "tests", "fixtures", "cn") - - -def cyp_fixture(r, total, a2): - """(bedcov text, mpileup text) of a CYP21A2 genome: 15x per copy, flanks at 30x - (two copies), two CYP21A2 copies on CYP21A2's body and the rest of the module - total on CYP21A1P's. At each site the reads carrying CYP21A2's base (a2 copies, - 2 by default) and CYP21A1P's (the rest) are split 4 to 1 between the CYP21A2 - and the CYP21A1P position, in both cases, with read starts, ends and an indel - in the bases column, as samtools mpileup without a reference writes them.""" - depth = {n: 30.0 for *_, n in cn.regions(r)} - depth["CYP21A2_BODY"], depth["CYP21A1P_BODY"] = 30.0, 15.0 * (total - 2) - bed = "".join(f"{c}\t{s}\t{e}\t{n}\t{round(depth[n] * (e - s))}\n" for c, s, e, n in cn.regions(r)) - lines = [] - for name, g, p, gb, pb, _ in r["_psv"]: - gene, pseudo = 15 * a2.get(name, 2), 15 * (total - a2.get(name, 2)) - gbase, pbase = sorted(gb)[-1], sorted(pb)[0] - for pos, share in ((g, 0.8), (p, 0.2)): - ng, np_ = round(gene * share), round(pseudo * share) - reads = [gbase] * ng + [pbase] * np_ - reads = [x.lower() if i % 2 else x for i, x in enumerate(reads)] - if reads: - reads[0] = "^]" + reads[0] - reads[-1] += "$" - if len(reads) > 2: - reads[1] += "+2AG" - lines.append((pos + 1, f"{r['chrom']}\t{pos + 1}\tN\t{len(reads)}\t{''.join(reads)}\t{'I' * len(reads)}\n")) - return bed, "".join(x for _, x in sorted(lines)) - - -def cyp_call(r, bedcov_text, pileup_text, mapq0=0.1): - """og cn call on a CYP21A2 genome; mapq0 is the share of every segment's - depth from MAPQ 0 reads (0.1 as HG002's BWA-MEM and minimap2 reads give - over the two targets; 0 for an aligner that leaves those reads out). The - table read back: status, copies, module_total, finding and reason (None - for '.'), reading, conversion_sites, each site's copies estimate and - ratio, and the table and stdout line themselves.""" - with tempfile.TemporaryDirectory() as t: - b, q, p, tab = (os.path.join(t, x) for x in ("b.tsv", "q.tsv", "p.tsv", "t.tsv")) - open(b, "w").write(bedcov_text) - open(q, "w").write("".join( - "\t".join(c[:4] + [repr(float(c[4]) * (1 - mapq0))]) + "\n" - for c in (ln.split("\t") for ln in bedcov_text.splitlines()))) - open(p, "w").write(pileup_text) - out = og_cn("call", "--catalog", CATALOG, "--locus", "CYP21A2", "--bedcov", b, "--bedcov-mapq1", q, - "--pileup", p, "--table", tab) - rows = table(tab) - val = (lambda k, f=str: None if rows.get(k, ".") == "." else f(rows[k])) - sites = {} - for k, v in rows.items(): - m = re.search(r"copies \S+ \(([0-9]+\.[0-9]+)\); ratio ([0-9]+\.[0-9]+)", v) if k.startswith("psv_") else None - if m: - sites[k[4:]] = {"estimate": float(m.group(1)), "ratio": float(m.group(2))} - return {"status": rows.get("status"), "copies": val("cyp21a2_copies", int), "module_total": val("module_total", int), - "finding": val("finding"), "reason": val("reason"), "reading": rows.get("reading", ""), - "conversion_sites": (val("conversion_sites") or "").split(",") if val("conversion_sites") else [], - "sites": sites, "table": rows, "line": out.stdout.strip(), "exit": out.returncode} - - -# run_in and cpath for the CYP21A2 block of step 35: og cn runs for real (OG), -# samtools bedcov and mpileup print tests/fixtures/cn/CYP21A2_${CYP_CASE}.*. -STUB_CYP = r""" -run_in() { - while [ $# -gt 0 ]; do case $1 in -v|--cpus|--memory) shift 2 ;; *) break ;; esac; done - shift - case "$1 $2" in - "/pgp-bin/og cn") shift; "${OG}" "$@" ;; - "samtools bedcov") - # the MAPQ >= 1 pass: 90% of each depth (HG002's share), or all of it for - # an aligner that left the MAPQ 0 reads out (CYP_NO_MAPQ0) - if [ "${3:-}" = "-Q" ] && [ -z "${CYP_NO_MAPQ0:-}" ]; then - awk -F'\t' -v OFS='\t' '{$5 = int($5 * 0.9); print}' "${FIX}/CYP21A2_${CYP_CASE}.bedcov.tsv" - else - cat "${FIX}/CYP21A2_${CYP_CASE}.bedcov.tsv" - fi ;; - "samtools mpileup") [ -z "${MPILEUP_FAILS:-}" ] && cat "${FIX}/CYP21A2_${CYP_CASE}.pileup.tsv" ;; - *) return 1 ;; - esac -} -cpath() { printf '%s' "$1"; } -""" - - -def psv(cat, read): - r = cat["CYP21A2"] - names = [n for n, *_ in r["_psv"]] - check("CYP21A2: six PSV sites, In2G and I172N among them, a call psv row with a reading per finding", - names == ["PSV_In2G", "PSV_I172N", "PSV_V237E", "PSV_V281L", "PSV_Q318X", "PSV_R356W"] - and r["_call"][0] == "psv" and set(cn.PSV_FINDINGS) <= set(r["_readings"]), (names, r["_call"])) - check("CYP21A2: two targets (CYP21A2 and CYP21A1P), expected module total 4", - [n for n, *_ in r["_targets"]] == ["CYP21A2_BODY", "CYP21A1P_BODY"]) - # The CYP21A2 sites lie 32.7 kb after their CYP21A1P matches, inside both targets. - tg = {n: (s, e) for n, s, e, _ in r["_targets"]} - check("CYP21A2: each site in the CYP21A2 target, its pseudogene position in CYP21A1P's, 32.7 to 32.8 kb apart", - all(tg["CYP21A2_BODY"][0] <= g < tg["CYP21A2_BODY"][1] and tg["CYP21A1P_BODY"][0] <= p < tg["CYP21A1P_BODY"][1] - and 32700 < g - p < 32800 for _, g, p, *_ in r["_psv"]), r["_psv"]) - check("CYP21A2: In2G's CYP21A2 base is A or C, its CYP21A1P base G (c.293-13A/C>G)", - r["_psv"][0][3:5] == ({"A", "C"}, {"G"}), r["_psv"][0]) - with tempfile.TemporaryDirectory() as t: - p = os.path.join(t, "c.tsv") - base = {k: v for k, v in r.items() if not k.startswith("_")} - for desc, kw, err in (("a PSV site of two bases", {"details": base["details"].replace( - "PSV_In2G:32039080-32039081", "PSV_In2G:32039080-32039082")}, "must be one base"), - ("a site with the same base on both genes", {"details": base["details"].replace( - "32006352 AC>G", "32006352 AC>C")}, "distinct gene and pseudogene"), - ("a site label without its pseudogene position", {"details": base["details"].replace( - "32006352 AC>G", "here AC>G")}, "is not 'PSEUDO_POS"), - ("a psv row without a conversion reading", {"readings": "|".join( - x for x in base["readings"].split("|") if not x.startswith("conversion="))}, - "a reading for each")): - write_catalog(p, [dict(base, **kw)]) - try: - cn.load_catalog(p) - got = "loaded" - except cn.CatalogError as e: - got = str(e) - check(f"CYP21A2: {desc} is refused", err in got, got) - - step, module = read("scripts/35-paralogs.sh"), read("modules/local/parascopy/main.nf") - for path, text, q in (("scripts/35-paralogs.sh", step, True), ("modules/local/parascopy/main.nf", module, False)): - for cmd in ("og cn bed --locus CYP21A2", "og cn bed --psv --locus CYP21A2", - "--bedcov-mapq1", "cyp21a2_bedcov_q1.tsv" if not q else 'bedcov_q1.tsv")"', - "samtools mpileup -A -B -q 0 -Q 13 -d 100000", "og cn call --locus CYP21A2 --bedcov", - "--pileup", "_cyp21a2_psv.tsv"): - check(f"{path}: {cmd}", cmd in text) - check("PARASCOPY module: its CYP21A2 table is an output", 'path("${meta.id}_cyp21a2_psv.tsv"), emit: cyp21a2' - in module) - check("CYP21A2 row: a MAPQ 0 guard (two bedcov passes, all reads and MAPQ >= 1; minimum share 0.02)", - r["_passes"] == [".", "-Q 1"] and r["_mapq0_min"] == 0.02 and r["_mapq0_max"] is None, - (r["_passes"], r["_mapq0_min"], r["_mapq0_max"])) - if not OG: - return - - # og's pileup reader (read starts, ends and indels are not bases) is - # checked by the crate's own test and by the parity cases. - out = og_cn("bed", "--psv", "--locus", "CYP21A2") - lines = out.stdout.splitlines() - check("bed --psv: twelve one-base positions (each site on CYP21A2 and on CYP21A1P), sorted", - out.returncode == 0 and len(lines) == 12 and all(int(x.split("\t")[2]) - int(x.split("\t")[1]) == 1 for x in lines) - and [int(x.split("\t")[1]) for x in lines] == sorted(int(x.split("\t")[1]) for x in lines), out) - out = og_cn("bed", "--psv", "--locus", "HBA") - check("bed --psv on a row without PSV sites is refused", out.returncode == 2 and "no psv locus" in out.stderr, out) - - # The committed fixtures are cyp_fixture's, so a change to either shows. - blocks = {} - for case, (total, a2) in CYP_CASES.items(): - bed, pile = cyp_fixture(r, total, a2) - fb, fp = (os.path.join(CYP_FIX, f"CYP21A2_{case}.{x}.tsv") for x in ("bedcov", "pileup")) - same = os.path.exists(fb) and os.path.exists(fp) and open(fb).read() == bed and open(fp).read() == pile - check(f"tests/fixtures/cn/CYP21A2_{case}: the tables cyp_fixture makes", same) - blocks[case] = cyp_call(r, bed, pile) - b = blocks["normal"] - check("CYP21A2 unmodified: called, 2 copies, module total 4, no finding (the red control: no call)", - (b["status"], b["copies"], b["module_total"], b["finding"]) == ("called", 2, 4, "none") - and b["reading"] == cn.reading(r, "none"), b) - b = blocks["deletion"] - check("CYP21A2 planted chimeric deletion: 1 copy, module total down by one (3), finding deletion", - (b["status"], b["copies"], b["module_total"], b["finding"]) == ("called", 1, 3, "deletion"), b) - b = blocks["conversion"] - ratio = (b.get("sites") or {}).get("PSV_In2G", {}).get("ratio") - check("CYP21A2 planted In2G conversion on one copy: In2G ratio 0.5 +/- 0.1, a possible conversion there only", - ratio is not None and abs(ratio - 0.5) <= 0.1 and b["finding"] == "conversion" - and b["conversion_sites"] == ["PSV_In2G"] and b["module_total"] == 4, b) - check("CYP21A2 conversion: the reading names the site and says possible conversion", - "c.293-13A/C>G (In2G" in b["reading"] and "possible conversion" in b["reading"], b["reading"]) - # The same depths with the PSV reads ignored (every site even) give no finding. - total, a2 = CYP_CASES["conversion"] - bed, _ = cyp_fixture(r, total, a2) - _, even = cyp_fixture(r, total, {}) - b = cyp_call(r, bed, even) - check("CYP21A2: the conversion's depths with the PSV ratios ignored give no finding", - b["finding"] == "none", b) - bed, pile = cyp_fixture(r, 3, {}) - b = cyp_call(r, bed, pile) - check("CYP21A2: a module fewer with CYP21A2 kept at every site is a pseudogene deletion, 2 copies", - (b["copies"], b["finding"]) == (2, "pseudogene_deletion"), b) - # One noisy 3' site (V281L at 1.47, rounds to 1) is inside the 0.6 margin: it never - # makes a deletion of a pseudogene deletion, nor a lower count than the finding says. - for total, want in ((3, "pseudogene_deletion"), (4, "none")): - bed, pile = cyp_fixture(r, total, {"PSV_V281L": 1.47}) - b = cyp_call(r, bed, pile) - est = b["sites"]["PSV_V281L"]["estimate"] - check(f"CYP21A2: module total {total}, one site at {est:.2f} (inside the margin): {want}, 2 copies", - 1.4 < est < 1.5 and (b["copies"], b["finding"]) == (2, want), b) - bed, pile = cyp_fixture(r, 4, {n: 1 for n in names}) - b = cyp_call(r, bed, pile) - check("CYP21A2: one copy converted at every site, the module total kept: a conversion at all six", - b["finding"] == "conversion" and b["conversion_sites"] == names and b["copies"] == 1, b) - bed, _ = cyp_fixture(r, 4, {}) - thin = "".join(ln.split("\t")[0] + "\t" + ln.split("\t")[1] + "\tN\t1\tC\tI\n" for ln in even.splitlines()) - b = cyp_call(r, bed, thin) - check("CYP21A2: sites with under 30 reads: not assessed (psv_low_depth)", - (b["status"], b["reason"]) == ("not assessed", "psv_low_depth"), b) - bed, pile = cyp_fixture(r, 4, {}) - b = cyp_call(r, bed.replace("CYP21A1P_BODY\t", "CYP21A1P_BODY\t").replace( - f"\t{round(30.0 * 2689)}\n", f"\t{round(37.5 * 2689)}\n"), pile) - check("CYP21A2: a module total between counts (4.5) is not assessed, no finding", - (b["status"], b["reason"], b["finding"]) == ("not assessed", "between_counts", None), b) - # The MAPQ 0 guard: an aligner that left out the reads fitting both copies lowers - # the module total, so the deletion's tables from such a BAM (no MAPQ 0 read) read - # not assessed, not a deletion; with HG002's 10% share they still give the deletion. - total, a2 = CYP_CASES["deletion"] - bed, pile = cyp_fixture(r, total, a2) - for share, want in ((0.0, ("not assessed", "low_mapq0_share", None, None)), - (0.019, ("not assessed", "low_mapq0_share", None, None)), - (0.021, ("called", None, "deletion", 1)), (0.1, ("called", None, "deletion", 1))): - b = cyp_call(r, bed, pile, share) - check(f"CYP21A2: the deletion's tables with a MAPQ 0 share of {share}: {want[0]} ({want[1] or want[2]})", - (b["status"], b["reason"], b["finding"], b["copies"]) == want, b) - b = cyp_call(r, bed, pile, 0.0) - tab, line = b["table"], b["line"] - check("CYP21A2 without MAPQ 0 reads: the table says so (message, share 0.000), no copies, module total or finding", - tab["message"].startswith("not assessed: the BAM has few or no multi-mapping reads here") - and "CYP21A2 and CYP21A1P" in tab["message"] and tab["mapq0_share"] == "0.000" - and (tab["cyp21a2_copies"], tab["module_total"], tab["finding"]) == (".", ".", ".") - and "low_mapq0_share" in line, (tab, line)) - bed, pile = cyp_fixture(r, 4, {}) - tab = cyp_call(r, bed, pile)["table"] - check("CYP21A2 unmodified with HG002's MAPQ 0 share: called, share 0.100, message '.'", - (tab["status"], tab["finding"], tab["mapq0_share"], tab["message"]) == ("called", "none", "0.100", "."), tab) - - blk = re.search(r'\n(CYP21_OUT=.*?\nrm -rf "\$CYP21_WORK"\n)', step, re.S) - check("step 35 holds the CYP21A2 block", blk is not None) - tabs = {} - with tempfile.TemporaryDirectory() as t: - for case in list(CYP_CASES) + ["fail", "no_mapq0"]: - env = dict(os.environ, REPO=REPO, FIX=CYP_FIX, OUTDIR=t, SAMPLE="S", PGP_ROOT=REPO, PARASCOPY_IMAGE="img", - BAM=os.path.join(t, "S.bam"), - CYP_CASE={"fail": "normal", "no_mapq0": "deletion"}.get(case, case)) - if case == "fail": - env["MPILEUP_FAILS"] = "1" - if case == "no_mapq0": # the deletion's reads from a BAM with no MAPQ 0 read - env["CYP_NO_MAPQ0"] = "1" - out = subprocess.run(["bash", "-euo", "pipefail", "-c", STUB_CYP + (blk.group(1) if blk else "false\n")], - capture_output=True, text=True, env=env) - tab = os.path.join(t, "S_cyp21a2_psv.tsv") - rows = dict(x.split("\t", 1) for x in open(tab).read().splitlines()[1:]) if os.path.exists(tab) else {} - check(f"step 35 CYP21A2 block ({case}): exit 0, a table", out.returncode == 0 and rows, - (out.returncode, out.stderr[-600:])) - tabs[case] = os.path.join(t, f"{case}.tsv") - if os.path.exists(tab): - os.replace(tab, tabs[case]) - want = {"normal": ("called", "2", "4", "none"), "deletion": ("called", "1", "3", "deletion"), - "conversion": ("called", "1", "4", "conversion"), "fail": ("not assessed", ".", ".", "."), - "no_mapq0": ("not assessed", ".", ".", ".")}[case] - check(f"step 35 CYP21A2 block ({case}): status, copies, module total, finding {want}", - tuple(rows.get(k) for k in ("status", "cyp21a2_copies", "module_total", "finding")) == want, rows) - check("step 35 CYP21A2 block with a failed mpileup: reason check_failed", - open(tabs["fail"]).read().count("reason\tcheck_failed") == 1 if os.path.exists(tabs["fail"]) else False) - rd = (lambda c: dict(x.split("\t", 1) for x in open(tabs[c]).read().splitlines()[1:]) - if os.path.exists(tabs[c]) else {}) - nm, nl = rd("no_mapq0"), rd("normal") - check("step 35 CYP21A2 block on the deletion's reads without MAPQ 0 reads: low_mapq0_share, the message says why", - (nm.get("reason"), nm.get("mapq0_share")) == ("low_mapq0_share", "0.000") - and nm.get("message", "").startswith("not assessed: the BAM has few or no multi-mapping reads here"), nm) - check("step 35 CYP21A2 block, unmodified: the MAPQ >= 1 pass ran (share 0.100), message '.'", - (nl.get("mapq0_share"), nl.get("message")) == ("0.100", "."), nl) - check("step 35 CYP21A2 block with a failed mpileup: the rows of a called table but the sites'", - list(rd("fail")) == [k for k in nl if not k.startswith("psv_")], (list(rd("fail")), list(nl))) - - # the couple screen (og couple screen) reads each table - ped = os.path.join(t, "ped.tsv") - open(ped, "w").write("sample\tsex\tfather\tmother\tpartner\nA\tfemale\t.\t.\tB\nB\tmale\t.\t.\tA\n") - vcf = os.path.join(t, "none.vcf") - open(vcf, "w").write("##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n") - carrier = os.path.join(REPO, "assets", "carrier_genes.tsv") - - def couple(name, extra, genes=carrier): - """og couple screen of two partners with no ClinVar hit: (exit, CYP21A2 row or None, md or stderr).""" - od = os.path.join(t, "couple-" + name) - out = subprocess.run([OG or "og", "couple", "screen", "--pedigree", ped, "--genes", genes, - "--clinvar", f"A={vcf}", "--clinvar", f"B={vcf}", "--out-dir", od] + extra, - capture_output=True, text=True) - if out.returncode != 0: - return out.returncode, None, out.stderr - with open(os.path.join(od, "couple_A_B.tsv")) as f: - head = f.readline().rstrip("\n").split("\t") - row = next((dict(zip(head, x.rstrip("\n").split("\t"))) for x in f if x.startswith("CYP21A2\t")), None) - return 0, row, open(os.path.join(od, "couple_A_B.md")).read() - have = lambda c: os.path.exists(tabs[c]) # noqa: E731 - rc, row, md = couple("deletion", ["--cyp21a2", f"A={tabs['deletion']}", "--cyp21a2", f"B={tabs['deletion']}"]) \ - if have("deletion") else (1, None, "") - check("couple screen: the deletion table is a CYP21A2 finding on one copy (the chimeric deletion; both: 1 in 4)", - rc == 0 and row is not None and row["result"] == "shared" and row["offspring_risk"] == "1 in 4 per pregnancy" - and "chimeric deletion" in row["findings_a"], (rc, row, md[-600:])) - rc, row, md = couple("conversion", ["--cyp21a2", f"A={tabs['conversion']}"]) if have("conversion") else (1, None, "") - check("couple screen: the conversion table is a possible conversion at In2G", - rc == 0 and row is not None and "possible CYP21A2 conversion at In2G" in row["findings_a"], (rc, row)) - gap = "CYP21A2 deletion and conversions of A: " - rc, row, md = couple("normal", ["--cyp21a2", f"A={tabs['normal']}"]) if have("normal") else (1, None, "") - check("couple screen: the unmodified table is no finding and no gap line", - rc == 0 and row is None and gap not in md, (rc, row, md[-600:])) - rc, row, md = couple("no_mapq0", ["--cyp21a2", f"A={tabs['no_mapq0']}"]) if have("no_mapq0") else (1, None, "") - check("couple screen: the table without MAPQ 0 reads is no finding and says not assessed (its message)", - rc == 0 and row is None and gap + "CYP21A2 not assessed: the BAM has few or no multi-mapping reads" in md, - (rc, row, md[-600:])) - rc, row, md = couple("fail", ["--cyp21a2", f"A={tabs['fail']}"]) if have("fail") else (1, None, "") - check("couple screen: the failed table is no finding and says not assessed", - rc == 0 and row is None and gap + "CYP21A2 not assessed" in md, (rc, row, md[-600:])) - if have("normal"): - bad = os.path.join(t, "bad.tsv") - open(bad, "w").write(open(tabs["normal"]).read().replace("finding\tnone", "finding\tlost")) - rc, row, md = couple("bad", ["--cyp21a2", f"A={bad}"]) - check("couple screen: a called table with an unknown finding stops the screen (exit 2)", rc == 2, (rc, md)) - no_cyp = os.path.join(t, "no_cyp21a2.tsv") - open(no_cyp, "w").write("".join(x for x in open(carrier) if not x.startswith("CYP21A2\t"))) - rc, row, md = couple("no-list", ["--cyp21a2", f"A={tabs['deletion']}"], no_cyp) if have("deletion") else (1, 0, "") - check("couple screen: a CYP21A2 finding needs CYP21A2 in the carrier list", rc == 0 and row is None, (rc, row)) - # A partner without a CYP21A2 table gets a gap line, also when no partner has one (step 39). - for desc, extra, want in (("no partner has one", [], {"A", "B"}), - ("one partner has one", ["--cyp21a2", f"A={tabs['normal']}"], {"B"})): - od = os.path.join(t, "couple-" + str(len(extra))) - out = subprocess.run([OG or "og", "couple", "screen", - "--pedigree", ped, "--genes", carrier, - "--clinvar", f"A={vcf}", "--clinvar", f"B={vcf}", "--out-dir", od] + extra, - capture_output=True, text=True) - md = open(os.path.join(od, "couple_A_B.md")).read() if out.returncode == 0 else out.stderr - got = {s for s in "AB" if (f"CYP21A2 deletion and conversions of {s}: step 35's CYP21A2 table was not passed " - "to the couple screen (--cyp21a2).") in md} - check(f"couple screen, {desc}: a gap line for each partner without a CYP21A2 table", got == want, (got, md[-600:])) - - # og summary collect: step 35's section carries the CYP21A2 table - import collect_summary as cs - sd = os.path.join(t, "sample") - os.makedirs(os.path.join(sd, "paralogs")) - with open(os.path.join(sd, "paralogs", "S_smn_copy_number.tsv"), "w") as f: - f.write("chrom\tstart\tend\tlocus\tagCN_filter\tagCN\tagCN_qual\tpsCN_filter\tpsCN\tpsCN_qual\t" - "homologous_regions\n") - if os.path.exists(tabs["deletion"]): - os.replace(tabs["deletion"], os.path.join(sd, "paralogs", "S_cyp21a2_psv.tsv")) - data = cs.collect("S", sd)["sections"]["parascopy"]["data"] - c = data.get("cyp21a2") or {} - check("summary: step 35's section carries the CYP21A2 table (deletion, 1 copy)", - (c.get("finding"), c.get("cyp21a2_copies")) == ("deletion", "1"), c) - # and the reports print why a table without MAPQ 0 reads is not assessed - import render_report - if os.path.exists(tabs["no_mapq0"]): - os.replace(tabs["no_mapq0"], os.path.join(sd, "paralogs", "S_cyp21a2_psv.tsv")) - data = cs.collect("S", sd)["sections"]["parascopy"]["data"] - got = render_report.cyp21a2_line(data.get("cyp21a2")) - check("report: the CYP21A2 line without MAPQ 0 reads is not assessed and says the BAM has few or no multi-mapping reads", - ": not assessed (the BAM has few or no multi-mapping reads here" in got and "deletion," not in got, got) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/test_cross_engine_biallelic.py b/tests/test_cross_engine_biallelic.py deleted file mode 100644 index f1b4f9d6d..000000000 --- a/tests/test_cross_engine_biallelic.py +++ /dev/null @@ -1,347 +0,0 @@ -#!/usr/bin/env python3 -"""Two hits in one gene across callers (og summary collect's two-hit rule): a -called deletion plus a small variant, on synthetic sample folders. Nothing -here describes a person: the calls are made up at real gene positions. - -Checks: - 1. MUTYH: an AnnotSV exon deletion plus a heterozygous ClinVar P/LP hit - elsewhere in the gene gives a "possibly biallelic" line, and the ACMG SF - tier now reports the hit (MUTYH needs two variants: the deletion is one); - 2. the same two with read-backed phase on different copies (GT with "|", - same PS) give "biallelic"; - 3. the same two in cis (same copy, same PS) give no two-hit line, and the SF - tier keeps the one hit apart; - 4. TAR syndrome: step 35's 1q21.1 deletion plus the RBM8A low-expression - allele (rs139428292, assets/carrier_genes.tsv) gives the TAR line; - 5. a hit called 1/1 inside a called deletion is shown as hemizygous, in the - carrier list and in the SF tier; a 1/1 hit at AnnotSV's SV_start (the - base before the deleted bases) stays homozygous; - 6. SMN1: one SMN1 copy (Parascopy) plus an SMN1 hit, and HBA: three alpha - genes plus an HBA2 hit, each give a "possibly biallelic" line; - 7. without any deletion caller's output the hit gives no line, and the - summary names the callers it could not read; - 8. both reports print the two-hit line and the hemizygous genotype; - 9. the couple screen (step 39): the RBM8A low-expression allele in one - partner and a deletion of RBM8A in the other is a shared gene (TAR risk - for a child); the allele in both partners, or in one alone, is set aside; - the assessed table counts the allele as no finding (0 of 1 hit); - 10. edges: a gene AnnotSV names that ends at the padding base gets no line; - 0 alpha-globin genes pair with nothing; a het hit inside a deletion - phased in cis is no second hit; a HIGH-impact record keeps its phase set - (trans gives biallelic); with a cis and a trans hit in one gene only the - trans one carries the SF two-hit note; an unreadable carrier list is a - note on the SF card. -""" -import gzip -import os -import shutil -import subprocess -import sys -import tempfile - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -import collect_summary # noqa: E402 -import render_report # noqa: E402 -sys.path.insert(0, os.path.join(REPO, "tests")) -from og_binary import og_binary # noqa: E402 - -FAILS = [] - - -def check(name, ok, detail=""): - print(("[PASS] " if ok else "[FAIL] ") + name + ("" if ok else " -- " + str(detail)[:1500])) - if not ok: - FAILS.append(name) - - -def put(path, text, gz=False): - os.makedirs(os.path.dirname(path), exist_ok=True) - with (gzip.open(path, "wt") if gz else open(path, "w")) as f: - f.write(text) - return path - - -VCF_HEAD = "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" -ANNOTSV_COLS = ("SV_chrom\tSV_start\tSV_end\tSV_length\tSV_type\tGene_name\tACMG_class\tCaller\tExon_edge\t" - "Exon_edge_detail\tPopulation_AF\tPopulation_source\tCommon\tFORMAT\tS\n") -# MUTYH spans chr1:45329163-45340893 (assets/carrier_genes.tsv). -SNV = ("chr1", 45332000, "C", "T") -DEL = ("chr1", 45335000, 45337000) -INSIDE = ("chr1", 45336000, "G", "A") -# RBM8A c.-21G>A (rs139428292), on the plus strand C>T. -TAR_ALLELE = ("chr1", 145927447, "C", "T") - - -def hit(v, gene, gt, ps=None): - chrom, pos, ref, alt = v - fmt, val = ("GT:GQ:DP:PS", f"{gt}:50:30:{ps}") if ps else ("GT:GQ:DP", f"{gt}:50:30") - return (f"{chrom}\t{pos}\t.\t{ref}\t{alt}\t50\tPASS\tGENEINFO={gene}:1;CLNSIG=Pathogenic;" - f"CLNREVSTAT=criteria_provided,_multiple_submitters,_no_conflicts\t{fmt}\t{val}\n") - - -def sv(span, genes, gt="0/1", ps=None): - chrom, a, b = span - fmt, val = ("GT:PS", f"{gt}:{ps}") if ps else ("GT", gt) - return f"{chrom}\t{a}\t{b}\t-{b - a}\tDEL\t{genes}\t4\tmanta\tyes\t.\t\t\tno\t{fmt}\t{val}\n" - - -SMN_COLS = "chrom\tstart\tend\tagCN\tpsCN\tpsCN_qual\tpsCN_filter\tagCN_qual\tagCN_filter\thomologous_regions\n" -INTERVAL_COLS = "locus\tname\tchrom\tstart\tend\tstatus\tcopies\texpected\tchange\testimate\treason\n" - - -def step35(d, smn1="2", intervals="", hba_genes=None): - """Step 35's tables: Parascopy's SMN table (SMN1 exon 7 at chr5:70951946, - SMN2's at chr5:70076526), the interval table and the HBA count.""" - put(f"{d}/paralogs/S_smn_copy_number.tsv", SMN_COLS - + f"chr5\t70951000\t70952000\t4\t{smn1},2\t40,40\tPASS\t40\tPASS\tchr5:70076000-70077000\n") - if intervals: - put(f"{d}/paralogs/S_cn_intervals.tsv", INTERVAL_COLS + intervals) - if hba_genes is not None: - put(f"{d}/paralogs/S_hba_copy_number.tsv", - f"metric\tvalue\nstatus\tcalled\nalpha_globin_genes\t{hba_genes}\nestimate\t{hba_genes}.02\n" - "message\tone gene lost on one chromosome\n") - - -def sample(work, name, hits, annotsv=None): - d = os.path.join(work, name, "S") - put(f"{d}/clinvar/S_clinvar_hits.vcf", VCF_HEAD + "".join(hits)) - if annotsv is not None: - put(f"{d}/annotsv/S_sv_class4_5_checked.tsv", ANNOTSV_COLS + annotsv) - return d - - -def carrier(d): - return collect_summary.collect("S", d)["sections"]["carrier"]["data"] - - -def two(d, gene): - return [x for x in (carrier(d).get("two_hits") or {}).get("genes", []) if x["gene"] == gene] - - -def sf(d): - """The SF tier of the folder: from the hits and the clinical VCF, which is - a VCF with no record when the case gives none (the Nextflow layout).""" - v = f"{d}/clinical/S_clinical.vcf.gz" - if not os.path.exists(v): - put(v, VCF_HEAD, gz=True) - return collect_summary.collect("S", d)["sections"]["clinical"]["data"].get("acmg_sf") or {} - - -def main(): - work = tempfile.mkdtemp(prefix="two_hits_") - try: - # 1. a MUTYH exon deletion plus a heterozygous hit: possibly biallelic - d = sample(work, "mutyh", [hit(SNV, "MUTYH", "0/1")], sv(DEL, "MUTYH")) - t = two(d, "MUTYH") - check("MUTYH deletion + het hit: one possibly biallelic line", - len(t) == 1 and t[0]["verdict"] == "possibly biallelic", t) - check("MUTYH deletion + het hit: the line names the deletion and the hit, phase unknown", - t and "DEL chr1:45335000-45337000" in t[0]["deletion"] and t[0]["variant"] == "chr1:45332000 C>T" - and "possibly in trans" in t[0]["why"], t) - s = sf(d) - rep = [x for x in s["clinvar_hits"] if x["gene"] == "MUTYH"] - check("MUTYH deletion + het hit: the SF tier reports the hit, counted with the deletion", - len(rep) == 1 and "possibly biallelic with DEL" in (rep[0].get("two_hit") or ""), rep) - check("MUTYH deletion + het hit: nothing left apart", - not [x for x in s["not_reported"] if x["gene"] == "MUTYH"], s["not_reported"]) - - # 2. read-backed phase on different copies: biallelic - d = sample(work, "trans", [hit(SNV, "MUTYH", "0|1", ps=45330000)], sv(DEL, "MUTYH", "1|0", ps=45330000)) - t = two(d, "MUTYH") - check("phased in trans: biallelic", len(t) == 1 and t[0]["verdict"] == "biallelic" - and "in trans" in t[0]["why"], t) - # a different phase set links nothing: possibly - d = sample(work, "otherps", [hit(SNV, "MUTYH", "0|1", ps=45330000)], sv(DEL, "MUTYH", "1|0", ps=45000000)) - t = two(d, "MUTYH") - check("phased in two different blocks: possibly biallelic", t and t[0]["verdict"] == "possibly biallelic", t) - - # 3. in cis: no line, and the SF tier keeps the one hit apart - d = sample(work, "cis", [hit(SNV, "MUTYH", "1|0", ps=45330000)], sv(DEL, "MUTYH", "1|0", ps=45330000)) - check("phased in cis: no two-hit line", two(d, "MUTYH") == [], two(d, "MUTYH")) - s = sf(d) - check("phased in cis: the SF tier lists the one hit apart, as before", - not [x for x in s["clinvar_hits"] if x["gene"] == "MUTYH"] - and [x for x in s["not_reported"] if x["gene"] == "MUTYH"], s) - - # 4. TAR: the 1q21.1 deletion of step 35 plus the RBM8A low-expression allele - tar_row = ("1q21.1_TAR_syndrome_proximal_BP2-BP3\t1q21.1_TAR_syndrome_proximal_BP2-BP3_t1\tchr1\t145686999\t" - "146048495\tcalled\t1\t2\tdeletion\t1.04\t.\n") - d = sample(work, "tar", [hit(TAR_ALLELE, "RBM8A", "1/1")]) - step35(d, intervals=tar_row) - c = carrier(d) - t = [x for x in (c.get("two_hits") or {}).get("genes", []) if x["gene"] == "RBM8A"] - check("TAR: the 1q21.1 deletion + the RBM8A allele give one RBM8A line", - len(t) == 1 and t[0]["verdict"] == "biallelic", c.get("two_hits")) - line = [x for x in render_report.carrier_lines(c) if "RBM8A" in x and "both copies" in x] - check("TAR: the line names the condition, the deletion and the allele", - len(line) == 1 and "Thrombocytopenia-absent radius syndrome" in line[0] - and "1q21.1_TAR_syndrome" in line[0] and "rs139428292" in line[0], line) - check("TAR: the allele alone stays set aside, never a carrier finding", - [x for x in c["set_aside"] if x["gene"] == "RBM8A"] and not - [x for x in c["carriers"] if x["gene"] == "RBM8A"], c) - # the allele without the deletion: no line - d = sample(work, "tar_alone", [hit(TAR_ALLELE, "RBM8A", "0/1")]) - step35(d, intervals=tar_row.replace("\t1\t2\tdeletion\t1.04", "\t2\t2\tno change\t2.01")) - check("TAR allele without a deletion: no line", two(d, "RBM8A") == [], two(d, "RBM8A")) - - # 5. a hit called 1/1 inside the deletion: hemizygous - d = sample(work, "hemi", [hit(INSIDE, "MUTYH", "1/1")], sv(DEL, "MUTYH")) - c = carrier(d) - item = [x for x in c["carriers"] if x["gene"] == "MUTYH"] - check("1/1 inside a deletion: the carrier list says hemizygous", - len(item) == 1 and item[0]["genotype"] == "hemizygous" and "inside DEL" in item[0].get("hemizygous", ""), - item) - t = two(d, "MUTYH") - check("1/1 inside a deletion: biallelic, the variant on the one copy left", - t and t[0]["verdict"] == "biallelic" and t[0]["genotype"] == "hemizygous", t) - rep = [x for x in sf(d)["clinvar_hits"] if x["gene"] == "MUTYH"] - check("1/1 inside a deletion: the SF tier says hemizygous", rep and rep[0]["genotype"] == "hemizygous", rep) - d = sample(work, "hom_out", [hit(SNV, "MUTYH", "1/1")], sv(DEL, "MUTYH")) - item = [x for x in carrier(d)["carriers"] if x["gene"] == "MUTYH"] - check("1/1 outside the deletion stays hom", item and item[0]["genotype"] == "hom", item) - # AnnotSV's SV_start is the base before the deleted bases: a 1/1 hit there is not inside - d = sample(work, "hom_edge", [hit(("chr1", DEL[1], "G", "A"), "MUTYH", "1/1")], sv(DEL, "MUTYH")) - item = [x for x in carrier(d)["carriers"] if x["gene"] == "MUTYH"] - check("1/1 at AnnotSV's SV_start (the padding base) stays hom", item and item[0]["genotype"] == "hom", item) - d = sample(work, "hom_first", [hit(("chr1", DEL[1] + 1, "G", "A"), "MUTYH", "1/1")], sv(DEL, "MUTYH")) - item = [x for x in carrier(d)["carriers"] if x["gene"] == "MUTYH"] - check("1/1 at the first deleted base is hemizygous", item and item[0]["genotype"] == "hemizygous", item) - - # 6. SMN1 and HBA - d = sample(work, "smn", [hit(("chr5", 70951950, "G", "A"), "SMN1", "0/1")]) - step35(d, smn1="1") - t = two(d, "SMN1") - check("one SMN1 copy + an SMN1 hit: a line", t and "one SMN1 copy" in t[0]["deletion"], t) - d = sample(work, "smn2", [hit(("chr5", 70951950, "G", "A"), "SMN1", "0/1")]) - step35(d, smn1="2") - check("two SMN1 copies + an SMN1 hit: no line", two(d, "SMN1") == [], two(d, "SMN1")) - d = sample(work, "hba", [hit(("chr16", 173600, "C", "T"), "HBA2", "0/1")]) - step35(d, hba_genes="3") - t = two(d, "HBA2") - check("three alpha genes + an HBA2 hit: possibly biallelic", - t and t[0]["verdict"] == "possibly biallelic" and "3 of 4 alpha-globin genes" in t[0]["deletion"], t) - d = sample(work, "hba4", [hit(("chr16", 173600, "C", "T"), "HBA2", "0/1")]) - step35(d, hba_genes="4") - check("four alpha genes + an HBA2 hit: no line", two(d, "HBA2") == [], two(d, "HBA2")) - - # 7. no deletion caller: no line, and the gap is named - d = sample(work, "none", [hit(SNV, "MUTYH", "0/1")]) - t = carrier(d).get("two_hits") or {"genes": None, "deletions_not_read": []} - check("no deletion caller: no line", t["genes"] == [], t) - check("no deletion caller: the callers not read are named", - any("AnnotSV" in x for x in t["deletions_not_read"]) - and any("step 35" in x for x in t["deletions_not_read"]), t) - check("no deletion caller: the report line says so", - any("not read: AnnotSV" in x for x in render_report.carrier_lines(carrier(d))), - render_report.carrier_lines(carrier(d))) - - # 8. both reports - d = sample(work, "report", [hit(INSIDE, "MUTYH", "1/1"), hit(("chr1", 45330500, "A", "G"), "MUTYH", "0/1")], - sv(DEL, "MUTYH")) - # step 23's clinical VCF, with no record of its own: the SF card is drawn from it - put(f"{d}/clinical/S_clinical.vcf.gz", VCF_HEAD, gz=True) - summ = collect_summary.collect("S", d) - txt, html = render_report.text_report(summ), render_report.html_report(summ) - check("text report: the two-hit line", "Biallelic: MUTYH" in txt, txt[-3000:]) - check("HTML report: the two-hit line", "Biallelic: MUTYH" in html) - check("text report: the hemizygous genotype", "45336000 G>A" not in txt and "hemizygous (called homozygous " - "inside DEL" in txt, txt[-3000:]) - check("text report: the SF row carries the two-hit detail", "biallelic with DEL chr1:45335000-45337000" in txt, - txt[-3000:]) - check("HTML report: the SF row carries the two-hit detail", "biallelic with DEL chr1:45335000-45337000" in html) - - # 9. the couple screen (og couple screen): hypomorph x null is a shared gene, hypomorph x hypomorph is not - tar = sample(work, "couple_tar", [hit(TAR_ALLELE, "RBM8A", "0/1")]) - empty = sample(work, "couple_empty", []) - cw = f"{work}/couple_run" - ped = put(f"{cw}/pedigree.tsv", "sample\tsex\tfather\tmother\tpartner\nA\tfemale\t.\t.\tB\n" - "B\tmale\t.\t.\tA\n") - # B's null allele: a 300 kb Delly deletion over RBM8A's exons (no RDCN, so read depth is no question) - null_sv = put(f"{cw}/B_sv.vcf", "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\t" - "FORMAT\tS\nchr1\t145700000\tDEL0\tN\t\t.\tPASS\tSVTYPE=DEL;END=146000000\tGT\t0/1\n") - exons = put(f"{cw}/exons.bed", "".join(f"chr1\t{s_}\t{s_ + 100}\n" for s_ in range(145921000, 145928000, 1200))) - - def couple(name, b_hits, *extra): - out = f"{cw}/{name}" - r = subprocess.run([og_binary(), "couple", "screen", "--pedigree", ped, - "--genes", os.path.join(REPO, "assets", "carrier_genes.tsv"), - "--clinvar", f"A={tar}/clinvar/S_clinvar_hits.vcf", - "--clinvar", f"B={b_hits}/clinvar/S_clinvar_hits.vcf", *extra, "--out-dir", out], - capture_output=True, text=True) - rows, md_text = [], "" - if r.returncode == 0: - with open(f"{out}/couple_A_B.tsv") as f: - head = f.readline().rstrip("\n").split("\t") - rows = [dict(zip(head, x.rstrip("\n").split("\t"))) for x in f] - with open(f"{out}/couple_A_B.md") as f: - md_text = f.read() - return r, rows, md_text - r, rows, _ = couple("null", empty, "--delly", f"B={null_sv}", "--exons", exons) - x = [y for y in rows if y["gene"] == "RBM8A"] - check("couple: RBM8A allele in A + RBM8A deletion in B is a shared gene, 1 in 4", - len(x) == 1 and x[0]["result"] == "shared" and x[0]["offspring_risk"] == "1 in 4 per pregnancy" - and "rs139428292" in x[0]["findings_a"] and "hypomorph" in x[0]["explanation"], (r.stderr[-500:], rows)) - r, rows, _ = couple("both", tar) - check("couple: the RBM8A allele in both partners is set aside, no row", - r.returncode == 0 and not [y for y in rows if y["gene"] == "RBM8A"], (r.stderr[-500:], rows)) - r, rows, md_text = couple("one", empty) - check("couple: the RBM8A allele in one partner alone is set aside, no row", - r.returncode == 0 and not [y for y in rows if y["gene"] == "RBM8A"], (r.stderr[-500:], rows)) - check("couple run: the RBM8A allele counts as no finding of the listed genes (0 / 1) and is a gap line", - r.returncode == 0 and "| A | 0 / 1 |" in md_text and "Set aside for A" in md_text, - (r.returncode, r.stderr[-500:], md_text[-1500:])) - - # 10. edges - # MUTYH ends at chr1:45340893: a deletion whose padding base is that one removes none of it - d = sample(work, "pad_gene", [hit(SNV, "MUTYH", "0/1")], sv(("chr1", 45340893, 45342000), "MUTYH")) - check("a gene AnnotSV names that ends at the padding base: no line", two(d, "MUTYH") == [], two(d, "MUTYH")) - d = sample(work, "hba0", [hit(("chr16", 173600, "C", "T"), "HBA2", "0/1")]) - step35(d, hba_genes="0") - check("no alpha-globin gene left + an HBA2 hit: no line", two(d, "HBA2") == [], two(d, "HBA2")) - d = sample(work, "cis_inside", [hit(INSIDE, "MUTYH", "1|0", ps=45330000)], - sv(DEL, "MUTYH", "1|0", ps=45330000)) - check("a het hit inside a deletion, phased in cis: no line", two(d, "MUTYH") == [], two(d, "MUTYH")) - # a HIGH-impact record (no ClinVar hit) phased in trans with the deletion - csq = ("##fileformat=VCFv4.2\n##INFO=\n" - "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" - f"chr1\t{SNV[1]}\t.\tC\tT\t50\tPASS\tCSQ=T|stop_gained|HIGH|MUTYH|0.0001\tGT:GQ:DP:AD:PS\t" - "0|1:50:30:15,15:45330000\n") - d = sample(work, "high_trans", [], sv(DEL, "MUTYH", "1|0", ps=45330000)) - put(f"{d}/clinical/S_clinical.vcf.gz", csq, gz=True) - rep = [x for x in sf(d)["high_impact"] if x["gene"] == "MUTYH"] - check("a HIGH-impact hit phased in trans with the deletion: biallelic in the SF tier", - len(rep) == 1 and (rep[0].get("two_hit") or "").startswith("biallelic with DEL"), rep) - # one hit in cis, one in trans: only the trans one carries the note - cis_hit = ("chr1", 45330500, "A", "G") - d = sample(work, "cis_trans", [hit(cis_hit, "MUTYH", "1|0", ps=45330000), - hit(SNV, "MUTYH", "0|1", ps=45330000)], sv(DEL, "MUTYH", "1|0", ps=45330000)) - rows = {x["variant"]: x.get("two_hit") for x in sf(d)["clinvar_hits"] if x["gene"] == "MUTYH"} - check("a cis and a trans hit in one gene: only the trans hit carries the two-hit note", - (rows.get("chr1:45332000 C>T") or "").startswith("biallelic with DEL") - and "chr1:45330500 A>G" in rows and rows["chr1:45330500 A>G"] is None, rows) - # an unreadable carrier list: the SF card says what was not read - d = sample(work, "no_list", [hit(SNV, "MUTYH", "0/1")], sv(DEL, "MUTYH")) - broken = os.path.join(work, "assets-broken") - shutil.copytree(collect_summary.ASSETS_DIR, broken) - with open(os.path.join(broken, "carrier_genes.tsv")) as f: - text = f.read() - with open(os.path.join(broken, "carrier_genes.tsv"), "w") as f: - f.write(text.replace("gene\tchrom\tstart", "name\tchrom\tstart", 1)) - real, collect_summary.ASSETS_DIR = collect_summary.ASSETS_DIR, broken - try: - s = sf(d) - finally: - collect_summary.ASSETS_DIR = real - check("an unreadable carrier list: a note on the SF card", - "carrier_genes.tsv" in (s.get("two_hit_note") or "") - and any("carrier_genes.tsv" in x for x in render_report.sf_apart_lines(s)), s.get("two_hit_note")) - finally: - shutil.rmtree(work, ignore_errors=True) - print(f"\n{len(FAILS)} failed" if FAILS else "\nall passed") - return 1 if FAILS else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/test_haplogroup_explore.py b/tests/test_haplogroup_explore.py deleted file mode 100755 index 635392884..000000000 --- a/tests/test_haplogroup_explore.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python3 -"""Both reports point to where a haplogroup can be explored further (steps 12 and 37). - -Each case writes the haplogroup files of one made-up sample S into a bash -output folder, collects the summary and renders both reports: - - 1. a male sample (Yleaf table with R-M269, haplogrep3 with H5a7): the Y and - the mtDNA links are built from those names, in the text and the HTML - report, with the note that nothing is uploaded - 2. Yleaf's paragroup form R-M269*(xR-L23): the links use R-M269 - 3. a female sample (no Yleaf table): no Y links; the mtDNA links stay - 4. no call (Hg NA, haplogroup '.'): no links at all - 5. a branch name with characters a URL cannot carry raw is escaped - -The branch names are the fixture's and public examples, not anyone's result. - -Run: python3 tests/test_haplogroup_explore.py -""" -import os -import sys -import tempfile - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -import collect_summary # noqa: E402 -import render_report # noqa: E402 - -FAILS = [] -Y_HEAD = "Sample_name\tHg\tHg_marker\tTotal_reads\tValid_markers\tQC-score\tQC-1\tQC-2\tQC-3\n" - - -def check(desc, ok, detail=""): - print(f"{'PASS' if ok else 'FAIL'} {desc}" + ("" if ok else f": {detail}")) - if not ok: - FAILS.append(desc) - - -def put(path, text): - os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, "w") as f: - f.write(text) - - -def render(d): - summ = collect_summary.collect("S", d) - return summ, render_report.text_report(summ), render_report.html_report(summ) - - -def sample(tmp, name, y_hg=None, mt_hg="H5a7"): - d = os.path.join(tmp, name) - put(f"{d}/mito/S_haplogroup.txt", f'"SampleID"\t"Haplogroup"\t"Rank"\n"S"\t"{mt_hg}"\t"0.98"\n') - if y_hg is not None: - put(f"{d}/y_haplogroup/S_y_haplogroup.txt", Y_HEAD + f"S_sorted\t{y_hg}\tM269\t100\t42\t0.97\t1\t1\t0.97\n") - return d - - -def main(): - with tempfile.TemporaryDirectory() as tmp: - # 1. male - summ, txt, html = render(sample(tmp, "male", y_hg="R-M269")) - for url in ("https://discover.familytreedna.com/y-dna/R-M269", "https://www.yfull.com/tree/R-M269/", - "https://discover.familytreedna.com/mtdna/H5a7", "https://www.yfull.com/mtree/H5a7/"): - check(f"male: {url} in the text report", url in txt, txt) - check(f"male: {url} as a link in the HTML report", f'' in html, "") - check("male: the docs section for the routes is linked", - "docs/37-y-haplogroup.md#explore-your-branch-further" in txt - and "docs/12-mito-haplogroup.md#explore-your-branch-further" in txt, txt) - check("male: both reports say the pipeline uploads nothing", - "Nothing is uploaded by this pipeline" in txt and "Nothing is uploaded by this pipeline" in html, txt) - - # 2. paragroup - summ, txt, html = render(sample(tmp, "para", y_hg="R-M269*(xR-L23)")) - check("paragroup: the link is built from R-M269", - "https://www.yfull.com/tree/R-M269/" in txt and "y-dna/R-M269*" not in txt and "%2A" not in txt, txt) - - # 3. female: no Yleaf table, so no Y section and no Y link - summ, txt, html = render(sample(tmp, "female")) - check("female: no Y link", "y-dna/" not in txt and "yfull.com/tree/" not in txt - and "y-dna/" not in html and "yfull.com/tree/" not in html, txt) - check("female: the mtDNA links stay", "https://discover.familytreedna.com/mtdna/H5a7" in txt - and "https://www.yfull.com/mtree/H5a7/" in html, txt) - - # 4. no call - summ, txt, html = render(sample(tmp, "nocall", y_hg="NA", mt_hg="")) - check("no call: no explore links", "discover.familytreedna.com" not in txt and "yfull.com" not in txt - and "discover.familytreedna.com" not in html, txt) - check("no call: the Y section still says insufficient markers", "insufficient markers" in txt, txt) - - # 5. escaping (a made-up PhyloTree-style name) - summ, txt, html = render(sample(tmp, "esc", mt_hg="L3e2b+152")) - check("escaped: '+' is encoded in the mtDNA links", "mtdna/L3e2b%2B152" in txt and "mtree/L3e2b%2B152/" in txt, txt) - - print(f"\n{len(FAILS)} failed" if FAILS else "\nall passed") - return 1 if FAILS else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/test_hla_report.py b/tests/test_hla_report.py deleted file mode 100644 index 94e1f3812..000000000 --- a/tests/test_hla_report.py +++ /dev/null @@ -1,246 +0,0 @@ -#!/usr/bin/env python3 -"""The HLA section of the reports (step 08): bin/hla_alleles.tsv, what -og summary collect derives from T1K's genotype TSV, and what -bin/render_report.py prints. Synthetic T1K rows only. - -Checks: - 1. bin/hla_alleles.tsv: its columns, an allele group per row (B*27, - B*57:01), kind drug or disease, PubMed ids as numbers, a citation, no - duplicate allele, and every allele the report must cover (the drug - alleles of docs/08-hla-typing.md, B*27, C*06:02, DQB1*06:02, - DRB1*15:01 and the shared-epitope DRB1 alleles); each planted problem - (a wrong kind, a PubMed id that is not a number, a duplicate, a missing - allele, a three-field allele) is caught; - 2. the coeliac DQ line from DQA1 and DQB1: DQ2.5 likely cis with DRB1*03, - DQ2.5 possible trans without it, DQ2.2, DQ8, none; the red control - swaps DQA1*05:01 for DQA1*01:01 so DQ2.5 must not be derived; a - low-confidence DQA1 gives "not determined"; - 3. the DQB1*02 copies: 0, 1, 2 from two alleles, and 2 from T1K's one - homozygous allele; - 4. the allele table: B*57:01 present and absent, B*27 matched by group, - a candidate list of which only some are B*57:01 is "uncertain", a - low-confidence locus is "not determined"; - 5. a quality-0 allele with 12 candidates is stored with three and printed - as "no confident call (12 candidates", in both reports; - 6. the HFE row carries its label in both reports; - 7. both reports print the derived lines and the not-a-diagnosis note, and - every summary validates against tests/schema/summary.schema.json. - -Run: python3 tests/test_hla_report.py -""" -import json -import os -import re -import shutil -import sys -import tempfile - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -sys.path.insert(0, os.path.join(REPO, "tests", "schema")) -import collect_summary as cs # noqa: E402 -import render_report # noqa: E402 -import validate # noqa: E402 - -TABLE = os.path.join(REPO, "bin", "hla_alleles.tsv") -COLUMNS = ["allele", "kind", "association", "pmids", "citation"] -REQUIRED = ["A*31:01", "B*15:02", "B*57:01", "B*58:01", "B*27", "C*06:02", "DQB1*06:02", "DRB1*15:01", - "DRB1*01:01", "DRB1*04:01", "DRB1*04:04", "DRB1*04:05", "DRB1*04:08", "DRB1*14:02"] -ALLELE = re.compile(r"^[A-Z]+[0-9]?\*[0-9]{2}(:[0-9]{2,3})?$") -FAILS = 0 - - -def check(desc, ok, detail=""): - global FAILS - print(f"[{'PASS' if ok else 'FAIL'}] {desc}{'' if ok else ' -- ' + str(detail)[:800]}") - if not ok: - FAILS += 1 - - -def table_errors(path): - """Every schema problem of an allele table, as a list of strings.""" - errs, head, seen = [], None, set() - with open(path) as f: - for n, line in enumerate(f, 1): - if line.startswith("#") or not line.strip(): - continue - c = line.rstrip("\n").split("\t") - if head is None: - head = c - if c != COLUMNS: - errs.append(f"header {c} is not {COLUMNS}") - continue - if len(c) != len(COLUMNS): - errs.append(f"line {n}: {len(c)} columns") - continue - r = dict(zip(COLUMNS, c)) - if not ALLELE.match(r["allele"]): - errs.append(f"line {n}: allele {r['allele']!r} is not a one- or two-field group") - if r["kind"] not in ("drug", "disease"): - errs.append(f"line {n}: kind {r['kind']!r}") - if not r["association"].strip(): - errs.append(f"line {n}: no association") - if not r["pmids"] or not all(p.isdigit() for p in r["pmids"].split(",")): - errs.append(f"line {n}: pmids {r['pmids']!r}") - if not r["citation"].strip(): - errs.append(f"line {n}: no citation") - if r["allele"] in seen: - errs.append(f"line {n}: duplicate {r['allele']}") - seen.add(r["allele"]) - errs += [f"missing {a}" for a in REQUIRED if a not in seen] - return errs - - -def t1k(rows): - """T1K's genotype TSV from (gene, allele1, q1, allele2, q2); allele2 None - writes T1K's one-allele row ('.', 0, -1).""" - out = [] - for gene, a1, q1, a2, q2 in rows: - if a2 is None: - out.append(f"{gene}\t1\t{a1}\t30\t{q1}\t.\t0\t-1\n") - else: - out.append(f"{gene}\t2\t{a1}\t30\t{q1}\t{a2}\t20\t{q2}\n") - return "".join(out) - - -# The label og summary collect gives T1K's HFE row. -HLA_HFE_LABEL = ("T1K sequence allele, not the HFE C282Y/H63D genotype: that genotype comes from the " - "variant calls (ClinVar section)") - - -def summary(work, name, rows): - d = os.path.join(work, name, "S") - os.makedirs(os.path.join(d, "hla_t1k"), exist_ok=True) - with open(os.path.join(d, "hla_t1k", "S_hla_genotype.tsv"), "w") as f: - f.write(t1k(rows)) - return cs.collect("S", d) - - -def main(): - schema = json.load(open(os.path.join(REPO, "tests", "schema", "summary.schema.json"))) - work = tempfile.mkdtemp() - try: - # 1. the table - errs = table_errors(TABLE) - check("bin/hla_alleles.tsv passes its schema and lists every required allele", not errs, errs) - derived = summary(work, "table", [("HLA-B", "B*57:01:01", 40, "B*08:01:01", 40)])["sections"]["hla"]["data"] - check("og summary collect reads the same rows", [r["allele"] for r in derived["derived"]["alleles"]] - == [l.split("\t")[0] for l in open(TABLE) if not l.startswith("#") and l.strip()][1:]) - good = open(TABLE).read() - for desc, bad in (("a wrong kind", good.replace("\tdrug\t", "\tdrugs\t", 1)), - ("a PubMed id that is not a number", good.replace("\t21428769\t", "\tPMID21428769\t")), - ("a duplicate allele", good + "B*27\tdisease\tx\t1\ty\n"), - ("a missing required allele", good.replace("C*06:02\t", "C*07:02\t")), - ("a three-field allele", good.replace("B*58:01\t", "B*58:01:01\t"))): - p = os.path.join(work, "bad.tsv") - with open(p, "w") as f: - f.write(bad) - check(f"control: the table check catches {desc}", bool(table_errors(p))) - - # 2. coeliac DQ - cis = summary(work, "cis", [("HLA-DQA1", "DQA1*05:01:01", 40, "DQA1*01:01:01", 40), - ("HLA-DQB1", "DQB1*02:01:01", 40, "DQB1*05:01:01", 40), - ("HLA-DRB1", "DRB1*03:01:01", 40, "DRB1*01:01:01", 40), - ("HLA-B", "B*57:01:01", 40, "B*08:01:01", 40)]) - c = cis["sections"]["hla"]["data"]["derived"]["coeliac"] - check("DQA1*05:01 with DQB1*02:01 and DRB1*03 -> DQ2.5 present, likely cis; one DQB1*02", - c["status"] == "derived" and c["dq2_5"] == "present" and c["dq2_5_phase"].startswith("likely cis") - and c["dq2_2"] == "absent" and c["dq8"] == "absent" and c["dqb1_02_dose"] == 1, c) - check("cis summary validates", not validate.validate(schema, json.loads(json.dumps(cis)))) - trans = summary(work, "trans", [("HLA-DQA1", "DQA1*05:05:01", 40, "DQA1*02:01:01", 40), - ("HLA-DQB1", "DQB1*03:01:01", 40, "DQB1*02:02:01", 40), - ("HLA-DRB1", "DRB1*11:01:01", 40, "DRB1*07:01:01", 40)]) - c = trans["sections"]["hla"]["data"]["derived"]["coeliac"] - check("DQA1*05:05 and DQB1*02:02 without DRB1*03 -> DQ2.5 possible trans (and DQ2.2 from the DR7 pair)", - c["dq2_5"] == "present" and c["dq2_5_phase"].startswith("possible trans") and c["dq2_2"] == "present", c) - dq22 = summary(work, "dq22", [("HLA-DQA1", "DQA1*02:01:01", 40, "DQA1*01:02:01", 40), - ("HLA-DQB1", "DQB1*02:02:01", 40, "DQB1*06:02:01", 40)]) - c = dq22["sections"]["hla"]["data"]["derived"]["coeliac"] - check("DQA1*02:01 with DQB1*02:02 and no DQA1*05 -> DQ2.2 only", - c["dq2_5"] == "absent" and c["dq2_2"] == "present" and c["dq8"] == "absent" and c["dq2_5_phase"] is None, c) - dq8 = summary(work, "dq8", [("HLA-DQA1", "DQA1*03:01:01", 40, "DQA1*01:01:01", 40), - ("HLA-DQB1", "DQB1*03:02:01", 40, "DQB1*05:01:01", 40)]) - c = dq8["sections"]["hla"]["data"]["derived"]["coeliac"] - check("DQA1*03 with DQB1*03:02 -> DQ8, no DQB1*02", - c["dq8"] == "present" and c["dq2_5"] == "absent" and c["dqb1_02_dose"] == 0, c) - red = summary(work, "red", [("HLA-DQA1", "DQA1*01:01:01", 40, "DQA1*01:02:01", 40), - ("HLA-DQB1", "DQB1*02:01:01", 40, "DQB1*05:01:01", 40), - ("HLA-DRB1", "DRB1*03:01:01", 40, "DRB1*01:01:01", 40)]) - c = red["sections"]["hla"]["data"]["derived"]["coeliac"] - check("red control: DQA1*05:01 swapped for DQA1*01:01 -> DQ2.5 must not be derived", - c["dq2_5"] == "absent" and c["dq2_5_phase"] is None, c) - low = summary(work, "low", [("HLA-DQA1", "DQA1*05:01:01", 0, "DQA1*01:01:01", 40), - ("HLA-DQB1", "DQB1*02:01:01", 40, "DQB1*05:01:01", 40)]) - c = low["sections"]["hla"]["data"]["derived"]["coeliac"] - check("a low-confidence DQA1 -> coeliac DQ not determined, the DQB1*02 count still given", - c["status"] == "not_determined" and c["dq2_5"] is None and "HLA-DQA1 low confidence" in c["reason"] - and c["dqb1_02_dose"] == 1, c) - - # 3. DQB1*02 copies - two = summary(work, "two", [("HLA-DQB1", "DQB1*02:01:01", 40, "DQB1*02:02:01", 40)]) - hom = summary(work, "hom", [("HLA-DQB1", "DQB1*02:01:01", 40, None, None)]) - check("DQB1*02:01 / DQB1*02:02 -> two DQB1*02", - two["sections"]["hla"]["data"]["derived"]["coeliac"]["dqb1_02_dose"] == 2) - check("T1K's one homozygous DQB1*02:01 -> two DQB1*02", - hom["sections"]["hla"]["data"]["derived"]["coeliac"]["dqb1_02_dose"] == 2) - check("without DQA1 the coeliac line says DQA1 was not typed", - hom["sections"]["hla"]["data"]["derived"]["coeliac"]["reason"] == "HLA-DQA1 not typed") - - # 4. the allele table - def state(summ, allele): - return {a["allele"]: a["state"] for a in summ["sections"]["hla"]["data"]["derived"]["alleles"]}[allele] - check("B*57:01:01 -> B*57:01 present", state(cis, "B*57:01") == "present") - b7 = summary(work, "b7", [("HLA-B", "B*07:02:01", 40, "B*08:01:01", 40)]) - check("B*57:01 absent with B*07:02 / B*08:01, and B*58:01 absent in the same sample", - state(b7, "B*57:01") == "absent" and state(b7, "B*58:01") == "absent") - grp = summary(work, "b27", [("HLA-B", "B*27:05:02", 40, "B*57:03:01,B*57:01:01", 40), - ("HLA-C", "C*06:02:01", 0, "C*07:01:01", 40)]) - check("B*27:05:02 -> the B*27 group present", state(grp, "B*27") == "present") - check("a candidate list of B*57:03 and B*57:01 -> B*57:01 uncertain", state(grp, "B*57:01") == "uncertain") - check("a low-confidence HLA-C -> C*06:02 not determined", state(grp, "C*06:02") == "not_determined") - check("an untyped DQB1 -> DQB1*06:02 not determined", state(grp, "DQB1*06:02") == "not_determined") - - # 5. quality-0 candidate list, 6. HFE label - many = ",".join(f"DRB5*01:{i:02d}" for i in range(1, 13)) - cap = summary(work, "cap", [("HLA-DRB5", many, 0, None, None), - ("HLA-HFE", "HFE*001:01", 40, "HFE*002:01", 40), - ("HLA-DQA1", "DQA1*05:01:01", 40, "DQA1*01:01:01", 40), - ("HLA-DQB1", "DQB1*02:01:01", 40, "DQB1*05:01:01", 40), - ("HLA-B", "B*57:01:01", 40, "B*08:01:01", 40)]) - loci = {l["gene"]: l for l in cap["sections"]["hla"]["data"]["loci"]} - check("a quality-0 allele with 12 candidates is stored with three and the count", - loci["HLA-DRB5"]["alleles"] == ["DRB5*01:01,DRB5*01:02,DRB5*01:03"] - and loci["HLA-DRB5"]["candidates"] == [12], loci["HLA-DRB5"]) - check("the HFE row has its label", loci["HLA-HFE"].get("label") == HLA_HFE_LABEL, loci["HLA-HFE"]) - check("cap summary validates", not validate.validate(schema, json.loads(json.dumps(cap))), - validate.validate(schema, json.loads(json.dumps(cap)))) - txt, html = render_report.text_report(cap), render_report.html_report(cap) - check("text: the quality-0 locus is one short line", - "HLA-DRB5 no confident call (12 candidates, quality 0); low confidence" in txt - and "DRB5*01:04" not in txt, [l for l in txt.splitlines() if "DRB5" in l]) - check("HTML: the same", "no confident call (12 candidates, quality 0)" in html and "DRB5*01:04" not in html) - check("both reports label the HFE row", txt.count("not the HFE C282Y/H63D genotype") == 1 - and html.count("not the HFE C282Y/H63D genotype") == 1) - - # 7. derived lines and the note - want = ["Coeliac DQ: DQ2.5 present, phase unknown (HLA-DRB1 not typed); DQ2.2 absent; DQ8 absent", - "DQB1*02 copies: 1", - "B*57:01 present: abacavir hypersensitivity; flucloxacillin liver injury", - "C*06:02 not determined (HLA-C not typed)"] - check("text: the derived lines", all(w in txt for w in want), [l for l in txt.splitlines() if "DQ" in l or "*" in l]) - check("HTML: the derived lines", all(render_report.E(w) in html for w in want)) - check("both reports say it is information about alleles, not a diagnosis", - txt.count("not a diagnosis") >= 1 and render_report.E(render_report.HLA_DERIVED_NOTE) in html) - words = re.compile(r"\byou (should|must)\b|\bconsult\b|\btreat", re.I) - check("no clinical advice in the derived lines or the note", - not words.search(" ".join(render_report.hla_derived_lines(cap["sections"]["hla"]["data"]["derived"])) - + render_report.HLA_DERIVED_NOTE)) - check("control: the advice check finds advice", bool(words.search("you should consult"))) - finally: - shutil.rmtree(work) - print("\nRESULT:", "ALL PASS" if FAILS == 0 else f"{FAILS} FAILED") - return 1 if FAILS else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/test_lpa_kiv2.py b/tests/test_lpa_kiv2.py deleted file mode 100644 index e597f1bda..000000000 --- a/tests/test_lpa_kiv2.py +++ /dev/null @@ -1,478 +0,0 @@ -#!/usr/bin/env python3 -# og-leg: OG_REQUIRED -"""og cn call --locus LPA_KIV2: step 46's LPA KIV-2 repeat count from -short-read depth, with KIV-2 4925G>A beside it, and its Lp(a) section in both -reports. Part 6 runs without og; the rest needs it: OG names the binary -(OG=rust/target/release/og, or bin/og); without it they are skipped, and with -OG_REQUIRED set too the test fails. - -On `samtools bedcov` tables made up for each case (summed depth = mean depth -x exon length), 30x over the single-copy LPA exons, with 90% of the KIV-2 -depth from reads of MAPQ 0 unless a case says otherwise: - 1. the arithmetic: the KIV-2 block holds N + 2 copies of the exon pair over - seven reference pairs, so a block depth of 30 x (N + 2) / 14 gives N, for - N = 12 (GRCh38's own six units on each chromosome), 20 and 40; - 2. the interval: equal depths on every exon give a zero-width interval; - planted spread gives the interval sqrt(cv_kiv2^2 / 14 + cv_flank^2 / 15) - on N + 2, computed here by hand, and the interval contains the estimate; - twice the spread gives a wider interval; - 3. the class: a count whose whole interval is under 30 is low (29 with a - narrow interval too), over 45 is high, one whose interval reaches into - 30 to 45 is typical (29 and 47 with a wide interval); - 4. under 10x over the single-copy exons is 'too little depth', with no - count; no read of MAPQ 0 over KIV-2 (novoalign's default) is 'not - assessed', naming the cause; two empty tables (a BAM with no chr6) is - 'not assessed'; - 5. the CLI: `og cn bed --locus LPA_KIV2` prints the 29 regions, 14 KIV-2 - exons of 2,394 bp and 15 single-copy exons; `og cn call --table` writes - the metric table and exits 0; a table with other regions or a short row - exits 2; - 6. assets/traits.tsv has the two LPA risk variants with the alleles - bin/collect_summary.py counts, and the blood-test advice; - 7. both reports: a low count with one risk allele is flagged with both - reasons and the advice; a typical count with no risk allele is not - flagged; without step 42's table the variants are 'not read'; 'too - little depth' is shown as such; no card and no text section without - step 46's table (only the line in the list of steps that did not run); - KIV-2 4925G>A prints between the count and the risk variants in both - reports and on the live card, with its caveat: present with its reads - and copies, not assessed without step 46's 4925G>A table or with no - status row in it; - 8. KIV-2 4925G>A (null-allele) on mpileup lines made up for each case, at - the seven GRCh38 copies of the site: 5% of the reads with T is - 'present' with copies = fraction x (N + 2); reads with C only (the - reference) is 'not seen'; 3 reads with T is 'uncertain'; under 100 reads - is 'too little depth'; an unreadable pileup is 'not assessed', exit 0; - reads at other positions and read starts, ends and indels in the bases - column are not counted; when step 46's table shows the aligner left out - the reads of MAPQ 0, 'not assessed'; the reading goes to its own table and step 46's - table is left byte for byte; without a count the copies are '.'; an - output that cannot be written exits 2. `og cn marker-read` on step 46's - table and `og cn call --marker` (what step 46 and the LPA process run) - write the same 4925G>A table. - -Run: OG=rust/target/release/og python3 tests/test_lpa_kiv2.py -""" -import csv -import math -import os -import subprocess -import sys -import tempfile - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -CATALOG = os.path.join(REPO, "bin", "cn_loci.tsv") -OG = os.environ.get("OG", "") -if OG and not os.path.realpath(OG).startswith(os.path.realpath(REPO) + os.sep): - # og cn finds bin/cn_loci.tsv beside its own binary. Guard's unit-tests job - # builds og in a temporary folder, so the tests run a copy that sits next - # to the catalog, as bin/.og/og-* does in a checkout. - import atexit, shutil # noqa: E401 - _og_dir = tempfile.mkdtemp(prefix="og-cn-") - atexit.register(shutil.rmtree, _og_dir, True) - shutil.copy2(OG, os.path.join(_og_dir, "og")) - shutil.copy2(os.path.join(REPO, "bin", "cn_loci.tsv"), os.path.join(_og_dir, "cn_loci.tsv")) - OG = os.environ["OG"] = os.path.join(_og_dir, "og") -sys.path.insert(0, os.path.join(REPO, "bin")) -from test_copy_number import cn # noqa: E402 pinned differential oracle - -FAILS = 0 -# The seven copies of KIV-2 4925G>A on GRCh38 (rust/crates/og-cn/src/markers.rs LPA_KIV2_4925). -KIV2_4925_COPIES = (160646214, 160640667, 160635123, 160629574, 160624030, 160618484, 160612932) -# The columns of step 42's traits table (og traits table --out). -TRAITS_COLUMNS = ["rsid", "gene", "chrom", "pos", "trait", "genotype", "status", "reading", "ancestry", - "note", "pmids"] -# The two LPA rows; named once here so no line of this file reads as an rsID with a genotype. -R1, R2 = "rs10455872", "rs3798220" -ROW = cn.load_catalog(CATALOG)["LPA_KIV2"] -REGIONS = cn.regions(ROW) -# in catalog order: the KIV-2 block, then the single-copy exons -KIV = [n for n, *_ in ROW["_targets"]] -FLANK = [n for n, *_ in ROW["_controls"]] -# The advice row of step 46's table. -ADVICE = ("A research estimate from read depth, not a diagnostic test. Lp(a) is measured in blood; a 2022 European " - "Atherosclerosis Society consensus advises measuring it at least once in every adult.") - - -def check(desc, ok, detail=""): - global FAILS - print(f"[{'PASS' if ok else 'FAIL'}] {desc}" + ("" if ok else f" -- {str(detail)[:800]}")) - if not ok: - FAILS += 1 - - -def depths(n_copies, flank=30.0, kiv_spread=0.0, flank_spread=0.0): - """Mean depth per region: the flanks at `flank`, the KIV-2 block at the - depth n_copies gives; a spread alternates +x and -x around each mean.""" - k = flank * (n_copies + 2) / 14 - m = {} - for i, n in enumerate(KIV): - m[n] = k * (1 + (kiv_spread if i % 2 == 0 else -kiv_spread)) - for i, n in enumerate(FLANK): - m[n] = flank * (1 + (flank_spread if i % 2 == 0 else -flank_spread)) - return m - - -def q1(m, share=0.9): - return {n: v * (1 - share) if n.startswith("kiv2_") else v for n, v in m.items()} - - -def bedcov(path, m, exact=False): - """samtools bedcov at these mean depths: whole sums, or with exact the - sums themselves, so each mean comes back as given.""" - with open(path, "w") as f: - for c, s, e, n in REGIONS: - v = m[n] * (e - s) - f.write(f"{c}\t{s}\t{e}\t{n}\t{v!r}\n" if exact else f"{c}\t{s}\t{e}\t{n}\t{round(v)}\n") - - -def og_cn(*args): - return subprocess.run([OG, "cn", *args], capture_output=True, text=True) - - -def read_table(path): - return dict(x.split("\t", 1) for x in open(path).read().splitlines()[1:]) if os.path.exists(path) else {} - - -def estimate(a, b, out): - """og cn call on step 46's two bedcov tables, writing its table.""" - return og_cn("call", "--catalog", CATALOG, "--locus", "LPA_KIV2", "--bedcov", a, "--bedcov-mapq1", b, - "--table", out) - - -def assess(m, q): - """Step 46's table for mean depths m (every read) and q (MAPQ >= 1).""" - with tempfile.TemporaryDirectory() as t: - a, b, o = (os.path.join(t, x) for x in ("a.tsv", "b.tsv", "o.tsv")) - if m: - bedcov(a, m, exact=True) - bedcov(b, q, exact=True) - else: - open(a, "w").close() - open(b, "w").close() - estimate(a, b, o) - return read_table(o) - - -def cv(vals): - mu = sum(vals) / len(vals) - return math.sqrt(sum((v - mu) ** 2 for v in vals) / (len(vals) - 1)) / mu - - -def pileup(path, per_site, extra=""): - """mpileup lines at the seven copies of 4925G>A (KIV2_4925_COPIES); - per_site is (C reads, T reads) at each.""" - with open(path, "w") as f: - for p in KIV2_4925_COPIES: - c, t_ = per_site - bases = "^]" + "C" * c + "t" * t_ + "$" - f.write(f"chr6\t{p}\tN\t{c + t_}\t{bases}\t{'I' * (c + t_)}\n") - f.write(extra) - - -def traits(): - """6. assets/traits.tsv holds the two LPA risk variants the report counts.""" - with open(os.path.join(REPO, "assets", "traits.tsv"), encoding="utf-8") as f: - table = {r["rsid"]: r for r in csv.DictReader((line for line in f if line.strip() and not line.startswith("#")), - delimiter="\t", quoting=csv.QUOTE_NONE)} - # The two risk alleles og summary collect counts for the Lp(a) flag. - for rs, alt in {"rs10455872": "G", "rs3798220": "C"}.items(): - r = table.get(rs, {}) - check(f"traits.tsv: {rs} is an LPA row whose alt ({alt}) is the risk allele the report counts", - r.get("gene") == "LPA" and r.get("alt") == alt, r) - check(f"traits.tsv: {rs} says the actionable test is a blood Lp(a) measurement", - "measure" in (r.get("note") or "").lower() and "blood" in (r.get("note") or ""), r) - - -def main(): - if not OG: - traits() - print("skip: OG is not set (OG=rust/target/release/og): only part 6 ran") - print("\nRESULT:", "ALL PASS" if FAILS == 0 else f"{FAILS} FAILED") - return 1 if FAILS or os.environ.get("OG_REQUIRED") else 0 - # 1. arithmetic - for n in (12, 20, 40): - r = assess(depths(n), q1(depths(n))) - check(f"arithmetic: a block at 30 x {n + 2} / 14 is {n} KIV-2 copies", - r["status"] == "estimated" and r["kiv2_copies"] == f"{n:.1f}", r) - # 2. interval - r = assess(depths(20), q1(depths(20))) - check("interval: equal depths give a zero-width interval", r["ci95_low"] == r["ci95_high"] == "20.0", r) - m = depths(20, kiv_spread=0.10, flank_spread=0.15) - r = assess(m, q1(m)) - rel = math.sqrt(cv([m[n] for n in KIV]) ** 2 / 14 + cv([m[n] for n in FLANK]) ** 2 / 15) - klen = sum(e - s for _, s, e, n in REGIONS if n in KIV) - flen = sum(e - s for _, s, e, n in REGIONS if n in FLANK) - k = sum(m[n] * (e - s) for _, s, e, n in REGIONS if n in KIV) / klen - f = sum(m[n] * (e - s) for _, s, e, n in REGIONS if n in FLANK) / flen - total = 14 * k / f - want = (f"{total - 2:.1f}", f"{total * math.exp(-1.96 * rel) - 2:.1f}", f"{total * math.exp(1.96 * rel) - 2:.1f}") - got = (r["kiv2_copies"], r["ci95_low"], r["ci95_high"]) - check(f"interval: by hand {want}", got == want, got) - check("interval: it contains the estimate", - float(r["ci95_low"]) < float(r["kiv2_copies"]) < float(r["ci95_high"]), r) - m2 = depths(20, kiv_spread=0.20, flank_spread=0.30) - r2 = assess(m2, q1(m2)) - check("interval: twice the spread is wider", - float(r2["ci95_high"]) - float(r2["ci95_low"]) > float(r["ci95_high"]) - float(r["ci95_low"]), (r, r2)) - # 3. class - for n, sp, cls in ((20, 0.03, "low"), (29, 0.03, "low"), (29, 0.25, "typical"), (38, 0.03, "typical"), - (47, 0.25, "typical"), (60, 0.03, "high")): - m = depths(n, kiv_spread=sp, flank_spread=sp) - r = assess(m, q1(m)) - check(f"class: {n} copies ({r['ci95_low']} to {r['ci95_high']}) is {cls}", r["class"] == cls, r) - r = assess(depths(20), q1(depths(20))) - check("class: the low message says what a low count means", - "a low count goes with small apo(a) isoforms and a higher Lp(a) level" in r["message"], r) - # 4. not assessed - r = assess(depths(20, flank=8.0), q1(depths(20, flank=8.0))) - check("8x over the single-copy exons: too little depth, no count", - r["status"] == "too little depth" and r["kiv2_copies"] == "." and "8.0x" in r["message"], r) - r = assess(depths(20, flank=10.0), q1(depths(20, flank=10.0))) - check("10x: estimated", r["status"] == "estimated", r) - r = assess(depths(20), q1(depths(20), share=0.0)) - check("no read of MAPQ 0 over KIV-2: not assessed, naming the cause", - r["status"] == "not assessed" and r["kiv2_copies"] == "." and "novoalign" in r["message"], r) - r = assess(depths(20), q1(depths(20), share=0.25)) - check("a quarter of the depth from MAPQ 0: estimated", r["status"] == "estimated", r) - r = assess({}, {}) - check("no chr6: not assessed", r["status"] == "not assessed" and "no chr6" in r["message"], r) - r = assess(depths(-1.5), q1(depths(-1.5))) - check("KIV-2 depth under the floor of any count: not assessed, never a negative count", - r["status"] == "not assessed" and r["kiv2_copies"] == "." and "too little for any repeat count" in r["message"], - r) - - # 5. CLI - out = og_cn("bed", "--catalog", CATALOG, "--locus", "LPA_KIV2") - rows = [x.split("\t") for x in out.stdout.splitlines()] - check("bed: 29 regions", out.returncode == 0 and len(rows) == 29, out) - check("bed: 14 KIV-2 exons of 2,394 bp in total", - sum(int(e) - int(s) for c, s, e, n in rows if n.startswith("kiv2_")) == 2394 - and sum(n.startswith("kiv2_") for *_, n in rows) == 14, rows) - check("bed: 15 single-copy exons", sum(n.startswith("flank_") for *_, n in rows) == 15, rows) - with tempfile.TemporaryDirectory() as t: - a, b, o = (os.path.join(t, x) for x in ("a.tsv", "b.tsv", "o.tsv")) - bedcov(a, depths(20)) - bedcov(b, q1(depths(20))) - cli = [OG, "cn", "call", "--catalog", CATALOG, "--locus", "LPA_KIV2", "--bedcov", a, "--bedcov-mapq1", b, - "--table", o] - out = subprocess.run(cli, capture_output=True, text=True) - tab = dict(x.split("\t", 1) for x in open(o).read().splitlines()[1:]) if os.path.exists(o) else {} - check("call: exit 0 and the metric table", out.returncode == 0 and tab.get("kiv2_copies") == "20.0" - and tab.get("status") == "estimated" and "blood" in tab.get("advice", ""), (out, tab)) - open(a, "w").close() - open(b, "w").close() - out = subprocess.run(cli, capture_output=True, text=True) - tab = dict(x.split("\t", 1) for x in open(o).read().splitlines()[1:]) - check("call: two empty tables (no chr6) exit 0, not assessed", - out.returncode == 0 and tab.get("status") == "not assessed", (out, tab)) - bedcov(a, depths(20)) - out = subprocess.run(cli, capture_output=True, text=True) - check("call: an empty MAPQ 1 table beside a full one exits 2", - out.returncode == 2 and "is empty but the other table is not" in out.stderr, out) - bedcov(a, depths(20)) - with open(a, "a") as f: - f.write("chr6\t1\t100\tother\t500\n") - out = subprocess.run(cli, capture_output=True, text=True) - check("call: other regions exit 2", out.returncode == 2 and "differ" in out.stderr, out) - with open(a, "w") as f: - f.write("chr6\t160646213\t160646395\n") - out = subprocess.run(cli, capture_output=True, text=True) - check("call: a short row exits 2", out.returncode == 2 and "expected chrom" in out.stderr, out) - - # 6. the traits table - traits() - import collect_summary - import render_report - - # 7. reports - def sample(t, name, n_copies, flank=30.0, genotypes=None): - d = os.path.join(t, name, "S") - os.makedirs(os.path.join(d, "lpa")) - a, b = os.path.join(t, name, "a.tsv"), os.path.join(t, name, "b.tsv") - m = depths(n_copies, flank=flank, kiv_spread=0.03, flank_spread=0.03) - bedcov(a, m) - bedcov(b, q1(m)) - estimate(a, b, os.path.join(d, "lpa", "S_lpa_kiv2.tsv")) - if genotypes: - os.makedirs(os.path.join(d, "traits")) - with open(os.path.join(d, "traits", "S_traits.tsv"), "w") as f: - f.write("\t".join(TRAITS_COLUMNS) + "\n") - for rs, gt in genotypes.items(): - row = {k: "." for k in TRAITS_COLUMNS} - row.update(rsid=rs, gene="LPA", genotype=gt, status="called" if gt != "." else "not_covered") - f.write("\t".join(row[k] for k in TRAITS_COLUMNS) + "\n") - summ = collect_summary.collect("S", d) - return summ, render_report.text_report(summ), render_report.html_report(summ) - - with tempfile.TemporaryDirectory() as t: - summ, txt, html = sample(t, "low", 20, genotypes={R1: "A/G", R2: "T/T"}) - sec = summ["sections"].get("lpa", {}) - check("report: the section is read", sec.get("state") not in (None, "missing", "unreadable"), sec) - check("report: a low count with one risk allele is flagged with both reasons", - sec.get("data", {}).get("flag") == "high" - and "1 risk allele at rs10455872 and rs3798220 and a low KIV-2 repeat count" - in sec.get("data", {}).get("flag_text", ""), sec) - check("report: the text section has the count, both variants and the advice", - "## Lp(a): LPA KIV-2 Repeat Count and Risk Variants" in txt and "KIV-2 repeats (both chromosomes): 20.0" in txt - and f"{R1}: A/G (1 risk allele)" in txt and f"{R2}: T/T (0 risk alleles)" in txt - and ADVICE in txt, txt) - check("report: the HTML card has the flag and the advice", - "

    Lp(a): LPA KIV-2 Repeat Count

    " in html and "Genetic sign of high Lp(a)" in html - and render_report.E(ADVICE) in html, html[-3000:]) - summ, txt, html = sample(t, "typ", 38, genotypes={R1: "A/A", R2: "T|T"}) - check("report: a typical count with no risk allele is not flagged", - summ["sections"]["lpa"]["data"]["flag"] == "." and "No genetic sign of a high Lp(a)" in txt - and "Genetic sign of high Lp(a)" not in html, txt) - summ, txt, html = sample(t, "nov", 38) - check("report: without step 42 the variants are not read", - "rs10455872 and rs3798220: not read (step 42, the traits panel, did not run)" in txt, txt) - summ, txt, html = sample(t, "thin", 38, flank=3.0, genotypes={R1: ".", R2: "C/C"}) - check("report: too little depth is said, and a not-covered variant is not read", - "KIV-2 repeats: too little depth" in txt and f"{R1}: . (not read)" in txt - and f"{R2}: C/C (2 risk alleles)" in txt, txt) - summ, txt, html = sample(t, "thin0", 38, flank=3.0) - check("report: no count and no variant read is not determined, never a negative result", - summ["sections"]["lpa"]["data"]["flag"] == "." and "Not determined" in txt - and "No genetic sign" not in txt, txt) - # KIV-2 4925G>A beside the count, in both reports and on the live page - import live_results - cav = render_report.LPA_NULL_CAVEAT - count = "KIV-2 repeats (both chromosomes): 38.0" - summ, txt, html = sample(t, "nul0", 38) - na = summ["sections"]["lpa"]["data"].get("null_allele") or {} - check("report: without step 46's 4925G>A table the null allele says not assessed, beside the count, with why", - na.get("status") == "not assessed" and count in txt - and "KIV-2 4925G>A null allele (rs1484919544): not assessed\n" in txt.split(count, 1)[-1].split(R1, 1)[0] - and "step 46 wrote no S_lpa_kiv2_4925.tsv" in txt and cav in txt, txt) - d = os.path.join(t, "nul0", "S", "lpa") - pileup(os.path.join(t, "nul0.pileup"), (95, 5)) - og_cn("marker-read", "LPA_KIV2_4925", "--pileup", os.path.join(t, "nul0.pileup"), "--copy-table", - os.path.join(d, "S_lpa_kiv2.tsv"), "--out", os.path.join(d, "S_lpa_kiv2_4925.tsv")) - summ = collect_summary.collect("S", os.path.join(t, "nul0", "S")) - txt, html = render_report.text_report(summ), render_report.html_report(summ) - beside = txt.split(count, 1)[-1].split(R1, 1)[0] - check("report: a 4925G>A table with 35 of 700 reads prints present, the reads and the copies, the " - "reader's message and the caveat, between the count and the risk variants", - "KIV-2 4925G>A null allele (rs1484919544): present, 35 of 700 reads carry it, about 2.0 copies\n" - in beside and "cannot say which chromosome or which repeat" in beside and cav in beside, beside) - card = html.split("

    Lp(a): LPA KIV-2 Repeat Count

    ", 1)[-1].split("

    ", 1)[0] - check("report: the HTML card shows the null allele and the caveat", - render_report.E("KIV-2 4925G>A null allele (rs1484919544): present, 35 of 700 reads carry it") - in card and render_report.E(cav) in card, card[:3000]) - live = live_results.a_lpa(summ["sections"], {}) - check("live: the Lp(a) card has the null allele as a fact and its lines with the caveat", - ["KIV-2 4925G>A null allele", "present, 35 of 700 reads carry it, about 2.0 copies"] in live["facts"] - and any(cav in x for x in live["lines"]) and any(x.startswith("KIV-2 4925G>A null allele") for x in live["lines"]), - live) - with open(os.path.join(d, "S_lpa_kiv2_4925.tsv"), "w") as f: - f.write("metric\tvalue\nmessage\tno status\n") - txt = render_report.text_report(collect_summary.collect("S", os.path.join(t, "nul0", "S"))) - check("report: a 4925G>A table with no status row reads as not assessed, never as a result", - "KIV-2 4925G>A null allele (rs1484919544): not assessed\n" in txt - and "S_lpa_kiv2_4925.tsv has no status row" in txt, txt) - os.makedirs(os.path.join(t, "none", "S")) - nothing = collect_summary.collect("S", os.path.join(t, "none", "S")) - check("report: no card and no text section without step 46", - "

    Lp(a)" not in render_report.html_report(nothing) and "## Lp(a)" not in render_report.text_report(nothing)) - - # 8. KIV-2 4925G>A beside the count - - def table_after(t, name, per_site, n_copies=38, extra=""): - """Step 46's table, a pileup, then null-allele; returns (rc, the 4925G>A table, LPA table, pileup, out).""" - d = os.path.join(t, name) - os.makedirs(d) - a, b, o, p, n = (os.path.join(d, x) for x in ("a.tsv", "b.tsv", "S_lpa_kiv2.tsv", "p.tsv", "S_lpa_kiv2_4925.tsv")) - bedcov(a, depths(n_copies)) - bedcov(b, q1(depths(n_copies))) - estimate(a, b, o) - pileup(p, per_site, extra) - rc = og_cn("marker-read", "LPA_KIV2_4925", "--pileup", p, "--copy-table", o, "--out", n) - tab = dict(x.split("\t", 1) for x in open(n).read().splitlines()[1:]) if os.path.exists(n) else {} - return rc, tab, o, p, n - - with tempfile.TemporaryDirectory() as t: - rc, tab, o, p, n = table_after(t, "present", (95, 5)) - check("4925G>A: 5% of the reads with T at the seven copies is present, exit 0", - rc.returncode == 0 and tab.get("status") == "present" - and tab.get("alt_reads") == "35" and tab.get("depth") == "700", (rc, tab)) - check("4925G>A: copies = fraction x (N + 2) = 0.05 x 40", - tab.get("fraction") == "0.0500" and tab.get("copies") == "2.0" and tab.get("kiv2_copies") == "38.0", tab) - check("4925G>A: the message says what a short-read fraction cannot say", - "cannot say which chromosome or which repeat" in tab.get("message", ""), tab) - check("4925G>A: the first rows name the marker and its dbSNP record", - open(n).read().splitlines()[:3] == ["metric\tvalue", "marker\tLPA KIV-2 4925G>A", - "rsid\trs1484919544 (first KIV-2 copy)"], open(n).read()) - d0 = os.path.join(t, "plain") - os.makedirs(d0) - a0, b0, o0 = (os.path.join(d0, x) for x in ("a.tsv", "b.tsv", "S_lpa_kiv2.tsv")) - bedcov(a0, depths(38)) - bedcov(b0, q1(depths(38))) - estimate(a0, b0, o0) - check("4925G>A: step 46's table is left byte for byte", open(o).read() == open(o0).read(), open(o).read()) - n3, o3 = os.path.join(t, "viacall.tsv"), os.path.join(t, "viacall_lpa.tsv") - res = og_cn("call", "--catalog", CATALOG, "--locus", "LPA_KIV2", "--bedcov", a0, "--bedcov-mapq1", b0, - "--table", o3, "--marker", "LPA_KIV2_4925", "--marker-pileup", p, "--marker-out", n3) - check("4925G>A: og cn call --marker LPA_KIV2_4925 (what step 46 and the LPA process run) writes the same " - "count table and the same 4925G>A table as marker-read on that table", - res.returncode == 0 and open(o3).read() == open(o0).read() and open(n3).read() == open(n).read() - and res.stdout.splitlines()[-1].startswith("LPA LPA KIV-2 4925G>A"), res) - res = og_cn("marker-read", "LPA_KIV2_4925", "--pileup", p, "--copy-table", os.path.join(t, "none.tsv"), - "--out", n3) - tab = dict(x.split("\t", 1) for x in open(n3).read().splitlines()[1:]) - check("4925G>A: without step 46's table the reading stands and the copies are '.'", - res.returncode == 0 and tab.get("status") == "present" and tab.get("copies") == "." - and tab.get("kiv2_copies") == ".", (res, tab)) - for share, want in (("0.050", "not assessed"), ("0.900", "present")): - lt = os.path.join(t, f"share{share}.tsv") - with open(lt, "w") as f: - f.write(f"metric\tvalue\nstatus\tnot assessed\nkiv2_copies\t.\nmapq0_share\t{share}\n") - res = og_cn("marker-read", "LPA_KIV2_4925", "--pileup", p, "--copy-table", lt, "--out", n3) - tab = dict(x.split("\t", 1) for x in open(n3).read().splitlines()[1:]) - check(f"4925G>A: a MAPQ 0 share of {share} in step 46's table gives {want} on the same reads", - res.returncode == 0 and tab.get("status") == want - and (want == "present" or "left out the reads that fit several copies" in tab.get("message", "")), - (res, tab)) - rc, tab, *_ = table_after(t, "ref", (100, 0)) - check("4925G>A: reads with the reference base only is not seen", - tab.get("status") == "not seen" and tab.get("copies") == "." - and "not proof of absence" in tab.get("message", ""), tab) - rc, tab, *_ = table_after(t, "unc", (100, 0), extra="chr6\t160640667\tN\t3\tTTT\tIII\n") - check("4925G>A: 3 reads with T is uncertain", tab.get("status") == "uncertain" - and tab.get("alt_reads") == "3", tab) - rc, tab, *_ = table_after(t, "thin", (10, 1)) - check("4925G>A: under 100 reads is too little depth", tab.get("status") == "too little depth", tab) - rc, tab, *_ = table_after(t, "other", (100, 0), - extra="chr6\t160640668\tN\t50\t" + "T" * 50 + "\t.\nchr7\t160640667\tN\t50\t" - + "T" * 50 + "\t.\nchr6\t160635123\tN\t2\tC+2TTC-1T\tII\n") - check("4925G>A: reads at other positions and indel bases are not counted", - tab.get("status") == "not seen" and tab.get("alt_reads") == "0" and tab.get("depth") == "702", tab) - rc, tab, o, p, n = table_after(t, "bad", (95, 5)) - with open(p, "w") as f: - f.write("not a pileup\n") - rc = og_cn("marker-read", "LPA_KIV2_4925", "--pileup", p, "--copy-table", o, "--out", n) - tab = dict(x.split("\t", 1) for x in open(n).read().splitlines()[1:]) - check("4925G>A: an unreadable pileup is not assessed, exit 0", - rc.returncode == 0 and tab.get("status") == "not assessed", (rc, tab)) - rc = og_cn("marker-read", "LPA_KIV2_4925", "--pileup", p, "--copy-table", o, - "--out", os.path.join(t, "no", "such", "dir.tsv")) - check("4925G>A: an output that cannot be written exits 2", rc.returncode == 2, rc) - - # The demo's step 46 table (tests/demo/make_demo_sample.py) is what og writes. - sys.path.insert(0, os.path.join(REPO, "tests", "demo")) - import make_demo_sample as demo - m, q = demo.lpa_depth(ROW) - with tempfile.TemporaryDirectory() as t: - a, b, o = (os.path.join(t, x) for x in ("a.tsv", "b.tsv", "o.tsv")) - bedcov(a, m, exact=True) - bedcov(b, q, exact=True) - estimate(a, b, o) - got = open(o).read() if os.path.exists(o) else "" - check("the demo's step 46 table is the one og cn call writes for its depths", got == demo.LPA_TABLE, got) - - print("\nRESULT:", "ALL PASS" if FAILS == 0 else f"{FAILS} FAILED") - return 1 if FAILS else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/test_mito_extended.py b/tests/test_mito_extended.py deleted file mode 100644 index 47304a04a..000000000 --- a/tests/test_mito_extended.py +++ /dev/null @@ -1,391 +0,0 @@ -#!/usr/bin/env python3 -"""Step 20's confirmed mtDNA variant list, large deletions (eKLIPse) and mtDNA -copy number, on synthetic inputs. - -Checks: - 1. assets/mito_confirmed.tsv: its header, and every row with a ClinVar id - or a PubMed id (a planted row with neither fails the schema check); the - first rows are there (m.3243A>G, m.8344A>G, the three LHON variants and - the common 4977 bp deletion). - 2. a chrM VCF with m.3243A>G at 12% is reported with its level, in the - summary and in both reports; the same VCF without it reports none. - 3. the copy number: idxstats with chrM at 100 times the autosomal depth - gives 200 copies per cell (+-5%); a sliced BAM and a BAM with no chrM - reads are not assessed; with no copy-number table both reports say why. - 4. eKLIPse's deletions CSV (decimal commas, quoted fields) is read; the - common deletion is named within 50 bp of its breakpoints, a deletion - elsewhere is not, one under 1% is counted but not listed, and a table - with no deletion lists none. - 5. EKLIPSE_IMAGE is pinned by digest, and its versions.env comment names - the upstream commit 3606cb2edac983d2623ddc667b49206c3d01373c (the e2e - case checks the Bioconda build's recipe against it). - -Run: python3 tests/test_mito_extended.py -Also: python3 tests/test_mito_extended.py synth-sam ... writes the synthetic -chrM reads tests/e2e/mito-extended-1-deletions.sh aligns nothing for: SAM -records with the soft clips an aligner writes at a deletion's junction. -""" -import argparse -import csv -import gzip -import os -import random -import re -import shutil -import subprocess -import sys -import tempfile - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -sys.path.insert(0, os.path.join(REPO, "tests")) -from og_binary import og_binary # noqa: E402 - -EKLIPSE_COMMIT = "3606cb2edac983d2623ddc667b49206c3d01373c" -FAILS = 0 - - -def og(*args): - """og's exit code: og mito deletions and og mito copies write step 20's tables.""" - return subprocess.run([og_binary(), *args], capture_output=True).returncode - - -def table(path): - """The rows of a table og wrote, as dicts.""" - with open(path) as f: - return list(csv.DictReader(f, delimiter="\t")) - - -def check(name, ok, detail=""): - global FAILS - print(f"[{'PASS' if ok else 'FAIL'}] {name}" + (f" ({detail})" if detail and not ok else "")) - if not ok: - FAILS += 1 - - -# --- the synthetic reads (synth-sam) ------------------------------------------------ - -def read_fasta(path): - with open(path) as f: - return "".join(ln.strip() for ln in f if not ln.startswith(">")).upper() - - -def synth_reads(seq, depth, deletion, level, read_len=150, seed=1): - """[(pos 1-based, cigar, flag, read sequence)] of single-end reads at the - given depth from molecules of which `level` carry the deletion (first and - last deleted base, 1-based), or none when deletion is None. A read across - the junction is written as an aligner writes it: aligned on its longer - side, the rest soft-clipped (a forward read clipped on the right, a - reverse read on the left, as eKLIPse reads them); a junction read with a - side under 30 bases is left out.""" - rnd = random.Random(seed) - n = int(len(seq) * depth / read_len) - out = [] - if deletion: - a, b = deletion - mol = seq[:a - 1] + seq[b:] - junction = a - 1 # first base of mol after the junction (0-based) - for i in range(n): - if deletion and rnd.random() < level: - s = rnd.randrange(0, len(mol) - read_len) - left = junction - s - r = mol[s:s + read_len] - if left <= 0: - out.append((s + 1 + (b - a + 1), f"{read_len}M", rnd.choice((0, 16)), r)) - elif left >= read_len: - out.append((s + 1, f"{read_len}M", rnd.choice((0, 16)), r)) - elif min(left, read_len - left) < 30: - continue - elif left >= read_len - left: - out.append((s + 1, f"{left}M{read_len - left}S", 0, r)) - else: - out.append((b + 1, f"{left}S{read_len - left}M", 16, r)) - else: - s = rnd.randrange(0, len(seq) - read_len) - out.append((s + 1, f"{read_len}M", rnd.choice((0, 16)), seq[s:s + read_len])) - return sorted(out) - - -def synth_sam_main(argv): - ap = argparse.ArgumentParser(prog="test_mito_extended.py synth-sam") - ap.add_argument("--fasta", required=True, help="chrM alone (samtools faidx REF chrM)") - ap.add_argument("--header", required=True, help="the @HD/@SQ lines the reads go under") - ap.add_argument("--sample", required=True) - ap.add_argument("--depth", type=float, default=400) - ap.add_argument("--deletion", help="FIRST-LAST deleted base, e.g. 8470-13446") - ap.add_argument("--level", type=float, default=0.1) - ap.add_argument("--out", required=True) - a = ap.parse_args(argv) - seq = read_fasta(a.fasta) - dele = tuple(int(x) for x in a.deletion.split("-")) if a.deletion else None - reads = synth_reads(seq, a.depth, dele, a.level) - with open(a.header) as f, open(a.out, "w") as o: - o.write("".join(ln for ln in f if ln.startswith(("@HD", "@SQ")))) - o.write(f"@RG\tID:{a.sample}\tSM:{a.sample}\tPL:ILLUMINA\tLB:{a.sample}\n") - for i, (pos, cigar, flag, r) in enumerate(reads): - o.write(f"r{i}\t{flag}\tchrM\t{pos}\t60\t{cigar}\t*\t0\t0\t{r}\t{'I' * len(r)}\tRG:Z:{a.sample}\n") - clipped = sum(1 for _, c, _, _ in reads if "S" in c) - print(f"{len(reads)} reads, {clipped} soft-clipped at the junction") - return 0 - - -# --- the checks ------------------------------------------------------------------- - -CONFIRMED_COLS = ["kind", "pos", "end", "ref", "alt", "hgvs", "gene", "disease", "clinvar_id", "pmids", - "heteroplasmy_note"] - - -def confirmed_errors(path): - """What is wrong with a mito_confirmed.tsv: a wrong header, a row with no - ClinVar id and no PubMed id, a malformed id, an unknown kind.""" - errs = [] - with open(path) as f: - rows = list(csv.DictReader((ln for ln in f if not ln.startswith("#")), delimiter="\t", - quoting=csv.QUOTE_NONE)) - if not rows or list(rows[0].keys()) != CONFIRMED_COLS: - return [f"header is not {' '.join(CONFIRMED_COLS)}"] - for r in rows: - name = r.get("hgvs") or r.get("pos") - if None in r or None in r.values(): - errs.append(f"{name}: wrong number of columns") - continue - cv = r["clinvar_id"] not in ("", ".") - pm = r["pmids"] not in ("", ".") - if not cv and not pm: - errs.append(f"{name}: neither a ClinVar id nor a PubMed id") - if cv and not r["clinvar_id"].isdigit(): - errs.append(f"{name}: ClinVar id {r['clinvar_id']!r}") - if pm and not all(x.isdigit() for x in r["pmids"].split(";")): - errs.append(f"{name}: PubMed ids {r['pmids']!r}") - if r["kind"] not in ("snv", "deletion"): - errs.append(f"{name}: kind {r['kind']!r}") - elif not (r["pos"].isdigit() and r["end"].isdigit() and 1 <= int(r["pos"]) <= int(r["end"]) <= 16569): - errs.append(f"{name}: positions {r['pos']}-{r['end']}") - return errs - - -def test_confirmed_schema(tmp): - path = os.path.join(REPO, "assets", "mito_confirmed.tsv") - errs = confirmed_errors(path) - check("assets/mito_confirmed.tsv: every row has a ClinVar or PubMed id and a valid shape", not errs, "; ".join(errs)) - with open(path) as f: - text = f.read() - for want in ("m.3243A>G", "m.8344A>G", "m.11778G>A", "m.3460G>A", "m.14484T>C"): - check(f"the list has {want}", f"\t{want}\t" in text) - check("the list has the common 4977 bp deletion (m.8470 to m.13446)", - re.search(r"^deletion\t8470\t13446\t", text, re.M) is not None) - bad = os.path.join(tmp, "bad.tsv") - with open(bad, "w") as f: - f.write(text + "snv\t3271\t3271\tT\tC\tm.3271T>C\tMT-TL1\tMELAS\t.\t.\tno id at all\n") - errs = confirmed_errors(bad) - check("red control: a planted row with neither id fails the schema check", - any("neither a ClinVar id nor a PubMed id" in e for e in errs), "; ".join(errs) or "no error") - - -def write_vcf(path, records): - with gzip.open(path, "wt") as f: - f.write("##fileformat=VCFv4.2\n##contig=\n") - f.write("#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n") - for pos, ref, alt, af in records: - f.write(f"chrM\t{pos}\t.\t{ref}\t{alt}\t.\tPASS\t.\tGT:AF\t0/1:{af}\n") - - -def sample_dir(tmp, name, records, deletions=None, idxstats=None): - d = os.path.join(tmp, name) - os.makedirs(os.path.join(d, "mito")) - write_vcf(os.path.join(d, "mito", f"{name}_chrM_filtered.vcf.gz"), records) - if deletions is not None: - csv_path = os.path.join(tmp, f"{name}_eklipse.csv") - with open(csv_path, "w") as f: - f.write("\"Title\";\"5' breakpoint\";\"3' breakpoint\";\"Freq\";\"Freq For\";\"Freq Rev\";\"5' Blast\";" - "\"3' Blast\";\"5' Depth\";\"3' Depth\";\"Repetition\"\n") - for b5, b3, lvl in deletions: - v = str(lvl).replace(".", ",") - f.write(f'"{name}";"{b5}";"{b3}";"{v}";"{v}";"{v}";"12";"14";"390.0";"402.0";"none"\n') - rc = og("mito", "deletions", "--eklipse", csv_path, "--out", os.path.join(d, "mito", f"{name}_chrM_deletions.tsv")) - check(f"{name}: mito-deletions reads eKLIPse's CSV", rc == 0) - if idxstats is not None: - ip = os.path.join(tmp, f"{name}.idxstats") - with open(ip, "w") as f: - f.write(idxstats) - rc = og("mito", "copies", "--idxstats", ip, "--out", os.path.join(d, "mito", f"{name}_mtdna_copies.tsv")) - check(f"{name}: mito-copies reads the idxstats", rc == 0) - return d - - -def mito_section(d, name): - import collect_summary - return collect_summary.collect(name, d)["sections"]["mito"]["data"] - - -def reports(d, name): - import collect_summary - import render_report - s = collect_summary.collect(name, d) - return render_report.text_report(s), render_report.html_report(s) - - -def test_confirmed_snv(tmp): - d = sample_dir(tmp, "S3243", [(73, "A", "G", "1.00"), (3243, "A", "G", "0.12"), (16519, "T", "C", "0.998")]) - data = mito_section(d, "S3243") - conf = data.get("confirmed") or [] - check("m.3243A>G at 12% is in the summary's confirmed list", [v.get("hgvs") for v in conf] == ["m.3243A>G"], - str(conf)) - check("its level is 0.12", bool(conf) and conf[0].get("af") == "0.12", str(conf)) - text, html = reports(d, "S3243") - check("the text report names m.3243A>G with AF 0.12 and MELAS", - re.search(r"m\.3243A>G \(MT-TL1\) AF 0\.12 .*MELAS", text) is not None) - check("the HTML report counts one confirmed variant and names it", - re.search(r'Confirmed disease variants1<', html) - is not None and "m.3243A>G (MT-TL1) AF 0.12" in html) - d = sample_dir(tmp, "Snone", [(73, "A", "G", "1.00"), (16519, "T", "C", "0.998")]) - data = mito_section(d, "Snone") - check("control: without m.3243A>G the confirmed list is empty", data.get("confirmed") == [], str(data.get("confirmed"))) - text, _ = reports(d, "Snone") - check("control: the text report counts no confirmed variant", - "Confirmed disease variants (assets/mito_confirmed.tsv): 0" in text) - import collect_summary - saved, collect_summary.ASSETS_DIR = collect_summary.ASSETS_DIR, os.path.join(tmp, "no-assets") - try: - data = mito_section(d, "Snone") - text, html = reports(d, "Snone") - finally: - collect_summary.ASSETS_DIR = saved - check("without the list the summary has no confirmed count, only why", - "confirmed" not in data and "mito_confirmed.tsv is not in" in (data.get("confirmed_note") or ""), str(data)) - check("without the list neither report shows a count of confirmed variants", - "Confirmed disease variants (assets" not in text and "Confirmed disease variants" not in html - and "Confirmed disease variants: not checked" in text) - - -def idxstats(chrm_reads, auto_reads_per_chrom, chroms=22, length=50_000_000): - lines = [f"chr{i}\t{length}\t{auto_reads_per_chrom}\t0" for i in range(1, chroms + 1)] - lines += [f"chrX\t156040895\t{auto_reads_per_chrom}\t0", f"chrM\t16569\t{chrm_reads}\t0", "*\t0\t0\t500"] - return "\n".join(lines) + "\n" - - -def mito_copies(tmp, text): - """og mito copies of idxstats TEXT, as {metric: value} ('.' read as None).""" - ip, out = os.path.join(tmp, "cn.idxstats"), os.path.join(tmp, "cn.tsv") - with open(ip, "w") as f: - f.write(text) - if og("mito", "copies", "--idxstats", ip, "--out", out): - return {"status": "og failed", "reason": None, "mtdna_copies": None} - return {r["metric"]: None if r["value"] == "." else r["value"] for r in table(out)} - - -def test_copies(tmp): - import collect_summary - # 10 reads per kb of autosome would be under the floor; 200 per kb is about 30x. - per_chrom = 200 * 50_000 - chrm = round(100 * 16569 * per_chrom / 50_000_000) - b = mito_copies(tmp, idxstats(chrm, per_chrom)) - got = float(b["mtdna_copies"] or 0) - check(f"chrM at 100x the autosomal depth reads 200 copies per cell (+-5%): {got}", - b["status"] == "called" and abs(got - 200) <= 10, str(b)) - b = mito_copies(tmp, idxstats(chrm // 2, per_chrom)) - check("half the chrM reads reads half the copies", abs(float(b["mtdna_copies"] or 0) - 100) <= 5, str(b)) - b = mito_copies(tmp, idxstats(chrm, 500)) - check("a sliced BAM (few autosomal reads) is not assessed", - b["status"] == "not assessed" and b["reason"] == "low_autosomal_depth", str(b)) - b = mito_copies(tmp, idxstats(0, per_chrom)) - check("a BAM with no chrM reads is not assessed", b["reason"] == "no_chrM_reads", str(b)) - b = mito_copies(tmp, idxstats(chrm, per_chrom).replace("chrM", "MT").replace("chr", "")) - check("contigs named 1..22 and MT work", b["status"] == "called", str(b)) - d = sample_dir(tmp, "Snocn", [(73, "A", "G", "1.00")]) - data = mito_section(d, "Snocn") - text, html = reports(d, "Snocn") - check("without a copy-number table the summary says why", - "copies" not in data and "mtdna_copies.tsv" in (data.get("copies_note") or ""), str(data)) - check("without a copy-number table both reports say it was not counted", - "mtDNA copy number: not counted (" in text and "mtDNA copy number: not counted (" in html) - d = sample_dir(tmp, "Scn", [(73, "A", "G", "1.00")], idxstats=idxstats(chrm, per_chrom)) - text, html = reports(d, "Scn") - check("the text report gives the copy number", - re.search(r"mtDNA copy number: 20[0-9]\.[0-9] per cell", text) is not None) - check("the HTML report gives the copy number", - re.search(r'mtDNA copies per cell20[0-9]\.[0-9]<', html) is not None) - # The mito card comes before the ClinVar hits table in html_report: a copy - # number there once hid the table, so the same summary with a ClinVar hit - # must still show its row. - import render_report - s = collect_summary.collect("Scn", d) - hit = {"chrom": "chr20", "pos": 10243238, "ref": "A", "alt": "G", "genotype": "hom", "gene": "SNAP25", - "significance": "Pathogenic", "review_status": "criteria provided, single submitter", "stars": 1} - s["sections"]["clinvar"] = dict(s["sections"]["clinvar"], state="ok", - data={"count": 1, "by_stars": {"1": 1}, "hits": [hit]}) - html = render_report.html_report(s) - check("with a copy number the HTML report still lists the ClinVar hit", - re.search(r"chr2010243238.*SNAP25Pathogenic", html) is not None) - - -def test_deletions(tmp): - d = sample_dir(tmp, "Sdel", [(73, "A", "G", "1.00")], - deletions=[(8469, 13447, 10.2), (8482, 13460, 2.5), (9000, 14000, 3.1), (5000, 7000, 0.4)]) - rows = [{k: int(v) if k == "length" else v for k, v in r.items()} - for r in table(os.path.join(d, "mito", "Sdel_chrM_deletions.tsv"))] - check("eKLIPse's decimal commas are read", [float(r["level_pct"]) for r in rows] == [10.2, 2.5, 3.1, 0.4], - str([r["level_pct"] for r in rows])) - check("the deletion's length is the bases between the breakpoints", rows[0]["length"] == 4977, str(rows[0])) - dl = mito_section(d, "Sdel").get("deletions") or {} - listed = dl.get("listed") or [] - named = [(r["bkp5"], r["bkp3"]) for r in listed if r.get("known")] - check("the common deletion is named at both of its writings across the 13 bp repeat", - named == [(8469, 13447), (8482, 13460)], str(listed)) - check("a deletion elsewhere (m.9001-13999) is listed but not named", - any(r["bkp5"] == 9000 and not r.get("known") for r in listed), str(listed)) - check("a deletion under 1% is counted, not listed", dl.get("total") == 4 and len(listed) == 3, str(dl)) - text, html = reports(d, "Sdel") - check("the HTML report counts four deletions, two of them named", - '>Large deletions (eKLIPse)4<' in html - and re.search(r'Named in the confirmed list2<', html) - is not None) - check("the text report names the common deletion with its level and ClinVar id", - re.search(r"m\.8470_13446del \(4977 bp\) level 10\.2% The common 4977 bp deletion.*ClinVar 430680", text) - is not None) - check("the text report says blood often lacks the deletion", "often found in muscle and not in blood" in text) - d = sample_dir(tmp, "Snodel", [(73, "A", "G", "1.00")], deletions=[]) - dl = mito_section(d, "Snodel").get("deletions") or {} - check("red control: a table with no deletion lists none", dl.get("total") == 0 and dl.get("listed") == [], str(dl)) - bad = os.path.join(tmp, "bad.csv") - with open(bad, "w") as f: - f.write("pos\tref\n") - check("a file that is not eKLIPse's table is refused", - og("mito", "deletions", "--eklipse", bad, "--out", os.path.join(tmp, "bad.tsv")) == 1) - wrap = os.path.join(tmp, "wrap.csv") - with open(wrap, "w") as f: - f.write("\"Title\";\"5' breakpoint\";\"3' breakpoint\";\"Freq\";\"Freq For\";\"Freq Rev\";\"5' Blast\";" - "\"3' Blast\";\"5' Depth\";\"3' Depth\";\"Repetition\"\n") - f.write('"S";"16000";"500";"2,0";"2,0";"2,0";"3";"3";"100";"100";"none"\n') - og("mito", "deletions", "--eklipse", wrap, "--out", os.path.join(tmp, "wrap.tsv")) - rows = [{k: int(v) if k == "length" else v for k, v in r.items()} for r in table(os.path.join(tmp, "wrap.tsv"))] - check("a deletion across the origin (m.16001 to m.499) gets its length, 1068 bp", - rows[0]["length"] == 1068, str(rows[0])) - - -def test_image_pin(): - with open(os.path.join(REPO, "versions.env")) as f: - line = next((ln for ln in f if ln.startswith("EKLIPSE_IMAGE=")), "") - check("EKLIPSE_IMAGE is pinned by digest", re.match(r'EKLIPSE_IMAGE="[^"@]+:[^"@]+@sha256:[0-9a-f]{64}"', line) - is not None, line.strip()) - check(f"its comment names the upstream commit {EKLIPSE_COMMIT[:7]}", EKLIPSE_COMMIT in line, line.strip()) - - -def main(argv): - if argv[:1] == ["synth-sam"]: - return synth_sam_main(argv[1:]) - tmp = tempfile.mkdtemp() - try: - test_confirmed_schema(tmp) - test_confirmed_snv(tmp) - test_copies(tmp) - test_deletions(tmp) - test_image_pin() - finally: - shutil.rmtree(tmp) - print(f"{FAILS} failed" if FAILS else "all passed") - return 1 if FAILS else 0 - - -if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) diff --git a/tests/test_render_report.py b/tests/test_render_report.py index 3035fb7a8..02079ed1f 100644 --- a/tests/test_render_report.py +++ b/tests/test_render_report.py @@ -73,10 +73,7 @@ the card is absent when step 41 did not run; and neither the reports nor og bloodtype say anything about transfusion or donation (the table is information). - 11. the traits card (step 42): both reports carry the not-a-diagnosis - note; a traits table older than the latest run-all.sh run, whose step - 42 was skipped in it, is shown as stale in both reports, and one newer - than the run is not. + 11. the traits card (step 42): rust/og/tests/genome_traits_report.rs. 12. the raw-data export (step 40): both reports show how many chip sites got a genotype, the '--' reasons, the two files and the privacy note; without the step's table there is no card and no text section. @@ -859,6 +856,7 @@ def mosaic_table(rows): and "PMS22" in card_p, card_p) # 12. raw-data export card d = f"{work}/consumer/S" + os.makedirs(d) summ = collect_summary.collect("S", d) txt, html = render_report.text_report(summ), render_report.html_report(summ) check("raw-data export: no card and no text section without the step's table", @@ -903,33 +901,7 @@ def mosaic_table(rows): check("blood groups: nothing about transfusion or donation", not said, said) check("control: the wording check finds the word", bool(words.search("for a transfusion"))) - # 11. traits card - import time - d = f"{work}/traits/S" - put(f"{d}/traits/S_traits.tsv", "") - from test_traits_report import traits_tables - g = os.path.join(work, "traits_empty.tsv") - put(g, "") - traits_tables(g, f"{d}/traits/S_traits.tsv", os.path.join(work, "traits_rules.tsv")) - summ = collect_summary.collect("S", d) - txt, html = render_report.text_report(summ), render_report.html_report(summ) - check("traits: both reports say a reading is not a diagnosis", - render_report.TRAITS_NOTE in txt and render_report.TRAITS_NOTE in html, txt) - check("traits: with no reads anywhere every row is not read, and the APOE line too", - summ["sections"]["traits"]["data"]["called"] == 0 - and summ["sections"]["traits"]["data"]["apoe"][-1]["status"] == "not_covered") - old = time.time() - 86400 - os.utime(f"{d}/traits/S_traits.tsv", (old, old)) - put(f"{d}/logs/run_status.tsv", f"meta\tstarted_epoch\t{time.time() - 3600}\nstep\t42\tskipped (no gVCF: S/vcf/S.g.vcf.gz)\n") - summ = collect_summary.collect("S", d) - txt, html = render_report.text_report(summ), render_report.html_report(summ) - card_tr = html.split("

    Traits

    ", 1)[-1].split("

    ", 1)[0] - check("traits: older than a run that skipped step 42 -> stale in both reports", - summ["sections"]["traits"]["state"] == "stale" and "[STALE: from an earlier run" in txt - and 'class="stale">STALE: from an earlier run' in card_tr, card_tr) - os.utime(f"{d}/traits/S_traits.tsv", None) - check("traits: newer than the run -> not stale", - collect_summary.collect("S", d)["sections"]["traits"]["state"] == "ok") + # 11. traits card: rust/og/tests/genome_traits_report.rs # 13. couple card (step 39) with open(os.path.join(REPO, "assets", "carrier_genes.tsv")) as f: diff --git a/tests/test_str_ranges.py b/tests/test_str_ranges.py deleted file mode 100644 index 8c316605d..000000000 --- a/tests/test_str_ranges.py +++ /dev/null @@ -1,291 +0,0 @@ -#!/usr/bin/env python3 -# og-leg: OG_REQUIRED -"""The repeat ranges: the generated str_ranges.tsv, the report's cited subset and the docs. - -1. str_ranges.tsv, which og str catalog writes beside the STRchive - catalog (built here with og from the pinned files, as - rust/crates/og-setup/tests/strchive_catalog.rs builds it): - one row per ExpansionHunter locus of the catalog (82), each traced to its - STRchive id; normal_max and pathogenic_min are the catalog's NormalMax and - PathologicMin, empty where the catalog has no thresholds and the reason in - no_thresholds; intermediate_min and intermediate_max are STRchive's own. - Controls: a table with one row dropped, or one row twice, is caught. -2. bin/str_intermediate_ranges.tsv, the GeneReviews ranges the report names - (HTT, FMR1, DMPK): the columns og summary collect reads, one row per - locus, whole numbers with min <= max ending right below the next class, a - label and a GeneReviews citation; docs/results/structural-and-repeats.md - prints the same range in the locus's row; every locus has a row in - str_ranges.tsv, and the range lies above its normal_max and below its - pathogenic_min, so a count the report calls intermediate is one Stranger - calls pre_mutation - (control: a planted HTT range of 20-35 is caught). That og reads every - row is og-summary's own test (collect::group_c::tests). -3. The "Key Disease Thresholds" table of docs/09-str-expansions.md: each - row's normal maximum, pre-mutation band and full-mutation minimum are the - catalog's (control: the table with HTT's minimum changed to 37 is caught). - -Run: python3 tests/test_str_ranges.py -""" -import hashlib -import json -import os -import re -import shutil -import subprocess -import sys -import tempfile -import time -import urllib.request - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -OG = os.environ.get("OG", "") - -TSV = os.path.join(REPO, "bin", "str_intermediate_ranges.tsv") -DOC = os.path.join(REPO, "docs", "results", "structural-and-repeats.md") -STEP_DOC = os.path.join(REPO, "docs", "09-str-expansions.md") -COLUMNS = ["locus", "intermediate_min", "intermediate_max", "next_min", "next_class", "label", "citation"] -RANGE_COLUMNS = ["locus", "source_id", "normal_max", "intermediate_min", "intermediate_max", "pathogenic_min", - "no_thresholds"] -CITATION = re.compile(r"^GeneReviews: .+, https://www\.ncbi\.nlm\.nih\.gov/books/NBK\d+/$") - -FAILS = [] - - -def check(desc, ok, detail=""): - print(f"{'PASS' if ok else 'FAIL'} {desc}" + ("" if ok else f": {detail}")) - if not ok: - FAILS.append(desc) - - -def og_or_skip(): - """None when OG names an og binary; else the exit code: 0 (skip, with a - note), or 1 with OG_REQUIRED set.""" - if OG and os.access(OG, os.X_OK): - return None - print("skip: OG is not set (OG=rust/target/release/og)") - return 1 if os.environ.get("OG_REQUIRED") else 0 - - -def env(name): - for line in open(os.path.join(REPO, "versions.env")): - if line.startswith(name + "="): - return line.split("=", 1)[1].split("#")[0].strip().strip('"') - raise SystemExit(f"FAIL no {name} line in versions.env") - - -def sha256(path): - h = hashlib.sha256() - with open(path, "rb") as f: - for block in iter(lambda: f.read(1 << 20), b""): - h.update(block) - return h.hexdigest() - - -def files(): - """(release, EH catalog path, loci path), downloaded and checked.""" - rel = env("STRCHIVE_VERSION") - d = os.environ.get("STRCHIVE_DIR") or os.path.join(tempfile.gettempdir(), f"pgp-strchive-{rel}") - os.makedirs(d, exist_ok=True) - out = [] - for name, var in ((f"STRchive-disease-loci-{rel}.hg38.expansionhunter.json", "STRCHIVE_EH_SHA256"), - (f"STRchive-loci-{rel}.json", "STRCHIVE_LOCI_SHA256")): - path = os.path.join(d, name) - if not (os.path.exists(path) and sha256(path) == env(var)): - url = f"https://github.com/dashnowlab/STRchive/releases/download/{rel}/{name}" - for i in range(3): - try: - with urllib.request.urlopen(url, timeout=60) as r, open(path + ".part", "wb") as f: - shutil.copyfileobj(r, f) - break - except OSError as e: - print(f"download {i + 1}/3 of {url} failed: {e}") - time.sleep(5 * (i + 1)) - os.replace(path + ".part", path) - if sha256(path) != env(var): - raise SystemExit(f"FAIL {name} does not have the sha256 {var} pins") - out.append(path) - return rel, out[0], out[1] - - -def build_catalog(): - """The converted catalog (list of entries) and its path, in a temp file; the - ranges table (--ranges) is str_ranges.tsv beside it.""" - rel, eh, loci = files() - out = os.path.join(tempfile.mkdtemp(), "catalog.json") - rc = subprocess.run([OG, "str", "catalog", "--eh", eh, "--loci", loci, "--release", rel, "--out", out, - "--ranges", os.path.join(os.path.dirname(out), "str_ranges.tsv")]).returncode - if rc != 0: - raise SystemExit("FAIL the converter failed on the pinned files") - return json.load(open(out)), out - - -def table(path): - head, out = None, [] - for line in open(path): - if line.startswith("#") or not line.strip(): - continue - c = line.rstrip("\n").split("\t") - if head is None: - head = c - continue - out.append(c) - return head, out - - -def doc_rows(): - """Locus -> its row of the repeat table in docs/results/structural-and-repeats.md.""" - out = {} - for line in open(DOC): - c = [x.strip() for x in line.strip().strip("|").split("|")] - if line.startswith("|") and len(c) >= 6: - out.setdefault(c[0].upper(), line) - return out - - -def num(x): - return int(x) if x != "" else None - - -def range_problems(rows, cat): - """What is wrong with the generated ranges rows against the catalog (empty: nothing).""" - bad = [] - by = {e["LocusId"]: e for e in cat} - loci = [r[0] for r in rows] - if sorted(loci) != sorted(by): - bad.append(f"loci differ from the catalog's: missing {sorted(set(by) - set(loci))}, " - f"extra {sorted(set(loci) - set(by))}") - if len(loci) != len(set(loci)): - bad.append(f"twice: {sorted({x for x in loci if loci.count(x) > 1})}") - for r in rows: - if len(r) != len(RANGE_COLUMNS) or r[0] not in by: - bad.append(f"bad row {r}") - continue - d, e = dict(zip(RANGE_COLUMNS, r)), by[r[0]] - if d["source_id"] != e["SourceId"]: - bad.append(f"{r[0]}: source_id {d['source_id']} is not {e['SourceId']}") - if (num(d["normal_max"]), num(d["pathogenic_min"])) != (e.get("NormalMax"), e.get("PathologicMin")): - bad.append(f"{r[0]}: {d['normal_max']}/{d['pathogenic_min']} are not the catalog's thresholds") - if d["no_thresholds"] != e.get("NoThresholds", ""): - bad.append(f"{r[0]}: no_thresholds differs from the catalog's NoThresholds") - return bad - - -def part_generated(cat, cat_path): - path = os.path.join(os.path.dirname(cat_path), "str_ranges.tsv") - if not os.path.exists(path): - check("the converter writes str_ranges.tsv beside the catalog", False, path) - return {} - head, rows = table(path) - check("str_ranges.tsv: the header", head == RANGE_COLUMNS, head) - check("str_ranges.tsv: 82 rows, one per ExpansionHunter locus of STRchive", len(rows) == 82, len(rows)) - bad = range_problems(rows, cat) - check("str_ranges.tsv: one row per catalog locus, with the catalog's thresholds", not bad, bad) - check("control: a table with one row dropped is caught", bool(range_problems(rows[1:], cat))) - check("control: a table with one row twice is caught", bool(range_problems(rows + rows[:1], cat))) - by = {r[0]: dict(zip(RANGE_COLUMNS, r)) for r in rows} - loci = files()[2] - rec = {x["id"]: x for x in json.load(open(loci))} - same = all((num(d["intermediate_min"]), num(d["intermediate_max"])) - == (rec[d["source_id"]].get("intermediate_min"), rec[d["source_id"]].get("intermediate_max")) - for d in by.values()) - check("str_ranges.tsv: the intermediate ranges are STRchive's", same) - check("ATXN2 (SCA2): normal to 28, intermediate 29-34, pathogenic from 35 (STRchive)", - [by.get("ATXN2", {}).get(k) for k in RANGE_COLUMNS[2:6]] == ["28", "29", "34", "35"], by.get("ATXN2")) - check("MUC1: no thresholds, and the reason", by.get("MUC1", {}).get("normal_max") == "" - and "short reads cannot size it" in by.get("MUC1", {}).get("no_thresholds", ""), by.get("MUC1")) - return by - - -def part_subset(gen): - head, body = table(TSV) - check("the header is the columns og summary collect reads", head == COLUMNS, head) - check("at least HTT, FMR1 and DMPK are listed", {"HTT", "FMR1", "DMPK"} <= {r[0] for r in body}, - [r[0] for r in body]) - loci = [r[0] for r in body] - check("one row per locus", len(loci) == len(set(loci)), loci) - docs = doc_rows() - - def inside(locus, lo, hi): - g = gen.get(locus, {}) - return bool(g.get("normal_max")) and int(g["normal_max"]) < lo and hi < int(g["pathogenic_min"]) - for r in body: - if len(r) != len(COLUMNS): - check(f"{r[0]}: {len(COLUMNS)} columns", False, r) - continue - d = dict(zip(COLUMNS, r)) - nums = [d["intermediate_min"], d["intermediate_max"], d["next_min"]] - if not all(x.isdigit() for x in nums): - check(f"{d['locus']}: whole-number range", False, nums) - continue - lo, hi, nxt = map(int, nums) - check(f"{d['locus']}: min <= max ({lo}-{hi})", lo <= hi, (lo, hi)) - check(f"{d['locus']}: the range ends right below {d['next_class']} ({nxt})", hi + 1 == nxt, (hi, nxt)) - check(f"{d['locus']}: a label and a next class", bool(d["label"].strip() and d["next_class"].strip()), d) - check(f"{d['locus']}: cited to a GeneReviews chapter", bool(CITATION.match(d["citation"])), d["citation"]) - row = docs.get(d["locus"].upper(), "") - check(f"{d['locus']}: docs/results/structural-and-repeats.md prints {lo}-{hi} in its row", - bool(re.search(rf"(? (normal max, pre-mutation band, full-mutation minimum) of the Key Disease Thresholds table.""" - out, on = {}, False - for line in text.splitlines(): - if line.startswith("## "): - on = line.strip() == "## Key Disease Thresholds" - continue - c = [x.strip() for x in line.strip().strip("|").split("|")] - if on and line.startswith("|") and len(c) == 6 and c[1] not in ("Gene", "---"): - out[c[1]] = (c[3], c[4], c[5]) - return out - - -def threshold_problems(rows, cat): - by = {e["LocusId"]: e for e in cat} - bad = [] if rows else ["no rows"] - for gene, got in rows.items(): - e = by.get(gene, {}) - if "NormalMax" not in e: - bad.append(f"{gene}: no thresholds in the catalog") - continue - n, p = e["NormalMax"], e["PathologicMin"] - want = (f"≤{n}", f"{n + 1}-{p - 1}", f"≥{p}") - if got != want: - bad.append(f"{gene}: the docs say {got}, the catalog {want}") - return bad - - -def part_docs(cat): - text = open(STEP_DOC).read() - rows = threshold_rows(text) - check("docs/09-str-expansions.md: the Key Disease Thresholds table lists HTT, FMR1, FXN, C9ORF72, DMPK, ATXN1, " - "ATXN2", {"HTT", "FMR1", "FXN", "C9ORF72", "DMPK", "ATXN1", "ATXN2"} <= set(rows), sorted(rows)) - bad = threshold_problems(rows, cat) - check("docs/09-str-expansions.md: every row gives the catalog's thresholds", not bad, bad) - planted = threshold_rows(text.replace("| HTT | CAG | ≤26 | 27-35 | ≥36 |", "| HTT | CAG | ≤26 | 27-35 | ≥37 |")) - check("control: the table with HTT's pathogenic minimum changed to 37 is caught", - planted != rows and bool(threshold_problems(planted, cat)), planted.get("HTT")) - - -def main(): - skip = og_or_skip() # the catalog is built with og (OG) - if skip is not None: - return skip - cat, cat_path = build_catalog() - gen = part_generated(cat, cat_path) - part_subset(gen) - part_docs(cat) - if FAILS: - print(f"{len(FAILS)} check(s) failed") - return 1 - print("all checks passed") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/test_traits_report.py b/tests/test_traits_report.py deleted file mode 100644 index b1ad8e57b..000000000 --- a/tests/test_traits_report.py +++ /dev/null @@ -1,422 +0,0 @@ -#!/usr/bin/env python3 -# og-leg: OG_REQUIRED -"""Step 42 CLI goldens, rule-allele controls and og cn marker outputs. - -Internal validation, phase and bounded-reader assertions live in -og-steps::traits::tests. The parity harness checks the old Python at oracle.ref. -""" -import csv -import hashlib -import os -import shutil -import subprocess -import sys -import tempfile -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -TABLE = os.path.join(REPO, "assets", "traits.tsv") -RULES = os.path.join(REPO, "assets", "genotype_rules.tsv") -CHECK = os.path.join(REPO, "scripts", "ci", "check-rule-alleles.sh") -OG = os.environ.get("OG", "") -SMN_HEADER = "chrom\tstart\tend\tlocus\tagCN_filter\tagCN\tagCN_qual\tpsCN_filter\tpsCN\tpsCN_qual\thomologous_regions\n" -SMN1_NOT_PROOF = "This marker does not prove 2+0, and its absence does not rule 2+0 out." -FAILS = 0 -# The traits table's columns (og traits table --out). -OUT_COLUMNS = ["rsid", "gene", "chrom", "pos", "trait", "genotype", "status", "reading", "ancestry", - "note", "pmids"] -# The copy-aware markers, pinned here as the test's own copy: og cn -# marker-regions must print the same copies (rust/crates/og-cn/src/markers.rs). -MARKERS = { - # KIV-2 4925G>A: the last base of the second KIV-2 exon (its donor splice - # site), G>A on LPA's minus strand, so C>T on the plus strand. GRCh38 holds - # it at the end of each of the six KIV-2 units (exons 5, 7, ..., 15 of - # ENST00000316300.10; NM_005577.4:c.733G>A at the first, p.Ala245Thr) and - # at the near-identical last exon of KIV-1 (exon 3, c.391): seven copies, - # the block og cn call --locus LPA_KIV2 counts as N + 2 copies (N KIV-2 repeats and - # one KIV-1 per chromosome). dbSNP names the first KIV-2 copy only. - "LPA_KIV2_4925": { - "name": "LPA KIV-2 4925G>A", "chrom": "chr6", "ref": "C", "alt": "T", - "dbsnp": ("rs1484919544", "chr6", 160640667, "C", "T"), - "copies": (160646214, 160640667, 160635123, 160629574, 160624030, 160618484, 160612932), - "context": "GACTCCTTACCTTGTTCGGAA", - "min_depth": 100, "min_alt": 5, "min_fraction": 0.008, "max_absent": 2, - }, - # SMN1 g.27134T>G is NM_000344.4:c.*3+80T>G in intron 7 (plus strand). - # SMN2 holds the same 21 bases 875,420 bases upstream, as at exon 7. - "SMN1_27134": { - "name": "SMN1 g.27134T>G", "chrom": "chr5", "ref": "T", "alt": "G", - "dbsnp": ("rs143838139", "chr5", 70952074, "T", "G"), - "copies": (70952074, 70076654), - "context": "TTTTTGAACATTTAAAAAGTT", - "min_depth": 10, "min_alt": 3, "min_fraction": 0.05, "max_absent": 1, - }, -} -APOE_GOLDEN = { - ("T/T", "C/C"): "e3d7512b649f1d23f5acbcaa08da19bd6a009d5b39ac57cb6856591c50e7cfda", - ("T/T", "C/T"): "62e910d8ec7ac45ab15e9c9ed85c93b128ac41703d1149a332082dc0d7b5f389", - ("T/T", "T/T"): "41b4ad51c67028bb19af1b7ab7e4f414f5c0354e4c4a7c3c19309186b429f960", - ("T/C", "C/C"): "08c5ed9e2898f2ae29cbd49d5ac97394758fa852f9d35e3c7173b2aa8d3111ad", - ("T/C", "C/T"): "555c9cc0e8b59b10bdc5bdc2e33d93fb4ccbee1e6ea12916b17e73ffd6fdb8d8", - ("T/C", "T/T"): "043f51b41486f217cac3a71e705e0e29c1a1ced9745e60abde2230df2fe901db", - ("C/C", "C/C"): "ca7f36540cf1269b84649b4ad6ace19e0abc03eab7984d495de584540bb11dcc", - ("C/C", "C/T"): "db499d1bfebedf77ed34a9d0525030d34be6336e685758a8efd53125197e3d00", - ("C/C", "T/T"): "16e5214f90c5f2c401e5849dffd76a08dc3fc5ba2deb324d7fb02c76a92113be", - ("T/C", "."): "34624db16a92981c0882edb3d8ce8d291162aa78b7c5208eba2360141e6ccbd8", - ("T/G", "C/C"): "1d0aee0b8d241561ebd7477d6f4a8be274a48157a297abecd444c6c6a0a6abac", -} -def check(desc, ok, detail=""): - global FAILS - print(f"[{'PASS' if ok else 'FAIL'}] {desc}{'' if ok else ' -- ' + str(detail)[:600]}") - if not ok: - FAILS += 1 - - -def write(path, text): - with open(path, "w") as f: - f.write(text) - - -def dbsnp_checks(rules): - """10. scripts/ci/check-rule-alleles.sh on the rules and on planted copies. Runs first, - so a rules table that breaks the other checks still shows what dbSNP says.""" - work = tempfile.mkdtemp() - try: - def dbsnp(path, *extra): - r = subprocess.run(["bash", CHECK, *extra, path], capture_output=True, text=True) - return r.returncode, r.stdout + r.stderr - - rc, log = dbsnp(RULES) - print(" " + log.strip().replace("\n", "\n ")) - n = sum(len(v["sites"]) for v in rules.values()) - check(f"check-rule-alleles.sh: every rule site matches dbSNP ({n} sites)", - rc == 0 and f"{n} rule sites checked" in log, log) - text = open(RULES).read() - site = "site\tTAS2R38\trs1726866\tchr7\t141972905\t" - - def planted(desc, new, want): - f = os.path.join(work, "planted.tsv") - write(f, text.replace(site + "G\tA", site + new)) - rc, log = dbsnp(f) - check(f"check-rule-alleles.sh control: {desc} fails, naming it", rc == 1 and want in log, (rc, log)) - - planted("ref and alt swapped", "A\tG", "rs1726866: ref and alt are swapped") - planted("the minus-strand bases", "C\tT", "rs1726866: the alleles are the minus-strand bases") - f = os.path.join(work, "planted.tsv") - write(f, text.replace("\trs1726866\tchr7\t141972905\t", "\trs1726867\tchr7\t141972905\t")) - rc, log = dbsnp(f) - check("check-rule-alleles.sh control: an rsID the dbSNP table lacks fails", - rc == 1 and "rs1726867: not in the pinned dbSNP table" in log, (rc, log)) - write(f, text.replace("\trs1726866\tchr7\t141972905\t", "\trs1726866\tchr7\t141972906\t")) - rc, log = dbsnp(f) - check("check-rule-alleles.sh control: a wrong position fails", - rc == 1 and "dbSNP at chr7:141972905" in log, (rc, log)) - write(f, "# no sites\n") - rc, log = dbsnp(f) - check("check-rule-alleles.sh control: a rules file with no site fails", rc == 1 and "no site line" in log, - (rc, log)) - root = os.path.join(work, "root") - os.makedirs(os.path.join(root, "scripts", "ci")) - shutil.copy(os.path.join(REPO, "versions.env"), root) - copy = os.path.join(root, "scripts", "ci", "check-rule-alleles.sh") - write(copy, open(CHECK).read().replace("\n#: # build ", "\n#: # build x")) - r = subprocess.run(["bash", copy, RULES], capture_output=True, text=True) - check("check-rule-alleles.sh control: a dbSNP table of another build than versions.env pins fails", - r.returncode == 1 and "versions.env pins" in r.stderr, (r.returncode, r.stdout, r.stderr)) - finally: - shutil.rmtree(work) - - -MULTI_SNP = { - "APOL1": [(("G/G", "G/G"), "G1/G1"), (("A/G", "T/G"), "G0/G1")], - "SERPINA1": [(("T/T", "T/T"), "PiZZ"), (("T/A", "C/T"), "PiSZ"), (("A/A", "C/C"), "PiSS")], - "HFE": [(("A/A", "C/C"), "C282Y/C282Y"), (("G/A", "C/G"), "C282Y/H63D"), (("G/G", "G/G"), "H63D/H63D")], - "GC": [(("C/C", "G/G"), "Gc1S/Gc1S"), (("A/C", "G/T"), "Gc1S/Gc2"), (("A/A", "T/T"), "Gc2/Gc2")], - "CAD9p21": [(("G/G", "C/C"), "two risk copies"), (("A/G", "G/C"), "one risk copy")], - "MAPT": [(("G/G", "G/G"), "H2/H2"), (("A/G", "A/G"), "H1/H2")], -} -# Independent labels for homozygous GRCh38 reference blocks. -MULTI_SNP_REF = {"APOL1": "G0/G0", "SERPINA1": "PiMM", "HFE": "wt/wt", "GC": "Gc1F/Gc1F", - "CAD9p21": "no risk copy", "MAPT": "H1/H1"} - - - -def says_not_proof(message): - """The SMN1 marker's message must say the marker does not prove 2+0.""" - return "does not prove 2+0" in message - - -def marker_errors(name, m=None, dbsnp=None): - """Problems of one marker (empty when it is consistent): the dbSNP record - is one of its copies with its ref and alt, on the plus strand and not - swapped; every copy sits in the middle of the same context, whose middle - base is the ref.""" - m = MARKERS[name] if m is None else m - rs, chrom, pos, ref, alt = m["dbsnp"] if dbsnp is None else dbsnp - comp = {"A": "T", "C": "G", "G": "C", "T": "A"} - errs = [] - if chrom != m["chrom"] or pos not in m["copies"]: - errs.append(f"{name}: dbSNP places {rs} at {chrom}:{pos}, not at one of the marker's copies") - if (m["ref"], m["alt"]) == (alt, ref): - errs.append(f"{name}: ref and alt are swapped (marker {m['ref']}>{m['alt']}, dbSNP {ref}>{alt})") - elif (m["ref"], m["alt"]) == (comp.get(ref), comp.get(alt)): - errs.append(f"{name}: the alleles are the minus-strand bases (marker {m['ref']}>{m['alt']}, " - f"dbSNP {ref}>{alt}): write the plus-strand bases") - elif (m["ref"], m["alt"]) != (ref, alt): - errs.append(f"{name}: marker {m['ref']}>{m['alt']}, dbSNP {ref}>{alt}") - ctx = m["context"] - if len(ctx) % 2 != 1 or ctx[len(ctx) // 2] != m["ref"]: - errs.append(f"{name}: the middle base of the context is not the ref {m['ref']}") - if len(set(m["copies"])) != len(m["copies"]): - errs.append(f"{name}: a copy is listed twice") - return errs - - -def marker_checks(): - """11. The copy-aware markers.""" - for name, m in MARKERS.items(): - check(f"marker {name}: the pinned dbSNP record agrees with the marker", marker_errors(name) == [], - marker_errors(name)) - rs, chrom, pos, ref, alt = m["dbsnp"] - comp = {"A": "T", "C": "G", "G": "C", "T": "A"} - e = marker_errors(name, dict(m, ref=alt, alt=ref)) - check(f"marker {name} control: ref and alt swapped is caught", any("swapped" in x for x in e), e) - e = marker_errors(name, dict(m, ref=comp[ref], alt=comp[alt], - context=m["context"][:10] + comp[ref] + m["context"][11:])) - check(f"marker {name} control: the minus-strand bases are caught", any("minus-strand" in x for x in e), e) - e = marker_errors(name, dbsnp=(rs, chrom, pos + 1, ref, alt)) - check(f"marker {name} control: a dbSNP position off the copies is caught", - any("not at one of the marker's copies" in x for x in e), e) - e = marker_errors(name, dict(m, context=m["context"][:10] + alt + m["context"][11:])) - check(f"marker {name} control: a context whose middle base is not the ref is caught", - any("middle base" in x for x in e), e) - check("LPA_KIV2_4925: seven copies (six KIV-2 units and KIV-1)", len(MARKERS["LPA_KIV2_4925"]["copies"]) == 7) - if not OG: - print("skip: OG is not set (OG=rust/target/release/og): og cn marker-regions and marker-read did not run") - return - for name, m in MARKERS.items(): - res = og_cn("marker-regions", name) - check(f"marker {name}: og cn marker-regions prints one region per copy of MARKERS", - res.returncode == 0 and res.stdout.split() == [f"{m['chrom']}:{p}-{p}" for p in m["copies"]], res) - - sm = MARKERS["SMN1_27134"] - work = tempfile.mkdtemp() - try: - def pileup(t_reads, g_reads, extra=""): - p = os.path.join(work, "p.tsv") - write(p, "".join(f"chr5\t{pos}\tN\t{t_reads + g_reads}\t^I{'T' * t_reads}{'g' * g_reads}$\t.\n" - for pos in sm["copies"]) + extra) - return p - - def smn(pscn="2,2", pqual="30,30", pfilt="PASS", agcn="4", homolog="chr5:70049000-70078000"): - p = os.path.join(work, "smn.tsv") - write(p, SMN_HEADER + f"chr5\t70925000\t70953000\tSMN1\tPASS\t{agcn}\t50\t{pfilt}\t{pscn}\t{pqual}\t{homolog}\n") - return p - - def table(p, smn_path): - out = os.path.join(work, "o.tsv") - if os.path.exists(out): - os.remove(out) - res = og_cn("marker-read", "SMN1_27134", "--pileup", p, "--copy-table", smn_path, "--out", out) - return res, dict(x.split("\t", 1) for x in open(out).read().splitlines()[1:]) if os.path.exists(out) else {} - - res, t = table(pileup(15, 5), smn()) - check("SMN1 marker: a quarter of the reads with G is present, exit 0", - res.returncode == 0 and t.get("status") == "present" and t.get("alt_reads") == "10" - and t.get("depth") == "40" and t.get("fraction") == "0.2500", (res, t)) - check("SMN1 marker: copies = fraction x agCN (0.25 x 4)", t.get("copies") == "1.0" and t.get("agcn") == "4", t) - check("SMN1 marker: the SMN1 psCN at exon 7 is read", t.get("smn1_pscn") == "2 (quality 30, filter PASS)", t) - check("SMN1 marker: present with two copies says it raises the chance of 2+0 and does not prove it", - "higher chance" in t.get("message", "") and says_not_proof(t.get("message", "")), t) - res, t = table(pileup(20, 0), smn()) - check("SMN1 marker: reads with T only (the reference) is not seen, copies '.'", - t.get("status") == "not seen" and t.get("copies") == "." and "lower" in t.get("message", ""), t) - res, t = table(pileup(20, 0, extra="chr5\t70952074\tN\t2\tGG\t.\n"), smn()) - check("SMN1 marker: 2 reads with G is uncertain", t.get("status") == "uncertain", t) - res, t = table(pileup(3, 1), smn()) - check("SMN1 marker: under 10 reads is too little depth", t.get("status") == "too little depth", t) - res, t = table(pileup(15, 5, extra="chr5\t70952075\tN\t30\t" + "G" * 30 + "\t.\n"), smn()) - check("SMN1 marker: reads at another position are not counted", t.get("alt_reads") == "10", t) - bad = os.path.join(work, "bad.tsv") - write(bad, "nonsense\n") - res, t = table(bad, smn()) - check("SMN1 marker: an unreadable pileup is not assessed, exit 0", - res.returncode == 0 and t.get("status") == "not assessed", (res, t)) - res, t = table(pileup(15, 5), smn(pscn="1,3")) - check("SMN1 marker: another SMN1 copy number says the marker refines a two-copy result only", - "SMN1 has 1 copy here" in t.get("message", ""), t) - res, t = table(pileup(15, 5), smn(pqual="12,30")) - check("SMN1 marker: a psCN of quality under 20 is not resolved", - t.get("smn1_pscn") == "2 (quality 12, filter PASS), not resolved" - and "SMN1 copy number: 2 (quality 12, filter PASS), not resolved;" in t.get("message", ""), t) - # the SMN table with SMN2's region first: exon 7 of SMN1 is the homologous region - p = os.path.join(work, "smn2.tsv") - write(p, SMN_HEADER + "chr5\t70049000\t70078000\tSMN1\tPASS\t4\t50\tPASS\t2,2\t30,40\tchr5:70925000-70953000\n") - res, t = table(pileup(15, 5), p) - check("SMN1 marker: exon 7 found through the homologous region (its own psCN and quality)", - t.get("smn1_pscn") == "2 (quality 40, filter PASS)" and t.get("agcn") == "4", t) - res, t = table(pileup(15, 5), os.path.join(work, "missing.tsv")) - check("SMN1 marker: no SMN table says so", "no SMN table" in t.get("smn1_pscn", ""), t) - res, t = table(pileup(15, 5), work) - check("SMN1 marker: an SMN table that cannot be read keeps the marker's reading, exit 0", - res.returncode == 0 and t.get("status") == "present" and t.get("agcn") == "." - and "could not be read" in t.get("smn1_pscn", ""), (res, t)) - - # Every status with every psCN: the message says the marker does not prove 2+0. - msgs = [] - for counts in [(30, 10), (40, 0), (38, 2), (3, 1), None]: - if counts is None: - p = bad - else: # every read at the SMN1 copy - p = os.path.join(work, "all.tsv") - write(p, f"chr5\t{sm['copies'][0]}\tN\t{sum(counts)}\t{'T' * counts[0]}{'G' * counts[1]}\t.\n") - for path in ("2,2", "1,3", "5,5", None): - smn_path = (os.path.join(work, "missing.tsv") if path is None - else smn(pqual=path) if path == "5,5" else smn(pscn=path)) - msgs.append(table(p, smn_path)[1].get("message", "")) - check(f"SMN1 marker: every message ({len(msgs)}) says it does not prove 2+0", - len(msgs) == 20 and all(says_not_proof(x) for x in msgs), [x for x in msgs if not says_not_proof(x)]) - check("SMN1 marker control: a message without the sentence fails that test", - not says_not_proof(msgs[0].replace(SMN1_NOT_PROOF, ""))) - finally: - shutil.rmtree(work) - - - -def traits_tables(genotypes, out, rules_out, sample="S"): - """Use og in the Rust job. Unit shards without og use the pinned oracle.""" - args = ["--table", TABLE, "--rules", RULES, "--genotypes", genotypes, - "--sample", sample, "--out", out, "--rules-out", rules_out] - if OG: - subprocess.run([OG, "traits", "table", *args], check=True, capture_output=True, text=True) - else: - with tempfile.TemporaryDirectory(prefix="traits-oracle-") as work: - ref = "669a8196cf4c377200f9b4af9c235ef85fb774a1" - if subprocess.run(["git", "-C", REPO, "cat-file", "-e", ref + "^{commit}"], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode: - subprocess.run(["git", "-C", REPO, "fetch", "--depth=1", "origin", ref], check=True) - source = subprocess.check_output(["git", "-C", REPO, "show", ref + ":bin/traits_report.py"]) - path = os.path.join(work, "traits_report.py") - with open(path, "wb") as f: - f.write(source) - subprocess.run([sys.executable, path, *args], check=True, capture_output=True, text=True) - - -def og_cn(*args): - return subprocess.run([OG, "cn", *args], capture_output=True, text=True) - - - -def plain_tsv_checks(): - """Reject CSV quoting before it can produce a table the collector cannot read.""" - import collect_summary as cs - with tempfile.TemporaryDirectory() as work: - folder = os.path.join(work, "traits") - os.makedirs(folder) - g = os.path.join(work, "g.tsv") - write(g, "") - out, rout = os.path.join(folder, "S_traits.tsv"), os.path.join(folder, "S_genotype_rules.tsv") - source = open(TABLE).read() - rules_source = open(RULES).read() - for name, text, rules_text in [ - ("quoted header", source.replace("chrom\t", '\"chrom\"\t', 1), rules_source), - ("quoted catalog field", source.replace("\tMCM6\t", '\t\"MCM6\"\t', 1), rules_source), - ("quoted rule text", source, rules_source.replace("\tAPOE\tAPOE\t", '\tAPOE\t\"APOE\"\t', 1)), - ("embedded tab", source.replace("\tMCM6\t", '\t\"MCM6\textra\"\t', 1), rules_source), - ]: - # Keep complete, collector-readable outputs from the previous run. - traits_tables(g, out, rout) - previous = open(out, "rb").read(), open(rout, "rb").read() - bad, badrules = os.path.join(work, "bad.tsv"), os.path.join(work, "badrules.tsv") - write(bad, text) - write(badrules, rules_text) - res = subprocess.run([OG, "traits", "table", "--table", bad, "--rules", badrules, - "--genotypes", g, "--sample", "S", "--out", out, - "--rules-out", rout], capture_output=True, text=True) - check(f"plain TSV control: {name} is refused", res.returncode == 1, res.stderr) - check(f"plain TSV control: {name} preserves both outputs", - previous == (open(out, "rb").read(), open(rout, "rb").read())) - sec = cs.collect("S", work)["sections"]["traits"] - readable = sec["state"] == "ok" and bool(sec["data"]["rows"]) - check(f"plain TSV control: {name} leaves collector-readable output", readable) - - -def main(): - rules = {} - with open(RULES) as f: - for cells in csv.reader((line for line in f if not line.startswith("#")), delimiter="\t"): - if cells and cells[0] == "site": - rules.setdefault(cells[1], {"sites": []})["sites"].append(cells[2:]) - dbsnp_checks(rules) - marker_checks() - if not OG: - check("OG_REQUIRED needs the og binary", not os.environ.get("OG_REQUIRED")) - print("skip: OG is not set: traits CLI and cn marker checks did not run") - return int(bool(FAILS)) - plain_tsv_checks() - with tempfile.TemporaryDirectory() as work: - def table(records): - g = os.path.join(work, "g.tsv") - write(g, "".join("\t".join(rec) + "\n" for rec in records)) - os.makedirs(os.path.join(work, "traits"), exist_ok=True) - out, rout = os.path.join(work, "traits", "S_traits.tsv"), os.path.join(work, "traits", "S_genotype_rules.tsv") - res = subprocess.run([OG, "traits", "table", "--table", TABLE, "--rules", RULES, - "--genotypes", g, "--sample", "S", "--out", out, - "--rules-out", rout], capture_output=True, text=True) - check("traits table exits 0", res.returncode == 0, res.stderr) - if res.returncode: - return [], [] - with open(out) as f: - traits = list(csv.DictReader(f, delimiter="\t")) - with open(rout) as f: - lines = list(csv.DictReader(f, delimiter="\t")) - return traits, lines - for (a, b), want in APOE_GOLDEN.items(): - records = [("chr19", pos, ref, alt, gt) for pos, ref, alt, gt in - [("44908684", "T", "C", a), ("44908822", "C", "T", b)] if gt != "."] - traits, lines = table(records) - apoe = next(r for r in traits if "+" in r["rsid"]) - got = hashlib.sha256("\t".join(apoe[k] for k in OUT_COLUMNS).encode()).hexdigest() - check(f"APOE golden {a}, {b}", got == want, got) - for rid, cases in MULTI_SNP.items(): - for gts, label in cases: - records = [(s[1], s[2], s[3], s[4], gt) for s, gt in zip(rules[rid]["sites"], gts)] - _, lines = table(records) - got = next(r for r in lines if r["rule"] == rid) - check(f"{rid} {gts}: {label}", got["status"] == "called" and got["result"] == label, got) - import collect_summary as cs - import render_report as rr - check("every multi-SNP rule is in the catalog", set(MULTI_SNP) <= set(rules)) - for rid, cases in MULTI_SNP.items(): - sites = rules[rid]["sites"] - records = [(s[1], s[2], s[3], ".", f"{s[3]}/{s[3]}", "40") for s in sites] - _, lines = table(records) - got = next(r for r in lines if r["rule"] == rid) - check(f"{rid}, reference blocks only: {MULTI_SNP_REF[rid]}", - (got["status"], got["result"]) == ("called", MULTI_SNP_REF[rid]), got) - gts, want = cases[0] - a, b = gts[0].split("/") - s0 = sites[0] - flip = s0[3] if b == s0[4] else s0[4] - flipped = (f"{a}/{flip}",) + tuple(gts[1:]) - records = [(s[1], s[2], s[3], s[4], gt) for s, gt in zip(sites, flipped)] - _, lines = table(records) - got = next(r for r in lines if r["rule"] == rid) - check(f"control: {rid} with one allele flipped is not {want}", got["result"] != want, got) - # Collect and render the actual CLI output, independently for each rule. - records = [(s[1], s[2], s[3], s[4], gt) for s, gt in zip(sites, gts)] - _, lines = table(records) - read = cs.collect("S", work)["sections"]["traits"]["data"]["rules"] or [] - page = "\n".join(rr.rules_html({"rules": read}, "S")) - got = next(r for r in lines if r["rule"] == rid) - check(f"HTML report: {rid} with result {want}", rr.E(got["trait"]) in page and rr.E(want) in page, page[:300]) - records = [(s[1], s[2], s[3], s[4], gt) for s, gt in zip(rules["APOL1"]["sites"], ("A/G", "T/G"))] - _, lines = table(records) - reading = next(r for r in lines if r["rule"] == "APOL1")["reading"] - check("APOL1 G0/G1 says G2 is not read and the other copy could carry it", - "G2 is not read" in reading and "G1/G2" in reading, reading) - print("RESULT:", "ALL PASS" if not FAILS else f"{FAILS} FAILED") - return int(bool(FAILS)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/test_ystr_report.py b/tests/test_ystr_report.py deleted file mode 100644 index d8608e342..000000000 --- a/tests/test_ystr_report.py +++ /dev/null @@ -1,360 +0,0 @@ -#!/usr/bin/env python3 -# og-leg: OG_REQUIRED -"""og y str-bed and og y str-report: Y-STR repeat counts from HipSTR's calls (step 45). - -Each case writes a synthetic HipSTR VCF (one sample, only the records it -plants), runs og y str-report on assets/ystr_panel.tsv, and checks the table: - - 1. the panel: 111 markers in FamilyTreeDNA's order (the Y-12, Y-25 and - Y-37 heads checked by name), every multi-copy marker kind multicopy, - the BED holds every region of the single and parts markers once, and - one region over the whole locus for each span marker - 2. counts: reference copies plus GB / motif length; a partial repeat as - N.r; a negative GB; the sum of a marker's parts (DYS449); a span - marker's parts' copies plus its one GB (DYS510, DYS389II at 250 bp); - an offset from a panel that gives one - 3. confidence: high; low for a posterior under 0.9, for over 15 % of - reads with stutter or with a flank indel, and for under 10 reads; no - call for a region without a record or without a genotype, for two - lengths on the haploid Y, and for a marker whose other part is missing; - a region without a record carries the reason HipSTR's log gives for it - 4. not callable: multi-copy markers (DYS464, DYF395S1), markers the - marker set has no region for (DYS439), and a region longer than the - reads can span (DYS452 at 150 bp reads, callable at 250 bp; the spans - of DYS389II and DYS448) - 5. the CSV: one row per marker in the panel's order, the count and the - confidence - 6. a damaged input: a VCF with two samples, a panel out of order - 7. the reports: bin/collect_summary.py reads the table (counts per - confidence) and refuses one with an unknown confidence; the HTML card of - bin/render_report.py lists the called markers only, with the note, and - the HTML report shows an unreadable table as Unreadable - -Without OG (the og binary) the test is skipped with a note; with -OG_REQUIRED set too, it fails (Guard's rust job sets both). - -Run: OG=bin/og python3 tests/test_ystr_report.py -""" -import atexit -import csv -import gzip -import os -import shutil -import subprocess -import sys -import tempfile - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -OG = os.environ.get("OG", "") -PANEL = os.path.join(REPO, "assets", "ystr_panel.tsv") -sys.path.insert(0, os.path.join(REPO, "bin")) -if not OG: - print("skip: OG is not set (OG=bin/og)") - sys.exit(1 if os.environ.get("OG_REQUIRED") else 0) - -FAILS = [] - - -def check(name, got, want): - if got == want: - print(f"[PASS] {name}") - else: - print(f"[FAIL] {name}: got {got!r}, want {want!r}") - FAILS.append(name) - - -def read_panel(path): - """The panel's rows as dicts, each region as (chrom, start, end, period, copies, name); a span - marker has one region over its whole locus, its parts' copies rounded half up and summed.""" - with open(path, encoding="utf-8") as f: - lines = [line.rstrip("\n").split("\t") for line in f if line.strip() and not line.startswith("#")] - rows = [dict(zip(lines[0], c)) for c in lines[1:]] - for r in rows: - r["regions"] = [] if r["regions"] == "." else [ - (where.split(":")[0], int(where.split(":")[1].split("-")[0]), int(where.split(":")[1].split("-")[1]), - int(period), float(copies), name) - for where, period, copies, name in (x.split("/", 3) for x in r["regions"].split(";"))] - if r["kind"] == "span": # one region over the whole locus, named after the marker - regs = r["regions"] - r["regions"] = [(regs[0][0], regs[0][1], regs[-1][2], regs[0][3], - float(sum(int(x[4] + 0.5) for x in regs)), r["marker"])] - return rows - - -TMP = tempfile.mkdtemp(prefix="ystr-test-") -atexit.register(shutil.rmtree, TMP, ignore_errors=True) -FORMAT = "GT:GB:Q:PQ:DP:DSNP:PSNP:PDP:GLDIFF:DSTUTTER:DFLANKINDEL" - - -def vcf(records, samples=("S1",), gz=True): - """records: {HipSTR name: (GT, GB, Q, DP, DSTUTTER, DFLANKINDEL)}.""" - panel = read_panel(PANEL) - where = {n: (c, s) for r in panel for c, s, _e, _p, _k, n in r["regions"]} - lines = ["##fileformat=VCFv4.1", "##source=HipSTR", - "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t" + "\t".join(samples)] - for name, (gt, gb, q, dp, st, fl) in records.items(): - c, s = where.get(name, ("chrY", 1)) - cell = f"{gt}:{gb}:{q}:{q}:{dp}:0:0|0:{dp}|0:.:{st}:{fl}" - lines.append("\t".join([c, str(s - 1), name, "A", ".", ".", "PASS", "PERIOD=4", FORMAT] - + [cell] * len(samples))) - p = os.path.join(TMP, f"calls{len(os.listdir(TMP))}.vcf" + (".gz" if gz else "")) - data = ("\n".join(lines) + "\n").encode() - with (gzip.open(p, "wb") if gz else open(p, "wb")) as f: - f.write(data) - return p - - -def run(records=None, panel=PANEL, read_length=None, samples=("S1",), want_rc=0, log=None): - v = vcf(records or {}, samples) - tsv, out_csv = v + ".tsv", v + ".csv" - cmd = [OG, "y", "str-report", "--panel", panel, "--vcf", v, "--tsv", tsv, "--csv", out_csv] - if read_length: - cmd += ["--read-length", str(read_length)] - if log is not None: - lp = v + ".log" - with open(lp, "w") as f: - f.write(log) - cmd += ["--log", lp] - p = subprocess.run(cmd, capture_output=True, text=True) - if p.returncode != want_rc: - print(p.stdout, p.stderr) - check(f"exit code {want_rc} ({' '.join(cmd[2:3])})", p.returncode, want_rc) - if want_rc: - return p - with open(tsv, encoding="utf-8") as f: - rows = list(csv.DictReader(f, delimiter="\t")) - with open(out_csv, encoding="utf-8") as f: - crow = list(csv.reader(f)) - return {r["marker"]: r for r in rows}, rows, crow, p.stdout - - -GOOD = ("0", "0", "0.99", "20", "0", "0") - - -def good(gb="0", dp="20"): - return ("0" if gb == "0" else "1", gb, "0.99", dp, "0", "0") - - -# --- 1. the panel -------------------------------------------------------------------- -panel = read_panel(PANEL) -names = [r["marker"] for r in panel] -check("1 111 markers", len(panel), 111) -check("1 Y-12 in FamilyTreeDNA's order", names[:12], - ["DYS393", "DYS390", "DYS19", "DYS391", "DYS385a", "DYS385b", "DYS426", "DYS388", "DYS439", - "DYS389I", "DYS392", "DYS389II"]) -check("1 Y-25 (13-25)", names[12:25], - ["DYS458", "DYS459a", "DYS459b", "DYS455", "DYS454", "DYS447", "DYS437", "DYS448", "DYS449", - "DYS464a", "DYS464b", "DYS464c", "DYS464d"]) -check("1 Y-37 (26-37)", names[25:37], - ["DYS460", "Y-GATA-H4", "YCAIIa", "YCAIIb", "DYS456", "DYS607", "DYS576", "DYS570", "CDYa", "CDYb", - "DYS442", "DYS438"]) -check("1 the last marker is DYS435", names[-1], "DYS435") -check("1 panel sizes", [sum(1 for r in panel if r["panel"] == p) for p in ("Y-12", "Y-25", "Y-37", "Y-67", "Y-111")], - [12, 13, 12, 30, 44]) -multi = {r["marker"] for r in panel if r["kind"] == "multicopy"} -check("1 the multi-copy markers", multi, - {"DYS385a", "DYS385b", "DYS459a", "DYS459b", "DYS464a", "DYS464b", "DYS464c", "DYS464d", "YCAIIa", "YCAIIb", - "CDYa", "CDYb", "DYF395S1a", "DYF395S1b", "DYS413a", "DYS413b"}) -bed = os.path.join(TMP, "regions.bed") -check("1 bed exits 0", subprocess.run([OG, "y", "str-bed", "--panel", PANEL, "--out", bed]).returncode, 0) -with open(bed) as f: - bl = [line.rstrip("\n").split("\t") for line in f] -want_regions = {n for r in panel for *_x, n in r["regions"]} -check("1 the BED has every region once", sorted(x[5] for x in bl), sorted(want_regions)) -check("1 the BED has six columns, 1-based start below the end", all(len(x) == 6 and 0 < int(x[1]) < int(x[2]) for x in bl), - True) -check("1 the BED is sorted", [int(x[1]) for x in bl], sorted(int(x[1]) for x in bl)) -check("1 DYS393's region", [x for x in bl if x[5] == "DYS393"], [["chrY", "3263111", "3263158", "4", "12", "DYS393"]]) -spans = {r["marker"] for r in panel if r["kind"] == "span"} -check("1 the span markers", spans, {"DYS389II", "DYS448", "DYS450", "DYS504", "DYS510", "DYS532"}) -check("1 the BED: 86 regions (76 single, 6 spans, the 4 parts of DYS449 and DYS552)", len(bl), 86) -check("1 DYS450 is one region from its first part's start to its last part's end, 9 + 5 copies", - [x for x in bl if x[5].startswith("DYS450")], [["chrY", "8258259", "8258341", "5", "14", "DYS450"]]) -check("1 DYS510's three parts are one region, 17 + 5 + 3 copies (4.5 halves up)", - [x for x in bl if x[5].startswith("DYS510")], [["chrY", "15188021", "15188138", "4", "25", "DYS510"]]) -check("1 no part of a span marker is in the BED", [x[5] for x in bl if x[5].split(".")[0] in spans and "." in x[5]], []) -check("1 DYS449 keeps its two parts", sorted(x[5] for x in bl if x[5].startswith("DYS449")), ["DYS449.1", "DYS449.2"]) - -# --- 2. counts ------------------------------------------------------------------------- -by, rows, crow, out = run({ - "DYS393": good("4"), # one repeat longer than the reference's 12 - "DYS390": good("-8"), # two shorter than 24 - "DYS391": good("2"), # a partial repeat: 11.2 - "DYS392": good("-1"), # three-base motif, one base short: 12.2 - "DYS449.1": good("4"), "DYS449.2": good("-4"), # 15 (15.25 rounded) + 1, plus 21 - 1 - "DYS510": good("4"), # one region: 17 + 5 (4.5 halves up) + 3, plus 1 - "DYS389II": good("4"), "DYS448": good("6"), # spans longer than 150 bp reads can cover - "DYS19/DYS394": GOOD, -}) -check("2 a reference call keeps the reference copies (DYS19: 16)", by["DYS19"]["count"], "16") -check("2 GB +4 at a 4-base motif: one repeat more (DYS393 13)", by["DYS393"]["count"], "13") -check("2 GB -8: two repeats fewer (DYS390 22)", by["DYS390"]["count"], "22") -check("2 GB +2: a partial repeat (DYS391 11.2)", by["DYS391"]["count"], "11.2") -check("2 GB -1 at a 3-base motif (DYS392 12.2)", by["DYS392"]["count"], "12.2") -check("2 DYS449 is the sum of its two parts (16 + 20)", by["DYS449"]["count"], "36") -check("2 DYS510: its parts' copies (17 + 5 + 3, 4.5 halves up) plus its one GB", by["DYS510"]["count"], "26") -check("2 DYS510 reads one region: one read count, no part names in the note", - (by["DYS510"]["confidence"], by["DYS510"]["reads"], by["DYS510"]["note"]), ("high", "20", ".")) -for m, bp in (("DYS389II", 164), ("DYS448", 156)): - check(f"2 {m}: the span is {bp} bp, not callable from 150 bp reads", - (by[m]["confidence"], by[m]["count"], f"a {bp} bp repeat" in by[m]["note"]), ("not callable", "", True)) -by250 = run({"DYS389II": good("4")}, read_length=250)[0] -check("2 DYS389II at 250 bp reads: 17 + 12 copies, plus 1", by250["DYS389II"]["count"], "30") -check("2 the DYS389II note says the count covers DYS389I", "covers DYS389I" in by250["DYS389II"]["note"], True) -check("2 the summary line counts the confidences", out.strip().startswith("Y-STR markers: "), True) -off = os.path.join(TMP, "offset_panel.tsv") -with open(PANEL) as f, open(off, "w") as g: - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) == 8 and c[2] == "DYS393": - c[6] = "-1" - g.write("\t".join(c) + "\n") -oby = run({"DYS393": good("4")}, panel=off)[0] -check("2 a panel offset is added (13 - 1)", oby["DYS393"]["count"], "12") - -# --- 3. confidence --------------------------------------------------------------------- -by = run({ - "DYS393": GOOD, - "DYS390": ("0", "0", "0.6", "20", "0", "0"), # posterior below 0.9 - "DYS391": ("0", "0", "0.99", "20", "4", "0"), # 20 % stutter - "DYS426": ("0", "0", "0.99", "20", "0", "4"), # 20 % flank indels - "DYS388": ("0", "0", "0.99", "6", "0", "0"), # six reads - "DYS392": (".", ".", ".", "0", "0", "0"), # no genotype - "DYS458": ("0/1", "0/4", "0.99", "20", "0", "0"), # two lengths - "DYS449.1": GOOD, # DYS449.2 missing - "DYS456": ("0", "0", "0.99", "20", "3", "3"), # exactly 15 %: still high -})[0] -check("3 a clean call is high", (by["DYS393"]["confidence"], by["DYS393"]["reads"], by["DYS393"]["q"]), - ("high", "20", "0.99")) -check("3 posterior 0.6: low", by["DYS390"]["confidence"], "low") -check("3 and the note says why", "posterior 0.6 below 0.9" in by["DYS390"]["note"], True) -check("3 a low call keeps its count", by["DYS390"]["count"], "24") -check("3 4 of 20 reads with stutter: low", (by["DYS391"]["confidence"], by["DYS391"]["stutter_reads"]), ("low", "4")) -check("3 4 of 20 reads with a flank indel: low", by["DYS426"]["confidence"], "low") -check("3 6 reads: low", (by["DYS388"]["confidence"], "6 reads, fewer than 10" in by["DYS388"]["note"]), ("low", True)) -check("3 3 of 20 (15 %) is not above the limit: high", by["DYS456"]["confidence"], "high") -check("3 no genotype: no call, no count", (by["DYS392"]["confidence"], by["DYS392"]["count"]), ("no call", "")) -check("3 two lengths on the Y: no call", (by["DYS458"]["confidence"], "two lengths" in by["DYS458"]["note"]), - ("no call", True)) -check("3 a missing part: no call", (by["DYS449"]["confidence"], "DYS449.2" in by["DYS449"]["note"]), - ("no call", True)) -check("3 a region without a record: no call", by["DYS19"]["confidence"], "no call") -check("3 without HipSTR's log, the note points to it", "see HipSTR's log" in by["DYS19"]["note"], True) -# HipSTR's log: a "Processing region" line per region (0-based start), then why it stopped. -LOG = """Processing region chrY 9684379 9684443 -Aligning reads to each candidate haplotype -Aborting genotyping of the locus as the sequence downstream of the repeat is too repetitive for accurate genotyping -\tFlanking sequence = AAACACTATATATATATAACACTATATATATAATAC -Processing region chrY 18041621 18041651 -Skipping locus with too few reads: TOTAL=3, MIN=5 -Processing region chrY 3263110 3263158 -Locus timing: -Processing region chrY 8258258 8258341 -Skipping locus with too few informative reads for stutter training: TOTAL=2, MIN=5 -Processing region chrY 12500447 12500611 -Skipping region as the reference allele length exceeds the threshold (164 vs 150) -""" -by = run({"DYS390": GOOD}, log=LOG, read_length=250)[0] -check("3 the log's reason for DYS19 (its flank)", by["DYS19"]["note"], - "HipSTR wrote no record for DYS19/DYS394; its log: Aborting genotyping of the locus as the sequence " - "downstream of the repeat is too repetitive for accurate genotyping") -check("3 the log's reason for DYS425 (too few reads)", by["DYS425"]["note"], - "HipSTR wrote no record for DYS425; its log: Skipping locus with too few reads: TOTAL=3, MIN=5") -check("3 a span's reason, found by its one region (DYS450)", - "too few informative reads for stutter training" in by["DYS450"]["note"], True) -check("3 a span longer than --max-str-len (DYS389II at 250 bp reads)", - "exceeds the threshold (164 vs 150)" in by["DYS389II"]["note"], True) -check("3 a region the log names without a reason: the note says so", by["DYS393"]["note"], - "HipSTR wrote no record for DYS393, and its log gives no reason") -check("3 a region the log does not name: the note says so", by["DYS391"]["note"], - "HipSTR wrote no record for DYS391, and its log gives no reason") -check("3 a called marker carries no log line", by["DYS390"]["note"], ".") - -# --- 4. not callable --------------------------------------------------------------------- -by = run({"DYS452": GOOD})[0] -for m in ("DYS464a", "DYS464d", "DYF395S1a", "DYS385b", "CDYa"): - check(f"4 {m}: not callable, multi-copy", (by[m]["confidence"], "multi-copy" in by[m]["note"], by[m]["count"]), - ("not callable", True, "")) -check("4 DYS439 (no region in the marker set): not callable", (by["DYS439"]["confidence"], "no region" in by["DYS439"]["note"]), - ("not callable", True)) -check("4 DYS452 (155 bp) at 150 bp reads: not callable even with a call", - (by["DYS452"]["confidence"], by["DYS452"]["count"]), ("not callable", "")) -by = run({"DYS452": GOOD}, read_length=250)[0] -check("4 DYS452 at 250 bp reads: called", (by["DYS452"]["confidence"], by["DYS452"]["count"]), ("high", "31")) - -# --- 5. the CSV ---------------------------------------------------------------------------- -_by, rows, crow, _o = run({"DYS393": good("4")}) -check("5 CSV header", crow[0], ["marker", "value", "confidence"]) -check("5 CSV: a row per marker in the panel's order", [r[0] for r in crow[1:]], names) -check("5 CSV: DYS393's row", crow[1], ["DYS393", "13", "high"]) -check("5 CSV: a multi-copy row has no value", crow[5], ["DYS385a", "", "not callable"]) -check("5 TSV: rows in the panel's order", [r["marker"] for r in rows], names) - -# --- 6. damaged input -------------------------------------------------------------------- -p = run({"DYS393": GOOD}, samples=("S1", "S2"), want_rc=1) -check("6 two samples: the error says so", "2 samples" in p.stderr, True) -bad = os.path.join(TMP, "bad_panel.tsv") -with open(PANEL) as f, open(bad, "w") as g: - lines = f.readlines() - head = [x for x in lines if x.startswith("#") or x.startswith("order\t")] - body = [x for x in lines if x not in head] - g.writelines(head + [body[1], body[0]] + body[2:]) -p = run({}, panel=bad, want_rc=1) -check("6 a panel out of order stops", "order column" in p.stderr, True) - -# --- 7. the reports ----------------------------------------------------------------------- -import collect_summary as cs # noqa: E402 -import render_report as rr # noqa: E402 - -class Unreadable(Exception): - """A section og summary collect could not read (its state is unreadable).""" - - -def sec(d, s, key): - """(source path, data) of section KEY of the summary og summary collect writes - for folder D: (None, {}) without its file; Unreadable when it cannot be read.""" - x = cs.collect(s, d)["sections"][key] - if x["state"] == "unreadable": - raise Unreadable(x["note"]) - return (os.path.join(d, x["source"]) if x["source"] else None), x["data"] - - - -sd = os.path.join(TMP, "sample") -os.makedirs(os.path.join(sd, "y_str")) -v = vcf({"DYS393": good("4"), "DYS390": ("0", "0", "0.6", "20", "0", "0")}) -subprocess.run([OG, "y", "str-report", "--panel", PANEL, "--vcf", v, "--tsv", - os.path.join(sd, "y_str", "S1_y_str.tsv"), "--csv", os.path.join(sd, "y_str", "S1_y_str.csv")], - check=True, capture_output=True) -path, data = sec(sd, "S1", "y_str") -check("7 collect_summary finds the table", os.path.basename(path or ""), "S1_y_str.tsv") -check("7 counts per confidence", (data["high"], data["low"], data["not_callable"], data["no_call"]), - (1, 1, sum(1 for r in panel if r["kind"] in ("multicopy", "absent")) + 3, # + DYS452, DYS389II, DYS448: too long - 111 - 2 - sum(1 for r in panel if r["kind"] in ("multicopy", "absent")) - 3)) -check("7 the markers keep their order", [x["marker"] for x in data["markers"]], names) -check("7 a sample without the step has no section", sec(TMP, "nobody", "y_str"), (None, {})) -html = "\n".join(rr.ystr_body(data)) -check("7 the card lists DYS393 at 13", "DYS39313high" in html, True) -check("7 the card lists the low call too", "DYS39024low" in html, True) -check("7 and leaves out markers without a count", "DYS464a" in html, False) -check("7 the card carries the convention note", "fixed number of repeats" in html, True) -with open(os.path.join(sd, "y_str", "S1_y_str.tsv")) as f: - t = f.read().replace("\thigh\t", "\tgreat\t", 1) -with open(os.path.join(sd, "y_str", "S1_y_str.tsv"), "w") as f: - f.write(t) -try: - sec(sd, "S1", "y_str") - check("7 an unknown confidence is unreadable", "read", "Unreadable") -except Unreadable: - check("7 an unknown confidence is unreadable", "Unreadable", "Unreadable") -try: - page = rr.html_report(cs.collect("S1", sd)) -except (KeyError, TypeError) as e: - page = f"{type(e).__name__}: {e}" -check("7 the HTML report renders an unreadable Y-STR table as such", - "

    Y-STR Markers (HipSTR)

    " in page and "Unreadable" in page, True) - -if FAILS: - print(f"{len(FAILS)} check(s) failed") - sys.exit(1) -print("all checks passed")