diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index d203d4668..55d40cb8b 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -480,6 +480,38 @@ jobs: - name: cargo test run: cargo test --workspace --locked --no-fail-fast + # The browser checks of og-browser (tests/browser/README.md) ran in + # cargo test above, in the runner's Chrome. Two controls: a page error + # planted in the clean page must fail its test, and with no Chrome the + # tests must fail in CI and only skip outside it. + - name: The browser checks fail on a planted page error, and without Chrome in CI + if: ${{ !cancelled() }} + run: | + log="${RUNNER_TEMP}/og-browser-red.log" + if cargo test --locked --no-fail-fast -p og-browser --features planted-page-error --test cdp > "$log" 2>&1; then + cat "$log" + echo "ERROR: the clean page's test passed with a planted page error" + exit 1 + fi + cat "$log" + grep -E 'test a_clean_page_loads_with_no_page_error \.\.\. FAILED' "$log" + grep -E "TypeError: .*\bplanted\b" "$log" + echo "Control: a planted page error failed the clean page's test." + if CHROME=/nonexistent cargo test --locked --no-fail-fast -p og-browser --test cdp > "$log" 2>&1; then + cat "$log" + echo "ERROR: the browser checks passed in CI without Chrome" + exit 1 + fi + cat "$log" + grep -F "[FAIL] no Chrome or Chromium to load the test page" "$log" + grep -E 'test a_clean_page_loads_with_no_page_error \.\.\. FAILED' "$log" + grep -E 'test a_planted_exception_and_console_error_are_reported \.\.\. FAILED' "$log" + echo "Control: without Chrome the browser checks fail in CI." + CHROME=/nonexistent CI='' cargo test --locked -p og-browser --test cdp -- --nocapture > "$log" 2>&1 || { cat "$log"; exit 1; } + cat "$log" + test "$(grep -c -F '[SKIP] no Chrome or Chromium to load the test page' "$log")" -eq 2 + echo "Outside CI, without Chrome the browser checks skip." + # The planned-verb test (og/tests/port_scaffold.rs) must fail when a # later merge drops a verb's arm. - name: The planned-verb test fails when a verb is dropped diff --git a/CHANGELOG.md b/CHANGELOG.md index c9dcf0c9a..e478c7c67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- `og report index build --summary summary.json [--sample-dir DIR] -o _search_index.json` writes the run's search index in Rust. `og report index query --index FILE QUERY` answers one query with the rule of the live site's search box (`assets/live/plugins/search.js`); a cargo test runs that script under node against the Rust rule. The file is the one `bin/results_index.py` writes: the same terms, sections, anchors, counts and mentions, and the same answer to each query. No step calls the verb yet: step 24 and the Nextflow report still write the index with `bin/results_index.py`. One field is not written yet: `veil`, which tags an item that names a result a child's page keeps closed. It needs the live site's findings, which are not ported yet. +- Step 37 and the Nextflow `Y_HAPLOGROUP` process run Yleaf through its own command line. `bin/yleaf_run.py`, our Python launcher that patched Yleaf at run time (its data folder, its reference, its samtools calls and its process pools), is gone. Yleaf 4.1.7 has the options the launcher stood in for: `-rp` reads the pileup the step makes beside the chrY BAM, `--ref-fasta` names the reference so it downloads none, and `--no-update-check` keeps it offline. Yleaf now predicts with the marker table and tree its image ships; the image test checks they are the files of the archive `setup.sh --yleaf-data` installs, whose positions the pileup is made at, and that Yleaf still has those options, that pileup file and the samtools flags the pileup uses. When no read falls on any marker, Yleaf stops before it predicts, so `og y yleaf-no-reads` writes the row it gives for too few markers (`Hg` NA, the BAM's mapped reads), as the launcher did. On the fixture's chrY slice the prediction table and every file Yleaf writes are the same, byte for byte, as with the launcher, and so is the empty-pileup row. No option, output file or column changes. Step 37 now stops with a clear message when `REF_FASTA` does not end in `.fa`, `.fasta` or `.fna` (a compressed FASTA, say): Yleaf's `--ref-fasta` takes no other name, and the Nextflow `--reference` already asks for a plain FASTA. `setup.sh` installs `.fasta`. - A new page, [User journeys](docs/user-journeys.md), records where a new user gets stuck on each surface: the docs, the demo, the shell and the local site. It gives the outcome of each journey in the first test-as-user run (2026-11-17, commit `65095e97`), the fix for each finding, and what the run could not reach. A fix that lands has a journey to walk again. - VCF intake now refuses a record with GT after another FORMAT key, such as `DP:GT:GQ`, before a reader can mistake depth for the genotype. The check exits 2 and names the sample, file, first offending position and FORMAT field. It tells users to reorder the keys and sample values together or get a corrected VCF. Records without GT remain allowed by this ordering check. Valid input and output formats are unchanged. Part of #411 (clin-37). - The `archaic-same-person` job of `public-evidence.yml` puts step 43's Neanderthal totals beside Chen et al.'s released calls with `og archaic compare` and `og archaic genomes`, in Rust. They replace `bin/archaic_compare.py`, which is gone with its test, `tests/test_archaic_compare.py`; those checks are now cargo tests in og-ancestry. The tables are the same, byte for byte, on 20 parity cases and on the inputs of the last green run of the job (NA12878, chromosomes 19 to 22). No step of the pipeline runs them, and no option or output changes. What changes on purpose: a missing file, an unreadable number or an empty spans file stops with an `ERROR:` line and exit 1 instead of a Python traceback; an interval with a negative start or an end before its start stops the command, where the Python counted a negative length; and a `--mine` value without `=` is a usage error (exit 2). diff --git a/assets/live/plugins/search.js b/assets/live/plugins/search.js index 5c7719d34..c6b105492 100644 --- a/assets/live/plugins/search.js +++ b/assets/live/plugins/search.js @@ -4,10 +4,11 @@ // filters that index here in the browser: no request per keystroke, so the query is in no // request and no server log. A hit lists each section that names the term, with a link to it in // the report and, for a mention with a locus, to the reads in the genome viewer. The query rule -// is bin/results_index.py's query(): an rsID, a gene symbol (a prefix also lists the terms that -// start with it), or a position, which the index cannot answer (scripts/owngenome-lookup.sh can). +// is og report index's query() (rust/crates/og-render/src/index.rs) and bin/results_index.py's: +// an rsID, a gene symbol (a prefix also lists the terms that start with it, in sorted order), or +// a position, which the index cannot answer (scripts/owngenome-lookup.sh can). // The box sits in the page header; on a person's page a hit opens its card (#s/
). -// tests/test_results_index.py runs match() under node against the Python rule. +// rust/crates/og-render/tests/index_search.rs runs match() under node against the Rust rule. (function (root) { 'use strict'; var SUGGESTIONS = 20; diff --git a/bin/yleaf_run.py b/bin/yleaf_run.py deleted file mode 100755 index 614372024..000000000 --- a/bin/yleaf_run.py +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env python3 -"""yleaf_run.py: run Yleaf 4.1.6 (YLEAF_IMAGE) offline, on a pileup made beside it. - -Step 37 makes Yleaf's pileup and the chrY BAM in one SAMTOOLS_IMAGE run, -and the current tree's pileup (og y tree-place) in a second one; the -Nextflow Y_PILEUP task makes all three in one task. Yleaf then reads its pileup from a file instead of running -samtools itself, with its data folder (marker positions, haplogroup trees) -set to the copy `setup.sh --yleaf-data` installs from the release archive as -reference/yleaf-/data, the folder that also holds the current tree. -Yleaf also downloads the whole hg38 FASTA when its read-only config names -none, and checks GitHub for a newer release. So it runs in three parts: - - og y yleaf-positions --data DATA OUT - (YLEAF_IMAGE) write Yleaf's GRCh38 marker positions as the "chrYpos" - list `samtools mpileup -l` reads (rust/crates/og-reports/src/yleaf.rs; - it reads DATA/hg38/new_positions.txt, the file named here by - yleaf_constants, so a Yleaf release that renames it stops that step) - samtools idxstats BAM > IDXSTATS; samtools mpileup -l OUT -ABQ20q1 BAM > PILEUP - (SAMTOOLS_IMAGE) the two commands Yleaf would run, with its default - quality threshold of 20 - yleaf_run.py predict --data DATA --bam BAM --reference FASTA --idxstats IDXSTATS --pileup PILEUP --out DIR - (YLEAF_IMAGE) Yleaf's own flow on its default tree (YFull v14.01), - with its data folder set to DATA, its reference constant set to FASTA - (a BAM never needs the sequence), no update check, each samtools call - served from those two files (a pileup made with other flags than the - ones Yleaf asks for stops the run), and its multiprocessing pools - replaced by a serial map, so every call runs in this process, where - the served samtools calls are. - -DIR/hg_prediction.hg is Yleaf's prediction table (Hg is NA when too few -markers had reads). Standard library plus what Yleaf itself imports. -""" -import multiprocessing -import shutil -import sys -from pathlib import Path - -QUALITY = 20 # Yleaf's -q default; the mpileup above uses the same -FLAGS = f"-ABQ{QUALITY}q1" # what Yleaf 4 gives samtools mpileup at that quality - - -class SerialPool: - """multiprocessing.Pool in this process: the initializer runs here, once.""" - - def __init__(self, processes=None, initializer=None, initargs=(), **kwargs): - if initializer is not None: - initializer(*initargs) - - def __enter__(self): - return self - - def __exit__(self, *args): - return False - - def map(self, fn, items): - return list(map(fn, items)) - - def imap(self, fn, items): - return map(fn, items) - - imap_unordered = imap - - -def use_data(data): - """Point Yleaf's constants at DATA (the yleaf/data folder of its release).""" - from yleaf import yleaf_constants as c - # Absolute: Yleaf opens DATA_FOLDER / HG_PREDICTION_FOLDER / tree.json. - # An absolute HG_PREDICTION_FOLDER replaces DATA_FOLDER in that join; a - # relative one gives data/data/... (a Nextflow task stages it as "data"). - data = Path(data).absolute() - for need in (data / c.HG38 / c.NEW_POSITION_FILE, data / "hg_prediction_tables" / c.TREE_FILE): - if not need.is_file(): - sys.exit(f"ERROR: {need} not found: install Yleaf's data with scripts/setup.sh --yleaf-data ") - c.DATA_FOLDER = data - c.HG_PREDICTION_FOLDER = data / "hg_prediction_tables" - return c - - -def predict(argv): - import argparse - ap = argparse.ArgumentParser(prog="yleaf_run.py predict") - for a in ("--data", "--bam", "--reference", "--idxstats", "--pileup", "--out"): - ap.add_argument(a, required=True) - a = ap.parse_args(argv) - multiprocessing.Pool = SerialPool - yleaf_constants = use_data(a.data) - yleaf_constants.HG38_FULL_GENOME = Path(a.reference) - if Path(a.pileup).stat().st_size == 0: - # No read at any marker: Yleaf's pandas read of an empty pileup fails - # before it predicts, so write the prediction it gives for too few - # markers (Hg NA) here, with the BAM's mapped reads from idxstats. - reads = 0 - with open(a.idxstats) as f: - for line in f: - cols = line.split("\t") - if len(cols) > 2 and cols[2].strip().isdigit(): - reads += int(cols[2]) - Path(a.out).mkdir(parents=True, exist_ok=True) - (Path(a.out) / "hg_prediction.hg").write_text( - "Sample_name\tHg\tHg_marker\tTotal_reads\tValid_markers\tQC-score\tQC-1\tQC-2\tQC-3\n" - f"{Path(a.bam).name.rsplit('.', 1)[0]}\tNA\t\t{reads}\t0\tNA\tNA\tNA\tNA\n") - print("No read at any Y marker: Hg NA (insufficient markers)") - return - from yleaf import Yleaf - - def serve(cmd, stdout_location=None): - if cmd.startswith("samtools idxstats "): - with open(a.idxstats) as f: - shutil.copyfileobj(f, stdout_location) - stdout_location.flush() - elif cmd.startswith("samtools mpileup ") and " > " in cmd: - if f" {FLAGS} " not in f" {cmd} ": - sys.exit(f"ERROR: Yleaf asks for samtools mpileup with other flags than {FLAGS}, the ones the " - f"pileup was made with: {cmd}") - shutil.copyfile(a.pileup, cmd.rsplit(" > ", 1)[1].strip().strip('"')) - elif cmd.startswith("samtools index "): - pass # the BAM is indexed; Yleaf only asks when it finds no .bai - else: - sys.exit(f"ERROR: Yleaf asked for a command this launcher does not serve: {cmd}") - - Yleaf.call_command = serve - sys.argv = ["Yleaf", "-bam", a.bam, "-o", a.out, "-rg", "hg38", "-tree", "yfull", "-force", "-t", "1", - "-q", str(QUALITY), "--no-update-check"] - Yleaf.main() - pred = Path(a.out) / "hg_prediction.hg" - if not pred.is_file() or len(pred.read_text().splitlines()) < 2: - # Yleaf 4 logs a failed sample as "Skipping" and goes on: its log says why. - sys.exit(f"ERROR: Yleaf wrote no prediction ({pred}); its log is in {a.out}") - - -def main(argv): - if argv[:1] == ["predict"]: - predict(argv[1:]) - else: - sys.exit(__doc__) - - -if __name__ == "__main__": - main(sys.argv[1:]) diff --git a/docs/24-html-report.md b/docs/24-html-report.md index d5551fecb..a46341805 100644 --- a/docs/24-html-report.md +++ b/docs/24-html-report.md @@ -68,7 +68,7 @@ After them, "What this report leaves out" lists the steps not run and what no st - **Links to the step's own files**, relative to the report, when the summary says they exist: the PharmCAT HTML report, the CPIC recommendations per gene, the CPSR HTML report (or the `cpsr/` folder that holds it) and the Exomiser ranking table. The report sits in the sample folder beside each step's folder, in the bash layout and in a Nextflow `--outdir`. MultiQC and the couple report are not linked: the summary does not record them, and the couple report sits in the family folder. - **Nothing is fetched.** A Content-Security-Policy (`default-src 'none'; img-src data:; style-src 'unsafe-inline'`) lets the browser load nothing but the file itself and its inline pictures, and there is no JavaScript. The only addresses off this computer are the haplogroup links (FamilyTreeDNA, YFull, the docs) and the project link in the footer, each marked "opens a website"; the FamilyTreeDNA and YFull ones add that the address carries the haplogroup. The genome viewer links go to `127.0.0.1`, this computer. - **A Viewer column on every findings table.** The ClinVar hits, the secondary-findings genes, the mitochondrial variants, the known deep intronic sites, the cDNA scan, the AnnotSV class 4 and 5 SVs and the calls moved to class 3, slivar's pairs in trans, the mosaic alterations and Exomiser's ranked variants each end with a Viewer column: one link per row, to that row's position (a range for an SV, a region or a pair). The link opens the reads there in the [genome viewer](viewer.md) once `scripts/owngenome-view.sh ` runs; a note under the table says so. The column is new, so every other cell keeps its text; it replaces the single line of links that sat under three of these tables. A row with no position has an empty Viewer cell. On the [live site](live-site.md) the served report points the same links at the site's own viewer of the run. -- **Genes and rsIDs in these results.** The closing list is built by `bin/results_index.py`. It holds every gene symbol and rsID that a section in state complete or stale names in `summary.json`, by the same rule `scripts/owngenome-lookup.sh` uses for "in the reports" (`names_of` in `bin/genome_lookup.py`, which `og lookup` ports in `rust/crates/og-lookup`; `tests/test_results_index.py` checks that the two agree). A section's locus name counts as a term too, when it holds an upper-case letter: a repeat expansion (HTT, FMR1, CFTR_TG), a Locityper locus (HLA-DRB1, FCGR) and a copy-number locus are indexed by their `locus` field, so a term is not always a gene symbol. An expansion record with no REPID or VARID takes its position as its locus (`chrX:147912050`), and that counts as a term only when it holds an upper-case letter (a chrX, chrY or chrM position). It adds the genes of the clinical filter (step 23) and variant prioritization (step 31), whose cards only count them: from their summary tables on the bash path, from the VCFs on the Nextflow path. The two give the same genes when a variant's most severe consequence is on the transcript the report ranks first (MANE, else canonical); the step 31 table keeps the gene of the most severe consequence, so where an overlapping gene's transcript carries the worse one, the two forms name different genes. A ClinVar hit's rsIDs come from step 13's VEP output (`Existing_variation`), because the hits file's ID column holds ClinVar's own id; without step 13 a hit has none. The Nextflow `HTML_REPORT` process does not get step 13's VEP output, so its index has no ClinVar rsIDs; step 24, which `run-all.sh` runs over the output folder afterwards, adds them. A failed, missing or unreadable section adds nothing. The browser's find (Ctrl+F) searches the list; there is still no script. Apart from those locus names a position is not in it, and the search does not answer a position: ask `scripts/owngenome-lookup.sh` for a genotype ([genotype lookup](lookup.md)). +- **Genes and rsIDs in these results.** The closing list is built by `bin/results_index.py`. It holds every gene symbol and rsID that a section in state complete or stale names in `summary.json`, by the same rule `scripts/owngenome-lookup.sh` uses for "in the reports" (`names_of` in `bin/genome_lookup.py`, which `og lookup` ports in `rust/crates/og-lookup`; `og report index` ports the list itself, and `rust/crates/og-render/tests/index_search.rs` checks the page search against it). A section's locus name counts as a term too, when it holds an upper-case letter: a repeat expansion (HTT, FMR1, CFTR_TG), a Locityper locus (HLA-DRB1, FCGR) and a copy-number locus are indexed by their `locus` field, so a term is not always a gene symbol. An expansion record with no REPID or VARID takes its position as its locus (`chrX:147912050`), and that counts as a term only when it holds an upper-case letter (a chrX, chrY or chrM position). It adds the genes of the clinical filter (step 23) and variant prioritization (step 31), whose cards only count them: from their summary tables on the bash path, from the VCFs on the Nextflow path. The two give the same genes when a variant's most severe consequence is on the transcript the report ranks first (MANE, else canonical); the step 31 table keeps the gene of the most severe consequence, so where an overlapping gene's transcript carries the worse one, the two forms name different genes. A ClinVar hit's rsIDs come from step 13's VEP output (`Existing_variation`), because the hits file's ID column holds ClinVar's own id; without step 13 a hit has none. The Nextflow `HTML_REPORT` process does not get step 13's VEP output, so its index has no ClinVar rsIDs; step 24, which `run-all.sh` runs over the output folder afterwards, adds them. A failed, missing or unreadable section adds nothing. The browser's find (Ctrl+F) searches the list; there is still no script. Apart from those locus names a position is not in it, and the search does not answer a position: ask `scripts/owngenome-lookup.sh` for a genotype ([genotype lookup](lookup.md)). - **Dark on screen, light on paper.** - **Print one section.** Open a section from the contents (or its "link to this section"), then print: only that section prints, with the header (sample, generation date, pipeline commit, ClinVar file date, ClinVar in CPSR's bundle, HLA database), the list of what is not assessed and the disclaimer. A chapter prints the same way from its title in the contents. Without an anchor in the address the whole report prints, as it does in a browser without CSS `:has()`. diff --git a/docs/37-y-haplogroup.md b/docs/37-y-haplogroup.md index 6b3cadbb0..55a65dee7 100644 --- a/docs/37-y-haplogroup.md +++ b/docs/37-y-haplogroup.md @@ -10,7 +10,7 @@ The mitochondrial haplogroup (step 12) traces the maternal line only. The Y chro ## Tool -- **Yleaf** 4.1.6 (Ralf et al., Mol Biol Evol 2018; GPL-3.0), Erasmus MC. It reads the BAM's pileup at its Y markers and predicts the haplogroup they support on its default tree, YFull v14.01 (about 183,000 marker positions on GRCh38). +- **Yleaf** 4.1.7 (Ralf et al., Mol Biol Evol 2018; GPL-3.0), Erasmus MC. It reads the BAM's pileup at its Y markers and predicts the haplogroup they support on its default tree, YFull v14.01 (about 183,000 marker positions on GRCh38). - **The current YFull YTree** (release 14.06.0 of 22 September 2026, CC BY 4.0, [github.com/YFullTeam/YTree](https://github.com/YFullTeam/YTree)), walked by `og y tree-place` (`rust/crates/og-reports/src/ytree.rs`). It is newer than Yleaf's v14.01, and the second position table below places SNPs Yleaf's own table does not have, so the branch it reports can sit below the one Yleaf reports. It also writes the path from the root with the calls of every level. The SNPs on YFull's tree have names only; their GRCh38 positions and bases come from two tables of Yleaf 4.1.6 (GPL-3.0): `ftdna_positions_hg38.txt` (made for FamilyTreeDNA's tree, the widest of its tables) first, then `new_positions.txt` (the one Yleaf 4 uses for its YFull v14 tree). SNP names are shared between the trees, so a name is looked up as it stands. Together they place 85% of the tree's 459,468 SNPs (76% from the first table alone), counted on 2026-10-25. @@ -20,7 +20,7 @@ The step reports both results, each with the tree it comes from. Both reports (s - `YLEAF_IMAGE`, and `SAMTOOLS_IMAGE` for the pileup -The step runs in three parts (`bin/yleaf_run.py` says why): Yleaf's marker positions, written by `og y yleaf-positions` in its image, `samtools idxstats` and `samtools mpileup -l -ABQ20q1` (Yleaf 4's own flags at its default quality 20) in `SAMTOOLS_IMAGE`, in the same run as the chrY BAM (step 37 makes the current tree's pileup in a second run; Nextflow's `Y_PILEUP` task makes all three in one task), then Yleaf on that pileup. If a later Yleaf asks samtools for other flags, the launcher stops instead of handing it a pileup made another way. +The step runs in three parts: Yleaf's marker positions, written by `og y yleaf-positions` in its image, `samtools idxstats` and `samtools mpileup -l -ABQ20q1` (Yleaf 4's own flags at its default quality 20) in `SAMTOOLS_IMAGE`, in the same run as the chrY BAM (step 37 makes the current tree's pileup in a second run; Nextflow's `Y_PILEUP` task makes all three in one task), then Yleaf's own command line on that pileup. The step puts the pileup where Yleaf keeps its own, `//temp_haplogroup_pileup.pu`, and runs `Yleaf -bam -rg hg38 --ref-fasta -tree yfull -q 20 -t 1 -rp -force --no-update-check`; `-rp` makes Yleaf read that file instead of running samtools. The image test (the `YLEAF_IMAGE` row of `tests/smoke/commands.tsv`) checks that the image still has these options, that pileup file name and those samtools flags, and that its marker table and tree are the ones in the archive `setup.sh --yleaf-data` installs. Yleaf stops on an empty pileup (no read at any marker) and writes no row; the step then writes the row Yleaf gives for too few markers (`Hg` NA) with `og y yleaf-no-reads`. Pinned in `versions.env`; [Image versions](versions.md) lists the current tag. @@ -43,7 +43,7 @@ In the Nextflow pipeline (`y_haplogroup` in `--tools`) the process runs for the With `run-all.sh`: `TOOLS=...,y_haplogroup` (it is opt-in, so a default run lists it as skipped; without the data it is skipped with the reason). -Yleaf downloads the whole hg38 FASTA on its first run unless its config file names one, and the image's config is read-only. The launcher points Yleaf at the pipeline's reference before it starts (a BAM never needs the sequence itself) and turns off its check for a newer release, so nothing is downloaded and every container runs without a network. +Yleaf downloads the whole hg38 FASTA on its first run unless it is given one. The step passes the pipeline's reference with `--ref-fasta` (Yleaf checks its chrY length against hg38 in the `.fai`; a BAM never needs the sequence itself) and `--no-update-check` turns off its check for a newer release, so nothing is downloaded and every container runs without a network. `--ref-fasta` takes only a file ending in `.fa`, `.fasta` or `.fna`, as `setup.sh` installs it, so step 37 stops with a message on any other `REF_FASTA` (a compressed FASTA, say). ## Output diff --git a/docs/rust.md b/docs/rust.md index 8505e841e..bda0a8ffb 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -31,7 +31,7 @@ rust/ crates/og-lookup/ the genotype lookup of scripts/owngenome-lookup.sh: og lookup query, resolve, report crates/og-fresh/ og dev freshness: the monthly freshness issue's upstream lookups, typed answers crates/og-recheck/ og recheck: newer databases since a run, the rerun and the ClinVar lock - crates/og-render/ og report render|index|charts: the reports from summary.json (render: the frame, QC and variants so far; no caller yet) + crates/og-render/ og report render|index|charts: the reports from summary.json (render: the frame, QC and variants so far; index: ported; neither has a caller yet) crates/og-live/ og live serve|results|family|demo, og view serve: the live site (not ported yet) crates/og-couple/ og couple screen|genes: step 39's couple screen (not ported yet) crates/og-admix/ what the fine-ancestry engines share: panel, map, ref.geno, RNG, simulator; painting and projection (not ported yet) @@ -42,7 +42,7 @@ tests/rust/ the differential test against CPython, the image test Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og cdna check|scan` (were `clip_check.py` and `cdna_scan.py`), `og comphet phase` (was `comphet_phase.py`), `og bam contigs`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Step 05 builds AnnotSV's input with `og sv add-callers` and checks its class 4 and 5 calls with `og sv postfilter` (were `cnvpytor_to_vcf.py` and `annotsv_postfilter.py`); both apply one read-depth rule for long Delly calls (`og-steps::sv`). Step 39's couple screen applies the same rule. `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on) and `og prs table` (was `prs-table`). Step 33 has `og qc sample` (was `sample-qc`), and step 20 has `og mito annotate|deletions|copies` (were `mito-annotate`, `mito-deletions` and `mito-copies`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (were the commands of `chip_report.py`). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, `copy_number.py` (the catalog tests materialise its pinned historical version as a differential oracle) and the marker reader of `traits_report.py`. Step 41 runs `og bloodtype report` (was `bloodtype_report.py`), which reads the marker sites through the VCF's index. Step 26b's six commands are `og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report`. Step 43 runs `og archaic table|strict|genotypes|report` (were the four commands of `archaic_report.py`), and the archaic jobs of `public-evidence.yml` run `og archaic compare|genomes` (were the two commands of `archaic_compare.py`), and step 44 runs `og ancient select|lift|person-geno|align|report` (were the five commands of `ancient_report.py`); `og ancient lift` reads UCSC's chain file with `og-intervals::chain`. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report`, `segments` and `ibd` (were `family_report.py`, `family_segments.py` and `ibd_report.py`), and index the phased family VCF with `og vcf index` (was a `pysam.tabix_index` one-liner). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. Step 39 and the Nextflow `COUPLE_CARRIER` process run `og couple screen`, and `og couple genes` rebuilds the carrier list (were the two commands of `couple_report.py`); the screen computes every couple before it writes a file. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. Step 37 places the sample on the current YFull YTree with `og y tree-positions` and `og y tree-place` (were the two commands of `y_tree_place.py`), lists its private SNPs with `og y private-snps` (was `y_private_snps.py`) and writes Yleaf's marker positions with `og y yleaf-positions` (was `yleaf_run.py positions`; `yleaf_run.py predict` stays Python, as glue inside Yleaf's own code). Steps 08, 32 and 36 run `og pgx hla-concordance`, `og pgx pypgx-summary` and `og pgx outside-calls` (were the three commands of `pgx_outside_calls.py`), and step 35b runs `og paraphase check-bam|table` (were `paraphase_report.py`). Both write their two output files together or not at all. `scripts/owngenome-lookup.sh` runs `og lookup query` ([genotype lookup](lookup.md)), which answers every query of the command line in one process; `og lookup resolve` and `og lookup report` are its two halves, with the command lines of `genome_lookup.py`, which keeps only the query reading and the names rule that `results_index.py` imports. `og recheck` replaced `owngenome_recheck.py` ([Newer databases since a run](recheck.md)); the file stays one release as a shim that runs it, for crontabs. `og dev freshness` replaced `scripts/ci/freshness.py`, the monthly freshness issue of `freshness.yml`. `og dev build cn-intervals|callability|sv-rule-regions` rebuild the committed tables `bin/cn_loci.tsv` (its interval rows), `assets/callability_genes.tsv` and `assets/sv_rule_regions.tsv`; they replaced `scripts/ci/build-cn-intervals.py` and the Python inside `build-callability.sh` and `build-sv-rule-regions.sh`. Those scripts keep the downloads (`curl` through `scripts/ci/lib/fetch-pinned.sh`, each file checked against its sha256) and hand `og` the folder, so `og dev build` reads local files only. Both ask the network through the host's `curl` (the config on stdin, so no token reaches the process list), which keeps a TLS stack out of the static binary and its cross-builds. -Every verb the 1.0.0 release still has to port is already registered, each in its own module, as a stub that prints `og : not ported yet` on stderr and exits 2: `og report index|charts`, `og live serve|results|family|demo`, `og view serve`, `og vcf shift-back`, `og setup chrm-shift|revel|roh-af`, `og sv survivor-prep|consensus-check`, `og ancestry-fine freqs|project|paint`, `og longread clair3-model`, `og map svg`, `og dev demo-sample|demo-events|python-gate|everything-check|docs-check|steps-check` and `og dev build roh-af|acmg-lof`. A port fills its module's body and does not edit `og/src/main.rs`, the workspace member list or a crate's module list. `rust/og/tests/port_scaffold.rs` lists these verbs and fails when one has no line in `og --help` or reaches a router's "unknown" error. `og report render` is part way: it writes the frame of both reports and the quality-control and variant sections, leaves the other sections out, and no step calls it yet; `bin/render_report.py` still writes the reports. +Every verb the 1.0.0 release still has to port is already registered, each in its own module, as a stub that prints `og : not ported yet` on stderr and exits 2: `og report charts`, `og live serve|results|family|demo`, `og view serve`, `og vcf shift-back`, `og setup chrm-shift|revel|roh-af`, `og sv survivor-prep|consensus-check`, `og ancestry-fine freqs|project|paint`, `og longread clair3-model`, `og map svg`, `og dev demo-sample|demo-events|python-gate|everything-check|docs-check|steps-check` and `og dev build roh-af|acmg-lof`. A port fills its module's body and does not edit `og/src/main.rs`, the workspace member list or a crate's module list. `rust/og/tests/port_scaffold.rs` lists these verbs and fails when one has no line in `og --help` or reaches a router's "unknown" error. `og report render` is part way: it writes the frame of both reports and the quality-control and variant sections, leaves the other sections out, and no step calls it yet; `bin/render_report.py` still writes the reports. `og ancestry-fine simulate` builds an admixed genome with known truth from held-out people of the fine-ancestry panel: an unphased VCF and a BED of each copy's ancestry. The engines that replace ADMIXTURE and RFMix test against it. It and the parts those engines share (the panel reader, the genetic map, `model/ref.geno` and a seeded xoshiro256** generator) live in `og-admix`. diff --git a/modules/local/y_haplogroup/main.nf b/modules/local/y_haplogroup/main.nf index 8cf601bfb..d23074029 100644 --- a/modules/local/y_haplogroup/main.nf +++ b/modules/local/y_haplogroup/main.nf @@ -6,17 +6,19 @@ workflows/bam_analysis.nf passes the BAMs whose sex is male, the sex INDEXCOV has checked against the reads. - Yleaf (4.1.6, its own tree YFull v14.01) reads its marker tables from - --yleaf_data (setup.sh --yleaf-data), and its pileup is made in the - samtools pass that also writes the chrY BAM, so the work is three - processes, as in scripts/37-y-haplogroup.sh (bin/yleaf_run.py says why): + Yleaf (its own tree YFull v14.01) reads a pileup at its marker positions + (from --yleaf_data, setup.sh --yleaf-data) made in the samtools pass that + also writes the chrY BAM, so the work is three processes, as in + scripts/37-y-haplogroup.sh (its comments say why): Y_POSITIONS Yleaf's GRCh38 marker positions (og y yleaf-positions, in the Yleaf image, once) Y_PILEUP samtools idxstats and the pileup at those positions, and _chrY.bam (+ .bai), the chrY reads only: the file YFull and the Y-DNA projects read. Nothing is uploaded. - Y_HAPLOGROUP Yleaf on that pileup, offline; Hg NA means too few Y - markers had reads to call a haplogroup + Y_HAPLOGROUP Yleaf's own command line on that pileup (-rp), offline; + Hg NA means too few Y markers had reads to call a + haplogroup (og y yleaf-no-reads writes that table when + the pileup is empty, where Yleaf stops) Each process also does the same for the pinned YFull YTree release in /ytree (og y tree-positions and og y tree-place): its SNP positions, a second pileup at them, and the placement, published as _y_tree.txt and @@ -98,8 +100,9 @@ process Y_PILEUP { script: // -ABQ20q1: the flags Yleaf 4 gives samtools mpileup at its default quality 20 - // (bin/yleaf_run.py stops if Yleaf asks for others). The current tree's - // pileup keeps -AQ20q1, the flags og y tree-place was checked with. + // (the YLEAF_IMAGE row of tests/smoke/commands.tsv checks they are still + // Yleaf's). The current tree's pileup keeps -AQ20q1, the flags og y + // tree-place was checked with. """ samtools idxstats ${bam} > ${meta.id}_idxstats.txt samtools mpileup -l ${positions} -ABQ20q1 ${bam} > ${meta.id}_y_pileup.txt @@ -149,8 +152,25 @@ process Y_HAPLOGROUP { script: """ - yleaf_run.py predict --data ${yleaf_data} --bam ${bam} --reference "\$(readlink -f ${reference})" \\ - --idxstats ${idxstats} --pileup ${pileup} --out yleaf + # Yleaf names the sample, and its folder, after the BAM without its + # extension, and reads the pileup it finds there (-rp). --ref-fasta names + # the reference so it downloads none; it never reads a BAM's sequence. + stem=\$(basename ${bam}); stem=\${stem%.*} + if [ -s ${pileup} ]; then + mkdir -p "yleaf/\$stem" + cp ${pileup} "yleaf/\$stem/temp_haplogroup_pileup.pu" + Yleaf -bam ${bam} -o yleaf -rg hg38 --ref-fasta "\$(readlink -f ${reference})" -tree yfull \\ + -q 20 -t 1 -rp -force --no-update-check + rm -f "yleaf/\$stem/temp_haplogroup_pileup.pu" + else + mkdir -p yleaf + og y yleaf-no-reads --bam ${bam} --idxstats ${idxstats} yleaf/hg_prediction.hg + fi + if [ ! -s yleaf/hg_prediction.hg ] || [ "\$(grep -c . yleaf/hg_prediction.hg)" -lt 2 ]; then + # Yleaf logs a failed sample as "Skipping" and goes on: its log says why. + echo "ERROR: Yleaf wrote no prediction (yleaf/hg_prediction.hg); its log is yleaf/run.log" >&2 + exit 1 + fi cp yleaf/hg_prediction.hg ${meta.id}_y_haplogroup.txt if [ -s ${yleaf_data}/ytree/version.txt ]; then og y tree-place --tree-dir ${yleaf_data}/ytree --pileup ${tree_pileup} \\ diff --git a/rust/Cargo.lock b/rust/Cargo.lock index ec8e92fcc..6740f57cd 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -671,6 +671,15 @@ dependencies = [ "pycompat", ] +[[package]] +name = "og-browser" +version = "0.1.0" +dependencies = [ + "indexmap", + "pycompat", + "regex", +] + [[package]] name = "og-chip" version = "0.1.0" @@ -850,6 +859,7 @@ version = "0.1.0" dependencies = [ "indexmap", "og-io", + "og-lookup", "og-summary", "pycompat", ] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 416ee70c9..386ca20d6 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "3" -members = ["crates/pycompat", "crates/og-io", "crates/og-vcf", "crates/og-bam", "crates/og-map", "crates/og-tsv", "crates/og-intervals", "crates/og-dev", "crates/og-qc", "crates/og-clinvar", "crates/og-pgx", "crates/og-filters", "crates/og-nf", "crates/og-runlog", "crates/og-steps", "crates/og-setup", "crates/og-chip", "crates/og-cn", "crates/og-bloodtype", "crates/og-ancestry", "crates/og-pedigree", "crates/og-reports", "crates/og-summary", "crates/og-lookup", "crates/og-fresh", "crates/og-recheck", "crates/og-render", "crates/og-live", "crates/og-couple", "crates/og-admix", "og"] +members = ["crates/pycompat", "crates/og-io", "crates/og-vcf", "crates/og-bam", "crates/og-map", "crates/og-tsv", "crates/og-intervals", "crates/og-dev", "crates/og-qc", "crates/og-clinvar", "crates/og-pgx", "crates/og-filters", "crates/og-nf", "crates/og-runlog", "crates/og-steps", "crates/og-setup", "crates/og-chip", "crates/og-cn", "crates/og-bloodtype", "crates/og-ancestry", "crates/og-pedigree", "crates/og-reports", "crates/og-summary", "crates/og-lookup", "crates/og-fresh", "crates/og-recheck", "crates/og-render", "crates/og-live", "crates/og-couple", "crates/og-admix", "crates/og-browser", "og"] [workspace.package] version = "0.1.0" diff --git a/rust/crates/og-browser/Cargo.toml b/rust/crates/og-browser/Cargo.toml new file mode 100644 index 000000000..a8c3633a5 --- /dev/null +++ b/rust/crates/og-browser/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "og-browser" +description = "Test support: headless Chrome over the DevTools protocol, a static file server and the asset scan, for the browser checks" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +[features] +# A planted page error: the clean page of tests/cdp.rs throws, and that test +# must fail (the red control in .github/workflows/guard.yml). +planted-page-error = [] + +[dependencies] +pycompat.workspace = true +indexmap.workspace = true +regex.workspace = true + +[lints] +workspace = true diff --git a/rust/crates/og-browser/src/assets.rs b/rust/crates/og-browser/src/assets.rs new file mode 100644 index 000000000..3bcdf6472 --- /dev/null +++ b/rust/crates/og-browser/src/assets.rs @@ -0,0 +1,218 @@ +//! The scan of a site's files for anything that would load or run code +//! from outside the files themselves. + +use regex::Regex; +use std::collections::BTreeMap; +use std::path::Path; +use std::sync::LazyLock; + +static URL: LazyLock = + LazyLock::new(|| Regex::new(r#"(?i)(?:https?:)?//[a-z0-9.-]+\.[a-z]{2,}[^\s'"]*"#).unwrap()); +static SCRIPT_TAG: LazyLock = LazyLock::new(|| Regex::new(r"(?i)]*>").unwrap()); +/// One attribute of a tag: its name, then an optional value, quoted or not. +/// Walking a tag with it reads ` src=` inside another attribute's quoted +/// value as part of that value, not as an attribute. +static ATTR: LazyLock = LazyLock::new(|| { + Regex::new(r#"\s+([^\s=>/"']+)(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s>"']+))?"#).unwrap() +}); +static ON_ATTR: LazyLock = LazyLock::new(|| Regex::new(r"(?i)\son[a-z]+\s*=").unwrap()); +static EVAL: LazyLock = + LazyLock::new(|| Regex::new(r"\beval\s*\(|new\s+Function\s*\(").unwrap()); + +/// The SVG namespace name: a URL in the text, never fetched. +const SVG_NS: &str = "http://www.w3.org/2000/svg"; + +/// (file, problem) for each outside URL in any file, each inline +/// `" + ), + [("a.html".into(), "" + ) + .is_empty() + ); + assert!(one("a.js", "fetch('/state.json'); var u = 'a//b';").is_empty()); + } + + #[test] + fn hosts_and_src_are_read_in_any_case() { + assert_eq!( + one("a.js", "fetch('HTTPS://CDN.Example.COM/x.js');"), + [("a.js".into(), "HTTPS://CDN.Example.COM/x.js".into())] + ); + assert_eq!(one("a.css", "url(//cdn.example.com/f.woff)").len(), 1); + // data-src is not src: the script is inline. + assert_eq!( + one("a.html", ""), + [("a.html".into(), "").is_empty()); + assert!(one("a.html", "").is_empty()); + assert!(one("a.html", "").is_empty()); + // " src=" inside another attribute's value is not a src attribute. + assert_eq!( + one( + "a.html", + "" + ), + [("a.html".into(), "\n"; + +#[cfg(not(feature = "planted-page-error"))] +const CLEAN_JS: &str = "document.body.dataset.ok = 'yes'; window.answer = 6 * 7;\n"; +#[cfg(feature = "planted-page-error")] +const CLEAN_JS: &str = "document.body.dataset.ok = 'yes'; window.answer = 6 * 7; null.planted;\n"; + +const ERRORS_JS: &str = "document.body.dataset.ok = 'yes'; console.error('planted console error', 7); \ + setTimeout(function () { throw new Error('planted page error'); }, 0);\n"; + +/// Serve PAGE with `js` as its script, open it and wait until it ran. The +/// scratch folder comes first, so the caller drops it after the browser +/// and the server, also when the test panics. +fn open(name: &str, js: &str) -> Option<(Scratch, StaticServer, Browser)> { + let chrome = chrome_or_skip("load the test page")?; + let d = Scratch::new(Path::new(env!("CARGO_TARGET_TMPDIR")), name); + let site = d.path().join("site"); + fs::create_dir(&site).unwrap(); + fs::write(site.join("index.html"), PAGE).unwrap(); + fs::write(site.join("app.js"), js).unwrap(); + let srv = StaticServer::start(&site, &[("Content-Security-Policy", CSP)]).unwrap(); + let mut b = Browser::launch(&chrome, &d.path().join("chrome")).unwrap(); + b.go(&srv.url("/")).unwrap(); + let ok = b.wait( + "document.body && document.body.dataset.ok", + Duration::from_secs(20), + ); + assert_eq!( + ok, + Some(Value::Str("yes".into())), + "the page did not run its script" + ); + Some((d, srv, b)) +} + +#[test] +fn a_clean_page_loads_with_no_page_error() { + let Some((_d, _srv, mut b)) = open("clean", CLEAN_JS) else { + return; + }; + assert_eq!(b.eval("window.answer").unwrap(), Value::Int(42)); + // Numbers JSON has no form for come back as floats; a BigInt is an error. + assert_eq!(b.eval("1 / 0").unwrap(), Value::Float(f64::INFINITY)); + assert_eq!(b.eval("-1 / 0").unwrap(), Value::Float(f64::NEG_INFINITY)); + assert!(matches!(b.eval("0 / 0").unwrap(), Value::Float(x) if x.is_nan())); + assert!(matches!(b.eval("-0").unwrap(), Value::Float(x) if x == 0.0 && x.is_sign_negative())); + assert!(b.eval("10n").unwrap_err().contains("no JSON form")); + assert_eq!( + b.eval(&format!("{} + '!'", js_str("it's \"quoted\""))) + .unwrap(), + Value::Str("it's \"quoted\"!".into()) + ); + assert_eq!(b.eval("undefined").unwrap(), Value::Null); + // An exception in our own expression is an Err, not a page error. + assert!( + b.eval("nothing.here") + .unwrap_err() + .contains("nothing is not defined") + ); + // wait gives up on a value that never turns truthy. + assert_eq!( + b.wait("document.body.dataset.never", Duration::from_millis(500)), + None + ); + // wait never runs past its timeout, even with a slow expression. + let t = std::time::Instant::now(); + assert_eq!( + b.wait( + "(function () { var e = Date.now() + 3000; while (Date.now() < e) {} return 0; })()", + Duration::from_millis(500) + ), + None + ); + assert!(t.elapsed() < Duration::from_secs(2), "{:?}", t.elapsed()); + std::thread::sleep(Duration::from_millis(300)); + assert_eq!( + b.page_errors(), + Vec::::new(), + "the clean page reported errors" + ); + // A navigation that cannot start is an error; the page stays usable. + let e = b.go("http://127.0.0.1:1/").unwrap_err(); + assert!(e.contains("net::ERR_"), "{e}"); + assert_eq!(b.eval("6 * 7").unwrap(), Value::Int(42)); +} + +#[test] +fn a_planted_exception_and_console_error_are_reported() { + let Some((_d, _srv, mut b)) = open("errors", ERRORS_JS) else { + return; + }; + let mut errors = Vec::new(); + for _ in 0..50 { + errors = b.page_errors(); + if errors.len() >= 2 { + break; + } + std::thread::sleep(Duration::from_millis(100)); + } + assert!( + errors + .iter() + .any(|e| e == "console.error: planted console error 7"), + "{errors:?}" + ); + assert!( + errors + .iter() + .any(|e| e.contains("Error: planted page error")), + "{errors:?}" + ); +} diff --git a/rust/crates/og-browser/tests/live_assets.rs b/rust/crates/og-browser/tests/live_assets.rs new file mode 100644 index 000000000..792ccac2e --- /dev/null +++ b/rust/crates/og-browser/tests/live_assets.rs @@ -0,0 +1,42 @@ +//! assets/live, its plugins included, holds no outside URL, inline script, +//! on* attribute or eval. Moved here from tests/test_owngenome_live.py +//! (check 7), with its two planted mistakes, which the scan must catch. + +use og_browser::{asset_problems, read_assets, repo_root}; + +#[test] +fn live_assets_load_nothing_from_outside() { + let texts = read_assets(&repo_root().join("assets/live")).unwrap(); + for f in ["index.html", "live.js", "plugins/charts.js"] { + assert!( + texts.contains_key(f), + "{f} was not read: {:?}", + texts.keys() + ); + } + assert_eq!(asset_problems(&texts), Vec::<(String, String)>::new()); + + let mut planted = texts.clone(); + let page = planted.get_mut("index.html").unwrap(); + assert!(page.contains("")); + *page = page.replace("", ""); + assert_eq!( + asset_problems(&planted), + [("index.html".to_string(), "") - control("an inline script in index.html", not asset_problems(planted)) - planted = dict(texts, **{"live.js": texts["live.js"] + "\nfetch('https://cdn.example.com/x.js');\n"}) - control("an outside URL in live.js", not asset_problems(planted)) # 8. plugins code, _, body = request(port, "GET", f"/fixture/{RUN}", headers=auth) diff --git a/tests/test_results_index.py b/tests/test_results_index.py deleted file mode 100755 index c8c0b7839..000000000 --- a/tests/test_results_index.py +++ /dev/null @@ -1,436 +0,0 @@ -#!/usr/bin/env python3 -# og-leg: OG_REQUIRED -"""bin/results_index.py, its report appendix and the live search on a synthetic sample. - -The sample (tests/fixtures/results_index/SYNTH, invented) has ClinVar hits in -CFTR and BRCA2, a CPIC table (CYP2C19, CYP2D6), and the tables of steps 23 and -31 in both forms: the summary TSVs the bash steps write and the VCFs the -Nextflow report stages. - -Checks: - 1. every term the index finds in summary.json, `og lookup report` finds - too under "in the reports" (the same names rule, names_of() here and - its port in rust/crates/og-lookup): with a GENE_KEYS entry removed the - index loses CYP2C19 and og still finds it (so the check can fail); a - gene field that holds prose ("VDBP (GC)", "SMN exon 7-8 deletion") - gives its symbols only. Without OG (the og binary) these checks are - skipped with a note; with OG_REQUIRED set too, the test fails; - 2. the step 23 and 31 genes come from the TSV on the bash path and from the - VCF on the Nextflow path, and both give the same entries; with the VCF - reader removed the Nextflow form has none (red); - 3. Existing_variation 'rs123&COSV1' gives rs123 only, and query('RS123') - finds it; - 4. a stale section's entries say stale; a failed or missing one gives none; - a recorded table that cannot be read adds nothing and the report is - still written (red: the reader raises on it); a text-only report never - builds the index (red: an HTML report does); - 5. query('BRCA') lists BRCA1 and BRCA2; a position gets the owngenome-lookup.sh - hint; nothing else is a query; - 6. the HTML report closes with one line per term, each link resolves to a - section of the page, a term that is missing from the index is missing - from the appendix (red), and the page has no script; - 7. the command line builds and queries the index (render_report.py - --search-index writes the same index); - 8. the live search: owngenome_live_search.py registers the file glob and its - state hook; search.js's match() under node answers each fixture query - as query() does, and has no request in it besides the one index fetch. - -Run: OG=bin/og python3 tests/test_results_index.py -""" -import copy -import gzip -import io -import json -import os -import re -import shutil -import subprocess -import sys -import tempfile -from contextlib import redirect_stdout - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -import collect_summary # noqa: E402 -import genome_lookup # noqa: E402 -import owngenome_live_search # noqa: E402 -import render_report # noqa: E402 -import results_index # noqa: E402 - -FIXTURE = os.path.join(REPO, "tests", "fixtures", "results_index", "SYNTH") -SEARCH_JS = os.path.join(REPO, "assets", "live", "plugins", "search.js") -OG = os.environ.get("OG", "") -FAILS = 0 - - -def check(desc, ok, detail=""): - global FAILS - print(f"[{'PASS' if ok else 'FAIL'}] {desc}{'' if ok else ' -- ' + str(detail)[:900]}") - if not ok: - FAILS += 1 - - -def sample(work, form): - """A copy of the fixture with the step 23 and 31 tables in one FORM: - 'tsv' (bash: summary TSVs, VCFs too) or 'vcf' (Nextflow: VCFs only).""" - d = os.path.join(work, form, "SYNTH") - shutil.copytree(FIXTURE, d) - for v in ("clinical/SYNTH_clinical.vcf", "slivar/SYNTH_prioritized.vcf"): - with open(os.path.join(d, v), "rb") as f, gzip.open(os.path.join(d, v + ".gz"), "wb") as g: - g.write(f.read()) - os.remove(os.path.join(d, v)) - if form == "vcf": - os.remove(os.path.join(d, "clinical/SYNTH_clinical_summary.tsv")) - os.remove(os.path.join(d, "slivar/SYNTH_slivar_summary.tsv")) - return d - - -def lookup_says(summary_path, term): - """What `og lookup report` prints under "in the reports" for one gene or - rsID (a site query for an rsID, a gene query for a gene, on an empty - gVCF), as one string.""" - gvcf = os.path.join(os.path.dirname(summary_path), "empty.g.vcf") - with open(gvcf, "w") as f: - f.write("##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n") - kind = "site" if term.startswith("rs") else "gene" - r = subprocess.run([OG, "lookup", "report", "--query", term, "--kind", kind, "--chrom", "chr1", "--start", "1", - "--end", "1", "--gvcf", gvcf, "--summary", summary_path], capture_output=True, text=True) - mark = " in the reports:\n" - if r.returncode != 0 or mark not in r.stdout: - return f"og lookup report failed: {r}" - return r.stdout.split(mark, 1)[1] - - -def table_entries(idx, key): - return {t: [m["text"] for it in items if it["section"] == key for m in it["mentions"]] - for t, items in idx["terms"].items() if any(it["section"] == key for it in items)} - - -def main(): - if not OG: - print("note: OG is not set (OG=bin/og), so the checks against og lookup are skipped") - if os.environ.get("OG_REQUIRED"): - print("FAIL: OG_REQUIRED is set but OG is not") - return 1 - work = tempfile.mkdtemp() - try: - tsv, vcf = sample(work, "tsv"), sample(work, "vcf") - s_tsv, s_vcf = collect_summary.collect("SYNTH", tsv), collect_summary.collect("SYNTH", vcf) - check("fixture: the clinical and slivar sections read the TSV on the bash form and the VCF on the other", - (s_tsv["sections"]["clinical"]["source"], s_vcf["sections"]["clinical"]["source"], - s_tsv["sections"]["slivar"]["source"], s_vcf["sections"]["slivar"]["source"]) - == ("clinical/SYNTH_clinical_summary.tsv", "clinical/SYNTH_clinical.vcf.gz", - "slivar/SYNTH_slivar_summary.tsv", "slivar/SYNTH_prioritized.vcf.gz"), - {k: s_tsv["sections"][k]["source"] for k in ("clinical", "slivar")}) - idx = results_index.build(s_tsv, tsv, render_report.SECTION_OF) - T = idx["terms"] - check("index: the genes and rsIDs of the fixture, genes upper-case, rsIDs lower-case", - sorted(T) == ["BRCA1", "BRCA2", "CFTR", "CYP2C19", "CYP2D6", "rs1000002", "rs123"], sorted(T)) - check("index: shape (schema_version 1, the sample, one item per section with its report anchor)", - idx["schema_version"] == 1 and idx["sample"] == "SYNTH" - and {(it["section"], it["anchor"]) for it in T["BRCA2"]} - == {("clinvar", "clinvar"), ("carrier", "carrier-list"), ("clinical", "clinical-filter"), - ("slivar", "slivar")}, T["BRCA2"]) - check("index: one mention per variant in a section (the secondary findings and the table name chr13:2000)", - [(it["section"], it["count"]) for it in T["BRCA2"]] - == [("clinvar", 1), ("carrier", 1), ("clinical", 1), ("slivar", 1)], T["BRCA2"]) - - # 1. the same mentions as owngenome-lookup.sh - spath = os.path.join(work, "summary.json") - with open(spath, "w") as f: - json.dump(s_tsv, f) - from_summary = sorted({t for t, items in T.items() if any(it["section"] not in results_index.TABLES - for it in items)}) - check("the index finds BRCA2, CFTR, CYP2C19 and CYP2D6 in summary.json", - from_summary == ["BRCA2", "CFTR", "CYP2C19", "CYP2D6"], from_summary) - if OG: - missed = [t for t in from_summary if "none of the reports mentions" in lookup_says(spath, t)] - check("lookup agrees: every term the index finds in summary.json, og lookup finds too", not missed, - (from_summary, missed)) - for t in from_summary: - secs = {s_tsv["sections"][it["section"]]["title"] for it in T[t] - if it["section"] not in results_index.TABLES} - said = lookup_says(spath, t) - check(f"lookup agrees on {t}: the same sections", all(x in said for x in secs), (secs, said)) - # A ClinVar hit's rsids (collect_summary puts them there from VEP): found by both. - withrs = copy.deepcopy(s_tsv) - withrs["sections"]["clinvar"]["data"]["hits"][0]["rsids"] = ["rs7000001"] - rpath = os.path.join(work, "summary-rs.json") - with open(rpath, "w") as f: - json.dump(withrs, f) - ri = results_index.build(withrs, tsv, render_report.SECTION_OF)["terms"] - check("a ClinVar hit's rsids: the index finds it in the ClinVar section", - [it["section"] for it in ri.get("rs7000001", [])] == ["clinvar"], ri.get("rs7000001")) - if OG: - check("a ClinVar hit's rsids: og lookup finds it in the ClinVar section too", - "ClinVar screen (step 06)" in lookup_says(rpath, "rs7000001"), lookup_says(rpath, "rs7000001")) - check("red control: without the rsids the lookup finds nothing for it", - "none of the reports mentions rs7000001" in lookup_says(spath, "rs7000001")) - keep = genome_lookup.GENE_KEYS - genome_lookup.GENE_KEYS = tuple(k for k in keep if k != "gene") - try: - gone = results_index.build(s_tsv, tsv, render_report.SECTION_OF)["terms"] - finally: - genome_lookup.GENE_KEYS = keep - check("red control: without the 'gene' key the index loses CYP2C19", "CYP2C19" not in gone, sorted(gone)) - if OG: - check("red control: og lookup still finds CYP2C19 then, so the agreement check above can fail", - "none of the reports mentions CYP2C19" not in lookup_says(spath, "CYP2C19"), - lookup_says(spath, "CYP2C19")) - - # A gene field that holds prose ("SMN exon 7-8 deletion" from - # paraphase_report.py, "VDBP (GC)" from assets/traits.tsv): only the - # symbols are terms. - prose = copy.deepcopy(s_tsv) - prose["sections"]["mito"].update(state="ok", data={"variants": [{"gene": "SMN exon 7-8 deletion"}, - {"gene": "VDBP (GC)"}]}) - ppath = os.path.join(work, "summary-prose.json") - with open(ppath, "w") as f: - json.dump(prose, f) - pt = set(results_index.build(prose, tsv, render_report.SECTION_OF)["terms"]) - check("a prose gene field: SMN, VDBP and GC are terms; EXON, DELETION, 7-8 and (GC) are not", - {"SMN", "VDBP", "GC"} <= pt and not {"EXON", "DELETION", "7-8", "(GC)", "(GC"} & pt, sorted(pt)) - if OG: - check("a prose gene field: the lookup finds GC too", - "none of the reports mentions GC" not in lookup_says(ppath, "GC"), lookup_says(ppath, "GC")) - - # Sections that name their gene under "locus": the repeat expansions - # (flagged, key loci, the CFTR and UGT1A1 readings) and Locityper. - loc = copy.deepcopy(s_tsv) - loc["sections"]["expansions"].update(state="ok", data={ - "flagged": [{"locus": "HTT", "status": "pathogenic"}], - "key_loci": [{"locus": "FMR1", "repeat_count": "."}], - "readings": [{"locus": "UGT1A1", "repeat": "promoter TA repeat"}]}) - loc["sections"]["locityper"].update(state="ok", data={ - "loci": [{"locus": "FCGR", "region": "chr1:161505430-161678654", "status": "called"}, - {"locus": "HLA-DRB1", "region": "chr6:32484348-32593538", "status": "called"}]}) - lpath = os.path.join(work, "summary-locus.json") - with open(lpath, "w") as f: - json.dump(loc, f) - lt = results_index.build(loc, tsv, render_report.SECTION_OF)["terms"] - want = {"HTT": "expansions", "FMR1": "expansions", "UGT1A1": "expansions", "FCGR": "locityper", - "HLA-DRB1": "locityper"} - check("a 'locus' key: HTT, FMR1 and UGT1A1 are terms of the expansions section, FCGR and HLA-DRB1 of Locityper", - all(sec in [it["section"] for it in lt.get(t, [])] for t, sec in want.items()), - {t: lt.get(t) for t in want}) - check("a 'locus' key: a region is not a term", - not [t for t in lt if t.startswith("CHR")], sorted(lt)) - if OG: - check("a 'locus' key: the lookup finds HTT and FCGR too", - all("none of the reports mentions" not in lookup_says(lpath, t) for t in ("HTT", "FCGR")), - [lookup_says(lpath, t) for t in ("HTT", "FCGR")]) - keep = genome_lookup.GENE_KEYS - genome_lookup.GENE_KEYS = tuple(k for k in keep if k != "locus") - try: - gone = results_index.build(loc, tsv, render_report.SECTION_OF)["terms"] - finally: - genome_lookup.GENE_KEYS = keep - check("red control: without the 'locus' key none of them is a term", - not set(want) & set(gone), sorted(set(want) & set(gone))) - - # 2. bash and Nextflow forms - nf = results_index.build(s_vcf, vcf, render_report.SECTION_OF) - for key in results_index.TABLES: - a, b = table_entries(idx, key), table_entries(nf, key) - check(f"{key}: the TSV form and the VCF form give the same entries", a == b and a, (a, b)) - check("clinical: BRCA2 and BRCA1 from the table; slivar: CFTR, BRCA2 and both rsIDs", - sorted(table_entries(idx, "clinical")) == ["BRCA1", "BRCA2"] - and sorted(table_entries(idx, "slivar")) == ["BRCA2", "CFTR", "rs1000002", "rs123"], - (table_entries(idx, "clinical"), table_entries(idx, "slivar"))) - keep = results_index.vcf_names - results_index.vcf_names = lambda path, gene_tag, with_rsids: [] - try: - red = results_index.build(s_vcf, vcf, render_report.SECTION_OF) - finally: - results_index.vcf_names = keep - check("red control: without the VCF reader the Nextflow form has no table entries, so the check above can fail", - not table_entries(red, "slivar") and table_entries(nf, "slivar"), table_entries(red, "slivar")) - bash_like = copy.deepcopy(nf) - nfc = copy.deepcopy(idx) - for x in (bash_like, nfc): - x["generated_utc"] = None - for items in x["terms"].values(): - for it in items: - it["source"] = None - check("bash and Nextflow: the whole index is the same, the file names aside", bash_like["terms"] == nfc["terms"], - (bash_like["terms"], nfc["terms"])) - - # 3. Existing_variation - check("Existing_variation: 'rs123&COSV1' gives rs123 only", results_index.rs_ids("rs123&COSV1") == ["rs123"] - and "COSV1" not in T and "cosv1" not in T) - r = results_index.query(idx, "RS123") - check("query('RS123') finds rs123 in variant prioritization", r["kind"] == "rsid" and r["term"] == "rs123" - and [it["section"] for it in r["items"]] == ["slivar"] - and r["items"][0]["mentions"][0]["locus"] == "chr7:3000", r) - - # 4. states - st = copy.deepcopy(s_tsv) - st["sections"]["cpic"]["state"] = "stale" - st["sections"]["mito"].update(state="failed", data={"variants": [{"gene": "MT-ND4", "variant": "chrM:11778 G>A"}]}) - st["sections"]["lpa"].update(state="missing", data={"variants": [{"rsid": "rs10455872", "gene": "LPA"}]}) - st["sections"]["haplogroup"].update(state="unreadable", data={"gene": "MT-CYB"}) - si = results_index.build(st, tsv, render_report.SECTION_OF)["terms"] - check("states: a stale section's entries say stale", [it["state"] for it in si["CYP2D6"]] == ["stale"], si["CYP2D6"]) - check("states: a failed, missing or unreadable section gives no entry", - not {"MT-ND4", "LPA", "rs10455872", "MT-CYB"} & set(si), sorted(si)) - ok = copy.deepcopy(st) - ok["sections"]["mito"]["state"] = "ok" - check("red control: the same data in an ok section is indexed", - "MT-ND4" in results_index.build(ok, tsv, render_report.SECTION_OF)["terms"]) - # A recorded table that cannot be read (a truncated gzip) adds nothing, - # and the HTML report is still written. - bad = sample(work, "bad") - os.remove(os.path.join(bad, "clinical/SYNTH_clinical_summary.tsv")) - os.remove(os.path.join(bad, "slivar/SYNTH_slivar_summary.tsv")) - gz = os.path.join(bad, "slivar/SYNTH_prioritized.vcf.gz") - with open(gz, "rb") as f: - head = f.read(40) - with open(gz, "wb") as f: - f.write(head) - try: - results_index.slivar_names(gz) - raises = False - except (OSError, EOFError): - raises = True - check("red control: the truncated table makes the slivar reader raise", raises) - # The summary of the readable run (slivar ok, its VCF recorded), the - # tables of the damaged copy: the table broke after step 31 ran. - sb = os.path.join(work, "sb.json") - with open(sb, "w") as f: - json.dump(s_vcf, f) - with redirect_stdout(io.StringIO()): - rc = render_report.main(["--summary", sb, "--sample-dir", bad, "-o", os.path.join(work, "rb.html"), - "--search-index", os.path.join(work, "rb.json")]) - with open(os.path.join(work, "rb.json")) as f: - bi = json.load(f) - check("an unreadable table: the report and the index are written, the table adds no entry", - rc == 0 and os.path.isfile(os.path.join(work, "rb.html")) and not table_entries(bi, "slivar") - and table_entries(bi, "clinical"), (rc, table_entries(bi, "slivar"))) - # A text-only run never builds the index (it would read the tables). - keep_build, calls = results_index.build, [] - results_index.build = lambda *a, **k: calls.append(1) or keep_build(*a, **k) - try: - with redirect_stdout(io.StringIO()): - render_report.main(["--summary", spath, "-o", os.path.join(work, "t.txt")]) - txt_calls = len(calls) - with redirect_stdout(io.StringIO()): - render_report.main(["--summary", spath, "-o", os.path.join(work, "t.html")]) - finally: - results_index.build = keep_build - check("a text-only report does not build the index", txt_calls == 0, txt_calls) - check("red control: an HTML report does build it", len(calls) > txt_calls, calls) - - # 5. queries - r = results_index.query(idx, "brca") - check("query('BRCA') lists BRCA1 and BRCA2", r["kind"] == "gene" and not r["items"] - and r["suggestions"] == ["BRCA1", "BRCA2"], r) - r = results_index.query(idx, "brca2") - check("query('brca2') finds BRCA2 in four sections", [it["section"] for it in r["items"]] - == ["clinvar", "carrier", "clinical", "slivar"] and r["suggestions"] == [], r) - r = results_index.query(idx, "chr7:1,000") - check("a position gets the owngenome-lookup.sh hint and no entries", r["kind"] == "site" and not r["items"] - and "owngenome-lookup.sh" in r["hint"], r) - r = results_index.query(idx, "TP53") - check("a gene the results do not name: no entries", r["kind"] == "gene" and not r["items"], r) - check("not a query: an error", results_index.query(idx, "two words")["kind"] == "error") - - # 6. the appendix - page = render_report.html_report(s_tsv, idx) - ids = set(re.findall(r'id="([^"]+)"', page)) - app = page[page.index('id="ch-terms"'):page.index('
')] - rows = re.findall(r"
  • ([^<]+): (.*)
  • ", app) - hrefs = re.findall(r'href="#([^"]+)"', app) - check("appendix: one line per term, in the index's order", [t for t, _ in rows] == list(T), rows) - check("appendix: every link resolves to an id of the page", hrefs and all(h in ids for h in hrefs), - [h for h in hrefs if h not in ids]) - check("appendix: BRCA2 links its four sections", re.search( - r'BRCA2:
    [^<]+, [^<]+, ' - r'[^<]+, [^<]+', app) is not None, app[:2000]) - check("appendix: in the contents", 'Genes and rsIDs in these results' in page) - check("the report has no script", "CYP2D6" in page and "CYP2D6" not in render_report.html_report(s_tsv, less)) - bare = render_report.html_report(s_tsv) - check("without an index the appendix is built from the summary alone (no step 23 or 31 genes)", - "CFTR" in bare and "BRCA1" not in bare, "") - - # 7. command lines - out = os.path.join(work, "idx.json") - p = subprocess.run([sys.executable, os.path.join(REPO, "bin", "results_index.py"), "build", "--summary", spath, - "--sample-dir", tsv, "-o", out], capture_output=True, text=True) - with open(out) as f: - cli = json.load(f) - check("CLI build: the same terms as build()", p.returncode == 0 and cli["terms"] == T, p.stderr) - q = subprocess.run([sys.executable, os.path.join(REPO, "bin", "results_index.py"), "query", "--index", out, - "rs123"], capture_output=True, text=True) - check("CLI query rs123: exit 0, names the section", q.returncode == 0 and "Variant prioritization" in q.stdout, - q.stdout + q.stderr) - q = subprocess.run([sys.executable, os.path.join(REPO, "bin", "results_index.py"), "query", "--index", out, - "7:117559590"], capture_output=True, text=True) - check("CLI query of a position: exit 2 with the hint", q.returncode == 2 and "owngenome-lookup.sh" in q.stdout, q.stdout) - rr = os.path.join(work, "rr.json") - with redirect_stdout(io.StringIO()): - render_report.main(["--sample", "SYNTH", "--sample-dir", tsv, "--json", os.path.join(work, "s2.json"), - "-o", os.path.join(work, "r.html"), "--search-index", rr]) - with open(rr) as f: - check("render_report.py --search-index writes the same terms", json.load(f)["terms"] == T) - - # 8. the live search - run = os.path.join(work, "SYNTH") - os.makedirs(run) - check("live: no index file, no state", owngenome_live_search.search_state(run, {}, []) == {}) - shutil.copy(rr, os.path.join(run, "SYNTH_search_index.json")) - st8 = owngenome_live_search.search_state(run, {}, []) - check("live: the state names the file and its mtime", st8.get("file") == "SYNTH_search_index.json" - and isinstance(st8.get("mtime"), int), st8) - - class App: - globs, hooks = [], {} - - def run_file(self, g): - self.globs.append(g) - - def state_hook(self, name, fn): - self.hooks[name] = fn - app_ = App() - owngenome_live_search.register(app_) - check("live: the plugin serves *_search_index.json and hooks plugins.search", - app_.globs == ["*_search_index.json"] and app_.hooks.get("search") is owngenome_live_search.search_state) - # the live server finds plugins by file name, so a plugin under a name its loader - # does not match would never load; load bin/ the way the server does - import owngenome_live - import argparse - real = owngenome_live.App() - real.parser = argparse.ArgumentParser() # other plugins add flags, as at the server's start - loaded = owngenome_live.load_plugins(real, os.path.join(REPO, "bin")) - check("live: the server's loader picks up the search plugin", - "owngenome_live_search.py" in loaded and "*_search_index.json" in real.run_globs - and "search" in [n for n, _ in real.state_hooks], loaded) - js = open(SEARCH_JS).read() - check("search.js: one fetch (the index), no XMLHttpRequest, no sendBeacon, no WebSocket, no eval", - js.count("fetch(") == 1 and not re.search(r"XMLHttpRequest|sendBeacon|WebSocket|eval\(|innerHTML", js)) - queries = ["RS123", "rs1000002", "brca", "BRCA2", "cyp2", "TP53", "chr7:1,000", "13-2000", "chr1:0", "two words", ""] - node = shutil.which("node") - check("node is on the PATH (ubuntu-latest has it)", node is not None) - if node: - prog = ("const m=require(process.argv[1]).match;const idx=JSON.parse(require('fs').readFileSync(process.argv[2]));" - "const qs=JSON.parse(process.argv[3]);console.log(JSON.stringify(qs.map(q=>{const r=m(idx,q);" - "return [r.kind,r.term,r.items.map(i=>i.section),r.suggestions]})))") - p = subprocess.run([node, "-e", prog, SEARCH_JS, rr, json.dumps(queries)], capture_output=True, text=True) - got = json.loads(p.stdout) if p.returncode == 0 else p.stderr - want = [] - for q in queries: - r = results_index.query(idx, q) - want.append([r["kind"], r["term"], [i["section"] for i in r["items"]], r["suggestions"]]) - check("search.js match() answers each fixture query as query() does", got == want, (got, want)) - check("search.js: rs123 lists variant prioritization", got[0] == ["rsid", "rs123", ["slivar"], []], got) - finally: - shutil.rmtree(work) - print("\nRESULT:", "ALL PASS" if FAILS == 0 else f"{FAILS} FAILED") - return 1 if FAILS else 0 - - -if __name__ == "__main__": - sys.exit(main())