From 2142fd01802d90acf3ac28f11b6d43c66b2d6d58 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 20:55:28 +0100 Subject: [PATCH 01/12] test(docs): fail when a docs table puts an escaped pipe inside a code span The docs site prints a code span as written, so a table cell that escapes a pipe for GitHub shows the reader a backslash, and the copied command breaks. The check finds every such row under docs/ and proves itself on planted pages. Bead: K013 Bead: K020 --- tests/test_docs_table_pipes.sh | 131 +++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 tests/test_docs_table_pipes.sh diff --git a/tests/test_docs_table_pipes.sh b/tests/test_docs_table_pipes.sh new file mode 100644 index 00000000..588e61d4 --- /dev/null +++ b/tests/test_docs_table_pipes.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# A command inside a code span in a docs table must not hold an escaped pipe. +# +# GitHub needs `\|` to keep a pipe inside a table cell, but the docs site +# prints a code span as written, so the reader sees and copies the backslash: +# `curl -s https://get.nextflow.io \| NXF_VER=... bash` is not a pipeline. +# A command that needs a pipe goes in a code block under the table, and a +# table cell names a separator in words. +# +# Checks: +# 1. a planted page with such a row fails, and the scanner names its line; +# 2. a planted page with a pipe only where it renders right passes: a +# fenced block, prose, a table cell outside a code span, a code span in +# prose, and backticks that do not pair; +# 3. every docs/**/*.md page passes. +# +# Run: bash tests/test_docs_table_pipes.sh [DIR] (DIR defaults to docs/) +set -euo pipefail + +REPO=$(cd "$(dirname "$0")/.." && pwd) +DOCS=${1:-"${REPO}/docs"} +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +FAILS=0 +fail() { echo "FAIL: $*"; FAILS=$((FAILS + 1)); } +pass() { echo "ok: $*"; } + +# scan FILE...: prints FILE:LINE: TEXT for each table row whose code span holds +# a backslash and a pipe; returns 1 when there is one. A table row starts with +# a pipe. Lines inside ``` or ~~~ fences are skipped. A code span opens with a +# run of N backticks and closes at the next run of exactly N; a run with no +# partner is plain text. +scan() { + awk ' + function bad(line, rest, tmp, n, off, found, code) { + rest = line + while (match(rest, /`+/)) { + n = RLENGTH + rest = substr(rest, RSTART + n) + tmp = rest; off = 0; found = 0 + while (match(tmp, /`+/)) { + if (RLENGTH == n) { + code = substr(rest, 1, off + RSTART - 1) + rest = substr(tmp, RSTART + RLENGTH) + found = 1 + break + } + off += RSTART + RLENGTH - 1 + tmp = substr(tmp, RSTART + RLENGTH) + } + if (found && index(code, "\\|") > 0) return 1 + } + return 0 + } + FNR == 1 { fence = 0 } + /^[ \t]*(```|~~~)/ { fence = !fence; next } + fence { next } + /^[ \t]*\|/ && bad($0) { print FILENAME ":" FNR ": " $0; hits++ } + END { exit hits > 0 } + ' "$@" +} + +# 1. A planted row like the old Nextflow line fails, and the output names it. +cat > "${WORK}/bad.md" <<'MD' +# Planted + +| Software | Install | +|---|---| +| Docker | from docs.docker.com | +| Nextflow | `curl -s https://get.nextflow.io \| NXF_VER=1.0.0 bash` | +MD +if out=$(scan "${WORK}/bad.md"); then + fail "a planted table row with an escaped pipe in a code span passed" +elif grep -qF "bad.md:6: | Nextflow |" <<<"$out"; then + pass "a planted table row with an escaped pipe in a code span fails and is named" +else + fail "the planted row failed but its line was not named: ${out}" +fi + +# Double backticks open a span too. +printf '| a | ``x \\| y`` |\n' > "${WORK}/double.md" +if scan "${WORK}/double.md" >/dev/null; then + fail "an escaped pipe in a double-backtick span passed" +else + pass "an escaped pipe in a double-backtick span fails" +fi + +# 2. Pipes where they render right pass. +cat > "${WORK}/good.md" <<'MD' +# Planted + +Prose may say `a \| b` outside a table. + +| Column | Meaning | +|---|---| +| `readings` | `KEY=text`, separated by a vertical bar | +| choice | `left` \| `right` | +| stray | a lone `` run, then \| and `code` | + +```bash +dmesg | grep -i "oom\|killed" +| `not a table \| row` | +``` + +~~~ +| `still fenced \| here` | +~~~ +MD +if out=$(scan "${WORK}/good.md"); then + pass "pipes in fences, prose, cells outside code spans and unpaired backticks pass" +else + fail "a page with no bad row failed: ${out}" +fi + +# 3. The real docs. +mapfile -t pages < <(find "$DOCS" -name '*.md' -type f | sort) +if [ "${#pages[@]}" -eq 0 ]; then + fail "no Markdown pages under ${DOCS}" +elif out=$(scan "${pages[@]}"); then + pass "no docs table row has an escaped pipe in a code span (${#pages[@]} pages)" +else + echo "$out" + fail "docs table rows with an escaped pipe in a code span; move the command into a code block under the table" +fi + +if [ "$FAILS" -gt 0 ]; then + echo "${FAILS} check(s) failed" + exit 1 +fi +echo "All checks passed." From ec9079651b54ae60ab433fe35fb39c688234d732 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 20:57:45 +0100 Subject: [PATCH 02/12] fix(docs): make the Nextflow install and three other table commands copy and run as written The docs site printed the backslash of each escaped pipe inside a table code span, so the copied command was broken. The Nextflow row of Getting started points at the install command in Full run, the FAQ's BAM build check and the site table hash of Keeping versions up to date move into code blocks under their tables, and the cn-loci readings row names its separator in words. Bead: K013 Bead: K020 --- CHANGELOG.md | 1 + docs/cn-loci.md | 2 +- docs/faq.md | 8 +++++++- docs/getting-started.md | 2 +- docs/updating.md | 9 ++++++++- 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05c3388e..1fd8c9be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Four commands in docs tables now copy and run as written. The docs site printed the backslash that escaped each pipe for GitHub, so the Nextflow install line in Getting started read `curl -s https://get.nextflow.io \| ...`, which is not a pipeline. That row now points at the install command in Full run. The BAM build check of the FAQ and the site table hash of Keeping versions up to date moved into code blocks under their tables, and the `readings` row of Copy Number from Depth names its separator in words. `tests/test_docs_table_pipes.sh` fails when a table row under `docs/` holds an escaped pipe inside a code span. - The self-test of `scripts/ci/check-images.sh` copies the whole tree again once the list of tracked paths passes 128 KiB. It piped that list through `xargs` to `tar`, and GNU `xargs` split a list that long over two `tar` runs; the reading `tar` stopped after the first archive, so Guard's `images` job failed on a broken pipe (exit 123). One `tar` now reads the list itself (`tar --null -T -`). Nothing the check reports changes. - The phased family VCF is indexed by the Rust `og` binary. WhatsHap's step (step 38 and the Nextflow WHATSHAP_PED process) wrote `family.phased.vcf.gz.tbi` with a Python one-liner, `pysam.tabix_index`. It now runs `og vcf index -f`. The `.tbi` keeps its name and format, and its bytes, once inflated, are the ones htslib 1.24 writes (the htslib of the pysam and bcftools the pipeline pins): the same bins in the same order, the same chunks and the same linear index. Only the deflate stream can differ, as two deflate libraries differ. On a 5-million-record three-sample VCF it took 0.75 to 0.91 s and 31 MB at peak, where htslib's own `tabix` took 0.81 to 0.93 s and 19 MB. - As in htslib 1.24, a record ends at the largest of INFO/END, the REF length, the SVLEN of a ``, ``, `` or `` ALT, and a sample's LEN on a gVCF block. A line with no position to read, unsorted records, a sequence that starts again, a record past 2^29 and an empty file stop the index with a message, and nothing is written. Without `-f` it never writes over an index, even one that appears while it runs. diff --git a/docs/cn-loci.md b/docs/cn-loci.md index 43882cab..a2577271 100644 --- a/docs/cn-loci.md +++ b/docs/cn-loci.md @@ -45,7 +45,7 @@ Then the row's `call` column decides: | `mapq0_min`, `mapq0_max` | the MAPQ 0 share guards, or `.` | | `min_control_depth`, `max_control_ratio` | the control guards, or `.` | | `call`, `interval` | as above | -| `readings` | `KEY=text`, separated by `\|`: the text for a count, a class, or `more` for any count above the keys | +| `readings` | `KEY=text`, separated by a vertical bar: the text for a count, a class, or `more` for any count above the keys | | `input` | `bedcov` (this engine) or `cnvpytor-level` (the row records thresholds on CNVpytor's depth level, where 1.0 is two copies) | | `owner` | the engine that calls the locus (`cn-depth`, this one) | | `cross_check` | another engine that also reports the locus (`parascopy:RHCE`, `paraphase:hba`, `cnvpytor:cnvs` for step 18's CNVpytor calls) or `none` | diff --git a/docs/faq.md b/docs/faq.md index 030c6659..d8594c01 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -8,11 +8,17 @@ | "Permission denied" writing output | Container runs as non-root | Add `--user root` to `docker run` (already done in all scripts) | | VEP cache download fails/times out | 26 GB download over unreliable connection | Use `wget -c` (supports resume). See [docs/13-vep-annotation.md](13-vep-annotation.md) | | DeepVariant crashes on Mac | amd64 emulation + memory pressure | `THREADS=2 DV_MEM=12g ./scripts/03-deepvariant.sh `, the sex being `male` or `female`. Will be slow. See [troubleshooting](troubleshooting.md#step-3-deepvariant-crashes-on-mac-amd64-emulation). | -| Wrong number of variants (too few) | Genome build mismatch | Ensure your BAM is aligned to GRCh38 (hg38), not hg19/GRCh37. Check with `samtools view -H your.bam \| grep SN:chr1` | +| Wrong number of variants (too few) | Genome build mismatch | Ensure your BAM is aligned to GRCh38 (hg38), not hg19/GRCh37. Check the BAM header with the command under this table | | 0-byte output files | Missing input or wrong path | Check that all input files exist. Run the script with `bash -x` for debug output. | | "No such image" on `docker pull` | Image name/tag changed | Check the exact image name in the step's documentation. Biocontainer tags change frequently. | | Very slow on macOS | Rosetta 2 emulation overhead | Expected. Consider running on a Linux machine or cloud instance for heavy steps. | +To check the genome build of a BAM, print the chr1 line of its header. GRCh38 has `LN:248956422`, hg19 has `LN:249250621`, and GRCh37 names the contig `1`, so the command prints nothing: + +```bash +samtools view -H your.bam | grep -w SN:chr1 +``` + For detailed solutions, see: - [docs/chip-data-guide.md](chip-data-guide.md) — using 23andMe/MyHeritage/AncestryDNA data with this pipeline - [docs/troubleshooting.md](troubleshooting.md) — comprehensive troubleshooting guide organized by symptom diff --git a/docs/getting-started.md b/docs/getting-started.md index e7223978..744e4815 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -30,7 +30,7 @@ | Docker | 20.10+ | [docs.docker.com/get-docker](https://docs.docker.com/get-docker/) | | bash | 4.4+ | Pre-installed on Linux; macOS ships 3.2, install a newer one with `brew install bash` | | Java *(for a full run)* | 17+ | Any OpenJDK build, e.g. `apt install openjdk-17-jre-headless` or [Adoptium Temurin](https://adoptium.net/) | -| Nextflow *(for a full run)* | `NEXTFLOW_VERSION` in `versions.env` | `curl -s https://get.nextflow.io \| NXF_VER=26.10.5 bash`, see [Full run](#full-run) | +| Nextflow *(for a full run)* | `NEXTFLOW_VERSION` in `versions.env` | The pinned install command in [Full run](#full-run) | | wget or curl | Any | For downloading references | | python3 *(optional)* | 3.6+ | Used by long-read alignment (02b) for symlink resolution. Falls back to `readlink -f` on GNU/Linux if absent | diff --git a/docs/updating.md b/docs/updating.md index 46a888da..a1a9dd51 100644 --- a/docs/updating.md +++ b/docs/updating.md @@ -24,7 +24,14 @@ This page is for maintainers. It says how a pinned version gets bumped, what che | IPD-KIR release (step 8 with `KIR=true`) | `KIR_DB_RELEASE` in `versions.env` | nobody: the monthly freshness report says when a newer release exists | | Parascopy data (step 35) | `PARASCOPY_DATA_VERSION` in `versions.env`, with the Zenodo record and the md5s in `scripts/setup.sh` | nobody: the monthly freshness report says when the record has a newer version | | Cyrius lock (step 21) | `CYRIUS_VERSION` in `versions.env` and every wheel in `scripts/cyrius-constraints.txt` | nobody: resolved again by hand in `PYTHON_IMAGE`, as the lock's header says. The freshness report says when PyPI has a newer Cyrius | -| The site table of step 40: the 23andMe v5 site list commit, the dbSNP build and the table's hash | `CONSUMER_SITES_COMMIT`, `CONSUMER_DBSNP_BUILD`, their checksums and `CONSUMER_SITES_TABLE_SHA256` in `versions.env` | nobody: dbSNP releases a build every year or two, and the site list has not changed since 2019. A new build, a new commit or a change to `og consumer build-sites` needs one full build (`setup.sh --consumer-sites`, about 30 GB), then `gzip -dc \| sha256sum` into `CONSUMER_SITES_TABLE_SHA256`. CI never builds the full table. The monthly freshness report says when NCBI publishes a newer build | +| The site table of step 40: the 23andMe v5 site list commit, the dbSNP build and the table's hash | `CONSUMER_SITES_COMMIT`, `CONSUMER_DBSNP_BUILD`, their checksums and `CONSUMER_SITES_TABLE_SHA256` in `versions.env` | nobody: dbSNP releases a build every year or two, and the site list has not changed since 2019. A new build, a new commit or a change to `og consumer build-sites` needs one full build (`setup.sh --consumer-sites`, about 30 GB), then record the sha256 of the unpacked table in `CONSUMER_SITES_TABLE_SHA256` (the command is under this table). CI never builds the full table. The monthly freshness report says when NCBI publishes a newer build | + +The hash of the site table of step 40 is the sha256 of its unpacked content, because the gzip wrapper holds the time it was written. Run this from the repository root after the build: + +```bash +. ./versions.env +gzip -dc "${GENOME_DIR}/reference/consumer/23andme_v5_sites_dbsnp${CONSUMER_DBSNP_BUILD}.tsv.gz" | sha256sum +``` Three pins are floors, not versions that drift: Node 24 in `renovate-dry-run.yml`, Java 17 in the workflows that run Nextflow, and `manifest.nextflowVersion` in `nextflow.config`. Nothing proposes a bump for them, on purpose. From b777b7d494d85c6cb92a1313b65ea2774d0feb9a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 20:58:48 +0100 Subject: [PATCH 03/12] test(docs): plant the double-backtick row with a heredoc ShellCheck read the single-quoted printf as a missed expansion. Bead: K013 --- tests/test_docs_table_pipes.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_docs_table_pipes.sh b/tests/test_docs_table_pipes.sh index 588e61d4..eee9daf4 100644 --- a/tests/test_docs_table_pipes.sh +++ b/tests/test_docs_table_pipes.sh @@ -79,7 +79,9 @@ else fi # Double backticks open a span too. -printf '| a | ``x \\| y`` |\n' > "${WORK}/double.md" +cat > "${WORK}/double.md" <<'MD' +| a | ``x \| y`` | +MD if scan "${WORK}/double.md" >/dev/null; then fail "an escaped pipe in a double-backtick span passed" else From 9935a5d825630d59937ea987ccb8572908d3f1f3 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 21:05:33 +0100 Subject: [PATCH 04/12] docs(privacy): say which servers the CPSR and MultiQC reports contact when opened The privacy page left the CPSR (step 17) and MultiQC (step 28) reports as "not checked", so a reader could not tell whether opening them tells a server their IP address. Both were opened in a browser that blocked and logged every request to the internet: neither loads anything. Their external addresses are plain links. Bead: K010 Bead: K028 --- CHANGELOG.md | 1 + docs/why-local.md | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05c3388e..c8f4b53e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- The privacy page now says what the CPSR (step 17) and MultiQC (step 28) reports contact when you open them: nothing. Both keep their stylesheets, scripts and images inside the HTML file, and their external addresses are plain links. The page said before that these two reports were not checked. Docs only; no output changes. - The self-test of `scripts/ci/check-images.sh` copies the whole tree again once the list of tracked paths passes 128 KiB. It piped that list through `xargs` to `tar`, and GNU `xargs` split a list that long over two `tar` runs; the reading `tar` stopped after the first archive, so Guard's `images` job failed on a broken pipe (exit 123). One `tar` now reads the list itself (`tar --null -T -`). Nothing the check reports changes. - The phased family VCF is indexed by the Rust `og` binary. WhatsHap's step (step 38 and the Nextflow WHATSHAP_PED process) wrote `family.phased.vcf.gz.tbi` with a Python one-liner, `pysam.tabix_index`. It now runs `og vcf index -f`. The `.tbi` keeps its name and format, and its bytes, once inflated, are the ones htslib 1.24 writes (the htslib of the pysam and bcftools the pipeline pins): the same bins in the same order, the same chunks and the same linear index. Only the deflate stream can differ, as two deflate libraries differ. On a 5-million-record three-sample VCF it took 0.75 to 0.91 s and 31 MB at peak, where htslib's own `tabix` took 0.81 to 0.93 s and 19 MB. - As in htslib 1.24, a record ends at the largest of INFO/END, the REF length, the SVLEN of a ``, ``, `` or `` ALT, and a sample's LEN on a gVCF block. A line with no position to read, unsorted records, a sequence that starts again, a record past 2^29 and an empty file stop the index with a message, and nothing is written. Without `-f` it never writes over an index, even one that appears while it runs. diff --git a/docs/why-local.md b/docs/why-local.md index 3fca718f..5d11bddb 100644 --- a/docs/why-local.md +++ b/docs/why-local.md @@ -46,7 +46,10 @@ A report that loads a file from the internet tells that server your IP address a - **indexcov report (step 16):** loads Chart.js from `cdnjs.cloudflare.com` and jQuery from `code.jquery.com`. Offline, the page opens without its plots. - **Nextflow report and timeline:** load nothing. -Not checked: the CPSR (step 17) and MultiQC (step 28) reports, which the test cannot produce. +The test cannot produce the CPSR and MultiQC reports. On 2026-11-17 we opened one of each in a browser that blocked and logged every request to the internet: + +- **CPSR report (step 17):** loads nothing. Its stylesheets, scripts and images are inside the HTML file. Its external addresses are plain links to the sources it cites (ClinVar, PanelApp, PubMed and others), which open only when you click them. We checked the CPSR project's own example reports of versions 2.2.5 and 2.4.0; the pinned PCGR 2.3.2 image builds its report from a template that embeds every resource in the same way. +- **MultiQC report (step 28):** loads nothing. Its external addresses are plain links to MultiQC, Seqera and the papers of the tools it reports. We checked a report from the pinned MultiQC 1.35 image, run with the step's options. **Recommendations for securing your data:** - Store genomic data on an encrypted filesystem (LUKS on Linux, FileVault on macOS, BitLocker on Windows) From 21c1cbab7f237afa7bc67997967fce57690e8cab Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 21:19:35 +0100 Subject: [PATCH 05/12] test(docs): check the active menu item contrast and the code line anchors A cargo test in og-dev reads docs/stylesheets/theme.css and zensical.toml. It fails when the accent a colour scheme ends up with is below 4.5:1 on the active menu item's tint, and when anchor_linenums is on. Both fail on main: 3.12:1 dark, 4.49:1 light, and the flag. Bead: S08 Bead: S10 --- rust/crates/og-dev/tests/docs_theme.rs | 183 +++++++++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 rust/crates/og-dev/tests/docs_theme.rs diff --git a/rust/crates/og-dev/tests/docs_theme.rs b/rust/crates/og-dev/tests/docs_theme.rs new file mode 100644 index 00000000..322b1878 --- /dev/null +++ b/rust/crates/og-dev/tests/docs_theme.rs @@ -0,0 +1,183 @@ +//! The docs theme on this checkout: the current page in the side menu stays +//! readable, and code blocks carry no empty line anchors. +//! +//! 1. Zensical paints the active navigation item in `--md-accent-fg-color` +//! on a background of `--md-accent-fg-color--transparent` laid over the +//! page. For each colour scheme, the accent that scheme ends up with (its +//! own block in `docs/stylesheets/theme.css`, else `:root`) must reach +//! WCAG AA for normal text, 4.5:1, against that background. +//! 2. `zensical.toml` must not turn on `pymdownx.highlight`'s +//! `anchor_linenums`. The site shows no line numbers, so each code line +//! would get an empty link with no name that still takes a Tab stop. + +use std::fs; +use std::path::{Path, PathBuf}; + +fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +fn read(rel: &str) -> String { + fs::read_to_string(repo().join(rel)).unwrap_or_else(|e| panic!("{rel}: {e}")) +} + +/// The CSS with every `/* ... */` comment removed. +fn strip_comments(css: &str) -> String { + let mut out = String::with_capacity(css.len()); + let mut rest = css; + while let Some(start) = rest.find("/*") { + out.push_str(&rest[..start]); + rest = match rest[start + 2..].find("*/") { + Some(end) => &rest[start + 2 + end + 2..], + None => "", + }; + } + out.push_str(rest); + out +} + +/// The body of the first rule whose selector is exactly `selector`. +fn block<'a>(css: &'a str, selector: &str) -> Option<&'a str> { + let mut rest = css; + while let Some(open) = rest.find('{') { + let head = rest[..open].rsplit('}').next().unwrap_or("").trim(); + let close = open + rest[open..].find('}')?; + if head == selector { + return Some(&rest[open + 1..close]); + } + rest = &rest[close + 1..]; + } + None +} + +/// The value of custom property `name` in a rule body, the last one if set twice. +fn prop(body: &str, name: &str) -> Option { + body.split(';') + .filter_map(|d| d.split_once(':')) + .filter(|(k, _)| k.trim() == name) + .map(|(_, v)| v.trim().to_owned()) + .next_back() +} + +/// `#rrggbb` or `#rrggbbaa` as red, green, blue (0-255) and alpha (0-1). +fn hex(v: &str) -> [f64; 4] { + let h = v + .strip_prefix('#') + .unwrap_or_else(|| panic!("not a hex colour: {v}")); + assert!(h.len() == 6 || h.len() == 8, "not #rrggbb(aa): {v}"); + let byte = |i: usize| { + f64::from(u8::from_str_radix(&h[i..i + 2], 16).unwrap_or_else(|_| panic!("bad hex: {v}"))) + }; + let a = if h.len() == 8 { byte(6) / 255.0 } else { 1.0 }; + [byte(0), byte(2), byte(4), a] +} + +/// `top` laid over the opaque `under`. +fn over(top: [f64; 4], under: [f64; 4]) -> [f64; 4] { + let a = top[3]; + [ + top[0] * a + under[0] * (1.0 - a), + top[1] * a + under[1] * (1.0 - a), + top[2] * a + under[2] * (1.0 - a), + 1.0, + ] +} + +/// WCAG 2 relative luminance of an opaque colour. +fn luminance(c: [f64; 4]) -> f64 { + let lin = |v: f64| { + let s = v / 255.0; + if s <= 0.040_45 { + s / 12.92 + } else { + ((s + 0.055) / 1.055).powf(2.4) + } + }; + 0.2126 * lin(c[0]) + 0.7152 * lin(c[1]) + 0.0722 * lin(c[2]) +} + +fn contrast(a: [f64; 4], b: [f64; 4]) -> f64 { + let (la, lb) = (luminance(a), luminance(b)); + (la.max(lb) + 0.05) / (la.min(lb) + 0.05) +} + +/// The scheme's accent and the active item's background, as the theme sets them. +/// `page_default` is the page colour Zensical uses when the block sets none; +/// `None` means the block must set its own. +fn active_item(css: &str, scheme: &str, page_default: Option<&str>) -> (String, [f64; 4], f64) { + let root = block(css, ":root").expect("theme.css has a :root block"); + let sel = format!("[data-md-color-scheme=\"{scheme}\"]"); + let own = block(css, &sel).unwrap_or_else(|| panic!("theme.css has no {sel} block")); + let pick = |name: &str| prop(own, name).or_else(|| prop(root, name)); + let accent = pick("--md-accent-fg-color").expect("an accent colour"); + let tint = pick("--md-accent-fg-color--transparent").expect("a transparent accent"); + let page = pick("--md-default-bg-color") + .or_else(|| page_default.map(str::to_owned)) + .unwrap_or_else(|| panic!("{sel} sets no --md-default-bg-color")); + let bg = over(hex(&tint), hex(&page)); + let ratio = contrast(hex(&accent), bg); + (accent, bg, ratio) +} + +#[test] +fn the_active_menu_item_reads_in_both_schemes() { + let css = strip_comments(&read("docs/stylesheets/theme.css")); + // The default scheme keeps Zensical's white page; the slate block sets + // its own page colour. + let mut low = Vec::new(); + for (scheme, page) in [("slate", None), ("default", Some("#ffffff"))] { + let (accent, bg, ratio) = active_item(&css, scheme, page); + if ratio < 4.5 { + low.push(format!( + "{scheme}: {accent} on rgb({:.0},{:.0},{:.0}) is {ratio:.2}:1", + bg[0], bg[1], bg[2] + )); + } + } + assert!( + low.is_empty(), + "the active menu item is below WCAG AA (4.5:1): {}", + low.join("; ") + ); +} + +#[test] +fn the_contrast_maths_matches_known_pairs() { + // Black on white is 21:1, and a colour on itself is 1:1. + assert!((contrast(hex("#000000"), hex("#ffffff")) - 21.0).abs() < 1e-9); + assert!((contrast(hex("#2563eb"), hex("#2563eb")) - 1.0).abs() < 1e-9); + // The two failing pairs the old theme had: #2563eb on its own 10% tint + // over each page. + let slate = over(hex("#2563eb1a"), hex("#1c1917")); + let light = over(hex("#2563eb1a"), hex("#ffffff")); + let (s, l) = ( + contrast(hex("#2563eb"), slate), + contrast(hex("#2563eb"), light), + ); + assert!((3.10..3.15).contains(&s), "slate {s}"); + assert!((4.45..4.5).contains(&l), "default {l}"); +} + +#[test] +fn code_lines_carry_no_empty_anchors() { + let toml = read("zensical.toml"); + let on: Vec<&str> = toml + .lines() + .map(str::trim) + .filter(|l| !l.starts_with('#')) + .filter(|l| l.contains("anchor_linenums")) + .filter(|l| { + let bare: String = l + .split('#') + .next() + .unwrap_or("") + .split_whitespace() + .collect(); + bare.contains("anchor_linenums=true") + }) + .collect(); + assert!( + on.is_empty(), + "zensical.toml turns on anchor_linenums, which adds an empty, unnamed link to every code line: {on:?}" + ); +} From 9a63979dca51c858ff87e653dfa1c991d6d15b62 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 21:19:35 +0100 Subject: [PATCH 06/12] fix(docs): give the active menu item the link colour of each scheme The side menu paints the current page in the accent, #2563eb for both schemes, on the accent's 10% tint. That is 3.1:1 on the dark page and 4.49:1 on the light one. The accent now takes #60a5fa on slate (6.3:1) and #1d4ed8 on default (5.8:1), the colours links already use. The transparent accent and the card hover border stay. Bead: S08 --- docs/stylesheets/theme.css | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/stylesheets/theme.css b/docs/stylesheets/theme.css index 5824741b..feee57ca 100644 --- a/docs/stylesheets/theme.css +++ b/docs/stylesheets/theme.css @@ -20,17 +20,24 @@ /* Dark scheme: the page body uses the header colour too, so the header and the page read as one surface. #2563eb on #1c1917 is 3.9:1, too faint for - body text; #60a5fa is the same hue at 7.4:1. */ + body text; #60a5fa is the same hue at 7.4:1. + The current page in the side menu is painted in the accent, on the + transparent accent laid over the page (#1d202c). #2563eb there is 3.1:1, + so the accent takes the link colour too: #60a5fa is 6.3:1. */ [data-md-color-scheme="slate"] { --md-default-bg-color: #1c1917; --md-code-bg-color: #292524; --md-typeset-a-color: #60a5fa; + --md-accent-fg-color: #60a5fa; } /* Light scheme: #2563eb on white is 4.5:1, on the edge. #1d4ed8 is the same - hue at 6.3:1. */ + hue at 6.3:1. On the current menu item's tint (#e9effd) #2563eb is 4.49:1, + under 4.5:1, so the accent takes #1d4ed8 as well: 5.8:1. + rust/crates/og-dev/tests/docs_theme.rs checks both schemes. */ [data-md-color-scheme="default"] { --md-typeset-a-color: #1d4ed8; + --md-accent-fg-color: #1d4ed8; } /* Fixed logo size so it does not crowd the site name. */ From 7b83aec23d946ef3104e43cf80ae465f68cbaf9c Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 21:19:35 +0100 Subject: [PATCH 07/12] fix(docs): drop the empty line anchors from code blocks anchor_linenums gave every code line an empty link with no name that takes a Tab stop: 122 on getting-started, 88 pages in all. The site shows no line numbers and no page links to these anchors. Highlighting and the copy button do not change. Bead: S10 --- CHANGELOG.md | 1 + zensical.toml | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05c3388e..645db450 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- The docs site: the current page in the side menu is readable, and code blocks no longer add a Tab stop per line. The menu paints the current page in the accent colour, #2563eb, on a light tint of the same blue. That was 3.1:1 on the dark page and 4.49:1 on the light one, below the 4.5:1 that WCAG AA asks of normal text. The accent now takes the link colour of each scheme: #60a5fa on dark (6.3:1) and #1d4ed8 on light (5.8:1). `zensical.toml` no longer sets `anchor_linenums`: the site shows no line numbers, yet every code line got an empty link with no name, 122 of them on the getting-started page, on 88 pages in all. Code highlighting and the copy button do not change. A cargo test, `rust/crates/og-dev/tests/docs_theme.rs`, checks both. - The self-test of `scripts/ci/check-images.sh` copies the whole tree again once the list of tracked paths passes 128 KiB. It piped that list through `xargs` to `tar`, and GNU `xargs` split a list that long over two `tar` runs; the reading `tar` stopped after the first archive, so Guard's `images` job failed on a broken pipe (exit 123). One `tar` now reads the list itself (`tar --null -T -`). Nothing the check reports changes. - The phased family VCF is indexed by the Rust `og` binary. WhatsHap's step (step 38 and the Nextflow WHATSHAP_PED process) wrote `family.phased.vcf.gz.tbi` with a Python one-liner, `pysam.tabix_index`. It now runs `og vcf index -f`. The `.tbi` keeps its name and format, and its bytes, once inflated, are the ones htslib 1.24 writes (the htslib of the pysam and bcftools the pipeline pins): the same bins in the same order, the same chunks and the same linear index. Only the deflate stream can differ, as two deflate libraries differ. On a 5-million-record three-sample VCF it took 0.75 to 0.91 s and 31 MB at peak, where htslib's own `tabix` took 0.81 to 0.93 s and 19 MB. - As in htslib 1.24, a record ends at the largest of INFO/END, the REF length, the SVLEN of a ``, ``, `` or `` ALT, and a sample's LEN on a gVCF block. A line with no position to read, unsorted records, a sequence that starts again, a record past 2^29 and an empty file stop the index with a message, and nothing is written. Without `-f` it never writes over an index, even one that appears while it runs. diff --git a/zensical.toml b/zensical.toml index 54449432..29ef912a 100644 --- a/zensical.toml +++ b/zensical.toml @@ -211,7 +211,9 @@ admonition = {} { name = "mermaid", class = "mermaid", format = "pymdownx.superfences.fence_code_format" }, ] "pymdownx.tabbed".alternate_style = true -"pymdownx.highlight".anchor_linenums = true +# No highlight anchor_linenums: the site shows no line numbers, and it would +# give every code line an empty, unnamed link that takes a Tab stop +# (rust/crates/og-dev/tests/docs_theme.rs). "pymdownx.inlinehilite" = {} "pymdownx.snippets".base_path = [".", "docs"] "pymdownx.snippets".check_paths = true From 2f5a61cbc5db35cb353f7b57db5231725e378968 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 18 Nov 2026 10:22:00 +0100 Subject: [PATCH 08/12] fix(docs): keep the text on the accent readable in the dark scheme The dark accent #60a5fa took Zensical's white --md-accent-bg-color text on the Back to top and side menu buttons at 2.5:1. That text now takes the page colour, 6.9:1, and the docs theme test checks text on the accent in both schemes. --- CHANGELOG.md | 2 +- docs/stylesheets/theme.css | 7 +++- rust/crates/og-dev/tests/docs_theme.rs | 46 +++++++++++++++++++++++++- 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 645db450..9a33f353 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- The docs site: the current page in the side menu is readable, and code blocks no longer add a Tab stop per line. The menu paints the current page in the accent colour, #2563eb, on a light tint of the same blue. That was 3.1:1 on the dark page and 4.49:1 on the light one, below the 4.5:1 that WCAG AA asks of normal text. The accent now takes the link colour of each scheme: #60a5fa on dark (6.3:1) and #1d4ed8 on light (5.8:1). `zensical.toml` no longer sets `anchor_linenums`: the site shows no line numbers, yet every code line got an empty link with no name, 122 of them on the getting-started page, on 88 pages in all. Code highlighting and the copy button do not change. A cargo test, `rust/crates/og-dev/tests/docs_theme.rs`, checks both. +- The docs site: the current page in the side menu is readable, and code blocks no longer add a Tab stop per line. The menu paints the current page in the accent colour, #2563eb, on a light tint of the same blue. That was 3.1:1 on the dark page and 4.49:1 on the light one, below the 4.5:1 that WCAG AA asks of normal text. The accent now takes the link colour of each scheme: #60a5fa on dark (6.3:1) and #1d4ed8 on light (5.8:1). On dark, the text Zensical writes on the accent (the "Back to top" button and the side menu button on hover or focus) takes the page colour, #1c1917, at 6.9:1: white on #60a5fa would be 2.5:1. `zensical.toml` no longer sets `anchor_linenums`: the site shows no line numbers, yet every code line got an empty link with no name, 122 of them on the getting-started page, on 88 pages in all. Code highlighting and the copy button do not change. A cargo test, `rust/crates/og-dev/tests/docs_theme.rs`, checks the colours both ways and the anchors. - The self-test of `scripts/ci/check-images.sh` copies the whole tree again once the list of tracked paths passes 128 KiB. It piped that list through `xargs` to `tar`, and GNU `xargs` split a list that long over two `tar` runs; the reading `tar` stopped after the first archive, so Guard's `images` job failed on a broken pipe (exit 123). One `tar` now reads the list itself (`tar --null -T -`). Nothing the check reports changes. - The phased family VCF is indexed by the Rust `og` binary. WhatsHap's step (step 38 and the Nextflow WHATSHAP_PED process) wrote `family.phased.vcf.gz.tbi` with a Python one-liner, `pysam.tabix_index`. It now runs `og vcf index -f`. The `.tbi` keeps its name and format, and its bytes, once inflated, are the ones htslib 1.24 writes (the htslib of the pysam and bcftools the pipeline pins): the same bins in the same order, the same chunks and the same linear index. Only the deflate stream can differ, as two deflate libraries differ. On a 5-million-record three-sample VCF it took 0.75 to 0.91 s and 31 MB at peak, where htslib's own `tabix` took 0.81 to 0.93 s and 19 MB. - As in htslib 1.24, a record ends at the largest of INFO/END, the REF length, the SVLEN of a ``, ``, `` or `` ALT, and a sample's LEN on a gVCF block. A line with no position to read, unsorted records, a sequence that starts again, a record past 2^29 and an empty file stop the index with a message, and nothing is written. Without `-f` it never writes over an index, even one that appears while it runs. diff --git a/docs/stylesheets/theme.css b/docs/stylesheets/theme.css index feee57ca..3b608785 100644 --- a/docs/stylesheets/theme.css +++ b/docs/stylesheets/theme.css @@ -23,12 +23,17 @@ body text; #60a5fa is the same hue at 7.4:1. The current page in the side menu is painted in the accent, on the transparent accent laid over the page (#1d202c). #2563eb there is 3.1:1, - so the accent takes the link colour too: #60a5fa is 6.3:1. */ + so the accent takes the link colour too: #60a5fa is 6.3:1. + Zensical writes --md-accent-bg-color, white, on the accent: the "Back to + top" button and the side menu button on hover or focus. White on #60a5fa + is 2.5:1, so that text takes the page colour: #1c1917 on #60a5fa is + 6.9:1, the same pair the "Copied" note uses. */ [data-md-color-scheme="slate"] { --md-default-bg-color: #1c1917; --md-code-bg-color: #292524; --md-typeset-a-color: #60a5fa; --md-accent-fg-color: #60a5fa; + --md-accent-bg-color: #1c1917; } /* Light scheme: #2563eb on white is 4.5:1, on the edge. #1d4ed8 is the same diff --git a/rust/crates/og-dev/tests/docs_theme.rs b/rust/crates/og-dev/tests/docs_theme.rs index 322b1878..2b20f7c8 100644 --- a/rust/crates/og-dev/tests/docs_theme.rs +++ b/rust/crates/og-dev/tests/docs_theme.rs @@ -5,7 +5,11 @@ //! on a background of `--md-accent-fg-color--transparent` laid over the //! page. For each colour scheme, the accent that scheme ends up with (its //! own block in `docs/stylesheets/theme.css`, else `:root`) must reach -//! WCAG AA for normal text, 4.5:1, against that background. +//! WCAG AA for normal text, 4.5:1, against that background. The reverse +//! holds too: where Zensical fills a control with the accent (the "Back to +//! top" button and the side menu button on hover or focus, a linked tag, +//! the "Copied" note), the text on it, `--md-accent-bg-color` or the page +//! colour, must reach 4.5:1 on the accent. //! 2. `zensical.toml` must not turn on `pymdownx.highlight`'s //! `anchor_linenums`. The site shows no line numbers, so each code line //! would get an empty link with no name that still takes a Tab stop. @@ -141,6 +145,46 @@ fn the_active_menu_item_reads_in_both_schemes() { ); } +/// The colour a scheme ends up with for `name`: its own block, then `:root`, +/// then `fallback` (Zensical's own value when the theme sets none). +fn effective(css: &str, scheme: &str, name: &str, fallback: Option<&str>) -> String { + let root = block(css, ":root").expect("theme.css has a :root block"); + let sel = format!("[data-md-color-scheme=\"{scheme}\"]"); + let own = block(css, &sel).unwrap_or_else(|| panic!("theme.css has no {sel} block")); + prop(own, name) + .or_else(|| prop(root, name)) + .or_else(|| fallback.map(str::to_owned)) + .unwrap_or_else(|| panic!("{sel} sets no {name}")) +} + +#[test] +fn text_on_the_accent_reads_in_both_schemes() { + let css = strip_comments(&read("docs/stylesheets/theme.css")); + let mut low = Vec::new(); + for (scheme, page) in [("slate", None), ("default", Some("#ffffff"))] { + let accent = effective(&css, scheme, "--md-accent-fg-color", None); + // Zensical sets --md-accent-bg-color to #fff for a custom accent. + // The "Back to top" button, the side menu button and linked tags + // paint it on the accent on hover and focus. + let on_accent = effective(&css, scheme, "--md-accent-bg-color", Some("#ffffff")); + // The "Copied" note writes the page colour on the accent. + let page = effective(&css, scheme, "--md-default-bg-color", page); + for (what, text) in [("--md-accent-bg-color", on_accent), ("page colour", page)] { + let ratio = contrast(hex(&text), hex(&accent)); + if ratio < 4.5 { + low.push(format!( + "{scheme}: {what} {text} on {accent} is {ratio:.2}:1" + )); + } + } + } + assert!( + low.is_empty(), + "text on the accent is below WCAG AA (4.5:1): {}", + low.join("; ") + ); +} + #[test] fn the_contrast_maths_matches_known_pairs() { // Black on white is 21:1, and a colour on itself is 1:1. From 8bcdbabada48d9f597ee18e4fe107d546051f5a5 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 18 Nov 2026 13:31:39 +0100 Subject: [PATCH 09/12] docs(privacy): name CPSR 2.3.0 as the checked example and say why the test misses both reports --- docs/why-local.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/why-local.md b/docs/why-local.md index 5d11bddb..2cb172d4 100644 --- a/docs/why-local.md +++ b/docs/why-local.md @@ -46,9 +46,9 @@ A report that loads a file from the internet tells that server your IP address a - **indexcov report (step 16):** loads Chart.js from `cdnjs.cloudflare.com` and jQuery from `code.jquery.com`. Offline, the page opens without its plots. - **Nextflow report and timeline:** load nothing. -The test cannot produce the CPSR and MultiQC reports. On 2026-11-17 we opened one of each in a browser that blocked and logged every request to the internet: +The test's address listing does not see a CPSR or MultiQC report: the test never builds a real CPSR report, and its MultiQC report is written in another job. On 2026-11-17 we opened one of each in a browser that blocked and logged every request to the internet: -- **CPSR report (step 17):** loads nothing. Its stylesheets, scripts and images are inside the HTML file. Its external addresses are plain links to the sources it cites (ClinVar, PanelApp, PubMed and others), which open only when you click them. We checked the CPSR project's own example reports of versions 2.2.5 and 2.4.0; the pinned PCGR 2.3.2 image builds its report from a template that embeds every resource in the same way. +- **CPSR report (step 17):** loads nothing. Its stylesheets, scripts and images are inside the HTML file. Its external addresses are plain links to the sources it cites (ClinVar, PanelApp, PubMed and others), which open only when you click them. We checked the CPSR project's own example reports of versions 2.3.0 and 2.4.0. Version 2.3.0 is the CPSR inside the pinned PCGR 2.3.2 image, and its report template embeds every resource. - **MultiQC report (step 28):** loads nothing. Its external addresses are plain links to MultiQC, Seqera and the papers of the tools it reports. We checked a report from the pinned MultiQC 1.35 image, run with the step's options. **Recommendations for securing your data:** From d5efc9b6b57f0c39608c81ebddf15c06294f09c4 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:01:24 +0100 Subject: [PATCH 10/12] docs(updating): set GENOME_DIR before hashing the step 40 site table --- docs/updating.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/updating.md b/docs/updating.md index a1a9dd51..03bc860e 100644 --- a/docs/updating.md +++ b/docs/updating.md @@ -26,9 +26,10 @@ This page is for maintainers. It says how a pinned version gets bumped, what che | Cyrius lock (step 21) | `CYRIUS_VERSION` in `versions.env` and every wheel in `scripts/cyrius-constraints.txt` | nobody: resolved again by hand in `PYTHON_IMAGE`, as the lock's header says. The freshness report says when PyPI has a newer Cyrius | | The site table of step 40: the 23andMe v5 site list commit, the dbSNP build and the table's hash | `CONSUMER_SITES_COMMIT`, `CONSUMER_DBSNP_BUILD`, their checksums and `CONSUMER_SITES_TABLE_SHA256` in `versions.env` | nobody: dbSNP releases a build every year or two, and the site list has not changed since 2019. A new build, a new commit or a change to `og consumer build-sites` needs one full build (`setup.sh --consumer-sites`, about 30 GB), then record the sha256 of the unpacked table in `CONSUMER_SITES_TABLE_SHA256` (the command is under this table). CI never builds the full table. The monthly freshness report says when NCBI publishes a newer build | -The hash of the site table of step 40 is the sha256 of its unpacked content, because the gzip wrapper holds the time it was written. Run this from the repository root after the build: +The hash of the site table of step 40 is the sha256 of its unpacked content, because the gzip wrapper holds the time it was written. Run this from the repository root after the build, with `GENOME_DIR` set to the folder you gave `setup.sh`: ```bash +export GENOME_DIR=/path/to/your/data . ./versions.env gzip -dc "${GENOME_DIR}/reference/consumer/23andme_v5_sites_dbsnp${CONSUMER_DBSNP_BUILD}.tsv.gz" | sha256sum ``` From 2dc458d801071cdcc0f7411a76eeac93ffe9f720 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:52:34 +0100 Subject: [PATCH 11/12] fix(docs): check borderless tables and real fence closers, and refuse an empty site table hash --- docs/updating.md | 7 +- tests/test_docs_table_pipes.sh | 128 ++++++++++++++++++++++++++------- 2 files changed, 105 insertions(+), 30 deletions(-) diff --git a/docs/updating.md b/docs/updating.md index 03bc860e..ec323751 100644 --- a/docs/updating.md +++ b/docs/updating.md @@ -24,14 +24,15 @@ This page is for maintainers. It says how a pinned version gets bumped, what che | IPD-KIR release (step 8 with `KIR=true`) | `KIR_DB_RELEASE` in `versions.env` | nobody: the monthly freshness report says when a newer release exists | | Parascopy data (step 35) | `PARASCOPY_DATA_VERSION` in `versions.env`, with the Zenodo record and the md5s in `scripts/setup.sh` | nobody: the monthly freshness report says when the record has a newer version | | Cyrius lock (step 21) | `CYRIUS_VERSION` in `versions.env` and every wheel in `scripts/cyrius-constraints.txt` | nobody: resolved again by hand in `PYTHON_IMAGE`, as the lock's header says. The freshness report says when PyPI has a newer Cyrius | -| The site table of step 40: the 23andMe v5 site list commit, the dbSNP build and the table's hash | `CONSUMER_SITES_COMMIT`, `CONSUMER_DBSNP_BUILD`, their checksums and `CONSUMER_SITES_TABLE_SHA256` in `versions.env` | nobody: dbSNP releases a build every year or two, and the site list has not changed since 2019. A new build, a new commit or a change to `og consumer build-sites` needs one full build (`setup.sh --consumer-sites`, about 30 GB), then record the sha256 of the unpacked table in `CONSUMER_SITES_TABLE_SHA256` (the command is under this table). CI never builds the full table. The monthly freshness report says when NCBI publishes a newer build | +| The site table of step 40: the 23andMe v5 site list commit, the dbSNP build and the table's hash | `CONSUMER_SITES_COMMIT`, `CONSUMER_DBSNP_BUILD`, their checksums and `CONSUMER_SITES_TABLE_SHA256` in `versions.env` | nobody: dbSNP releases a build every year or two, and the site list has not changed since 2019. A new build, a new commit or a change to `og consumer build-sites` needs one full build: remove the old table from `reference/consumer/` (`setup.sh` keeps a table it finds), run `setup.sh --consumer-sites` (about 30 GB), then record the sha256 of the unpacked table in `CONSUMER_SITES_TABLE_SHA256` (the command is under this table). CI never builds the full table. The monthly freshness report says when NCBI publishes a newer build | -The hash of the site table of step 40 is the sha256 of its unpacked content, because the gzip wrapper holds the time it was written. Run this from the repository root after the build, with `GENOME_DIR` set to the folder you gave `setup.sh`: +The hash of the site table of step 40 is the sha256 of its unpacked content, because the gzip wrapper holds the time it was written. Run this from the repository root after the build, with `GENOME_DIR` set to the folder you gave `setup.sh`. `gzip -t` stops it when the table is missing or broken, so it never prints the hash of an empty input: ```bash export GENOME_DIR=/path/to/your/data . ./versions.env -gzip -dc "${GENOME_DIR}/reference/consumer/23andme_v5_sites_dbsnp${CONSUMER_DBSNP_BUILD}.tsv.gz" | sha256sum +table="${GENOME_DIR}/reference/consumer/23andme_v5_sites_dbsnp${CONSUMER_DBSNP_BUILD}.tsv.gz" +gzip -t "$table" && gzip -dc "$table" | sha256sum ``` Three pins are floors, not versions that drift: Node 24 in `renovate-dry-run.yml`, Java 17 in the workflows that run Nextflow, and `manifest.nextflowVersion` in `nextflow.config`. Nothing proposes a bump for them, on purpose. diff --git a/tests/test_docs_table_pipes.sh b/tests/test_docs_table_pipes.sh index eee9daf4..16fe3fd4 100644 --- a/tests/test_docs_table_pipes.sh +++ b/tests/test_docs_table_pipes.sh @@ -8,7 +8,10 @@ # table cell names a separator in words. # # Checks: -# 1. a planted page with such a row fails, and the scanner names its line; +# 1. planted pages with such a row fail, and the scanner names the line: +# a bordered table, a double-backtick span, a table without border +# pipes, a span after an escaped backtick, and a table after a fenced +# block that holds a shorter fence line; # 2. a planted page with a pipe only where it renders right passes: a # fenced block, prose, a table cell outside a code span, a code span in # prose, and backticks that do not pair; @@ -27,17 +30,25 @@ fail() { echo "FAIL: $*"; FAILS=$((FAILS + 1)); } pass() { echo "ok: $*"; } # scan FILE...: prints FILE:LINE: TEXT for each table row whose code span holds -# a backslash and a pipe; returns 1 when there is one. A table row starts with -# a pipe. Lines inside ``` or ~~~ fences are skipped. A code span opens with a -# run of N backticks and closes at the next run of exactly N; a run with no -# partner is plain text. +# a backslash and a pipe; returns 1 when there is one. A table starts at a line +# with a pipe followed by a separator line (`|---|---|` or `--- | ---`, with or +# without border pipes) and ends at a blank line; its header and every row are +# checked. A fenced block opens with 3 or more backticks or tildes and closes +# only at a run of the same character at least as long with nothing after it; +# lines inside are skipped. A code span opens with a run of N backticks and +# closes at the next run of exactly N; a run with no partner is plain text, and +# a backtick after a backslash is a literal, not an opener. scan() { awk ' - function bad(line, rest, tmp, n, off, found, code) { + function runlen(s, c, i) { i = 0; while (substr(s, i + 1, 1) == c) i++; return i } + function bad(line, rest, tmp, n, s, k, off, found, code) { rest = line while (match(rest, /`+/)) { - n = RLENGTH - rest = substr(rest, RSTART + n) + n = RLENGTH; s = RSTART + k = 0; while (s - k > 1 && substr(rest, s - k - 1, 1) == "\\") k++ + if (k % 2) { s++; n-- } + rest = substr(rest, s + n) + if (n == 0) continue tmp = rest; off = 0; found = 0 while (match(tmp, /`+/)) { if (RLENGTH == n) { @@ -53,15 +64,47 @@ scan() { } return 0 } - FNR == 1 { fence = 0 } - /^[ \t]*(```|~~~)/ { fence = !fence; next } - fence { next } - /^[ \t]*\|/ && bad($0) { print FILENAME ":" FNR ": " $0; hits++ } + function hit(n, text) { print FILENAME ":" n ": " text; hits++ } + FNR == 1 { fence = 0; table = 0; prev = ""; prevn = 0 } + fence { + t = $0; sub(/^[ \t]*/, "", t); n = runlen(t, fch) + if (n >= flen && substr(t, n + 1) ~ /^[ \t]*$/) fence = 0 + next + } + /^[ \t]*(```|~~~)/ { + t = $0; sub(/^[ \t]*/, "", t); fch = substr(t, 1, 1); flen = runlen(t, fch) + fence = 1; table = 0; prev = "" + next + } + /^[ \t]*$/ { table = 0; prev = ""; next } + table { if (bad($0)) hit(FNR, $0); next } + /\|/ && /^[ \t]*\|?[ \t]*:?-+:?[ \t]*(\|[ \t]*:?-+:?[ \t]*)*\|?[ \t]*$/ && index(prev, "|") > 0 { + table = 1 + if (bad(prev)) hit(prevn, prev) + next + } + { prev = $0; prevn = FNR } END { exit hits > 0 } ' "$@" } -# 1. A planted row like the old Nextflow line fails, and the output names it. +# expect_hit NAME LINE WHAT: the planted page NAME must fail, and the scanner +# must name exactly one row, at line LINE. A scanner error that names no row +# is a failure too. +expect_hit() { + local name=$1 line=$2 what=$3 out rc=0 + out=$(scan "${WORK}/${name}" 2>&1) || rc=$? + if [ "$rc" -eq 0 ]; then + fail "${what} passed" + elif [ "$(grep -c "^${WORK}/${name}:" <<<"$out")" -eq 1 ] && grep -q "^${WORK}/${name}:${line}: " <<<"$out"; then + pass "${what} fails and is named" + else + fail "${what} failed, but the scanner did not name only line ${line}: ${out}" + fi +} + +# 1. Planted rows that must fail. +# A row like the old Nextflow line. cat > "${WORK}/bad.md" <<'MD' # Planted @@ -70,23 +113,46 @@ cat > "${WORK}/bad.md" <<'MD' | Docker | from docs.docker.com | | Nextflow | `curl -s https://get.nextflow.io \| NXF_VER=1.0.0 bash` | MD -if out=$(scan "${WORK}/bad.md"); then - fail "a planted table row with an escaped pipe in a code span passed" -elif grep -qF "bad.md:6: | Nextflow |" <<<"$out"; then - pass "a planted table row with an escaped pipe in a code span fails and is named" -else - fail "the planted row failed but its line was not named: ${out}" -fi +expect_hit bad.md 6 "a planted table row with an escaped pipe in a code span" # Double backticks open a span too. cat > "${WORK}/double.md" <<'MD' +| a | b | +|---|---| | a | ``x \| y`` | MD -if scan "${WORK}/double.md" >/dev/null; then - fail "an escaped pipe in a double-backtick span passed" -else - pass "an escaped pipe in a double-backtick span fails" -fi +expect_hit double.md 3 "an escaped pipe in a double-backtick span" + +# A table without border pipes is a table too. +cat > "${WORK}/borderless.md" <<'MD' +Tool | Install +--- | --- +Docker | from docs.docker.com +Nextflow | `curl -s https://get.nextflow.io \| bash` +MD +expect_hit borderless.md 4 "an escaped pipe in a code span of a table without border pipes" + +# An escaped backtick is a literal, so the span starts at the next backtick. +cat > "${WORK}/escaped.md" <<'MD' +| a | b | +|---|---| +| a | \`example `x \| y` | +MD +expect_hit escaped.md 3 "an escaped pipe in a code span after an escaped backtick" + +# A shorter fence line inside a four-backtick block does not close it, so the +# row inside stays skipped and the table after the block is checked. +cat > "${WORK}/fence.md" <<'MD' +````markdown +``` +| `inside the block \| here` | +```` + +| a | b | +|---|---| +| a | `x \| y` | +MD +expect_hit fence.md 8 "an escaped pipe in a table after a block with a shorter fence line" # 2. Pipes where they render right pass. cat > "${WORK}/good.md" <<'MD' @@ -100,6 +166,9 @@ Prose may say `a \| b` outside a table. | choice | `left` \| `right` | | stray | a lone `` run, then \| and `code` | +A line that starts with a pipe but has no separator under it is not a table: +| `not a table \| row` | + ```bash dmesg | grep -i "oom\|killed" | `not a table \| row` | @@ -115,8 +184,13 @@ else fail "a page with no bad row failed: ${out}" fi -# 3. The real docs. -mapfile -t pages < <(find "$DOCS" -name '*.md' -type f | sort) +# 3. The real docs. A find that cannot read part of the tree fails the test, +# so a partial list never passes as a clean scan. +if ! find "$DOCS" -name '*.md' -type f > "${WORK}/pages.raw"; then + fail "could not list the Markdown pages under ${DOCS}" +fi +pages=() +while IFS= read -r page; do pages+=("$page"); done < <(sort "${WORK}/pages.raw") if [ "${#pages[@]}" -eq 0 ]; then fail "no Markdown pages under ${DOCS}" elif out=$(scan "${pages[@]}"); then From 65253be3136eebf352047d0ff5ffde55f817ac54 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 18 Nov 2026 16:12:33 +0100 Subject: [PATCH 12/12] ci(guard): stop timing out the rust and parity jobs on a slow runner The rust job took 23 minutes on main (limit 30) and parity 19 (limit 20). The same trees on the geiresx fork ran past both limits, twice, so a docs-only PR (#494) could not get a green ci-ok. Raise rust to 45 and parity to 35: the limit is there to stop a hung job, not a slow one. --- .github/workflows/guard.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index d203d466..a2818dd1 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -257,11 +257,12 @@ jobs: # unit-tests is close to its 10 minutes. Format, lint, unit tests with a # planted failure that must go red, pycompat against CPython, then the two # musl builds as scripts/setup.sh --og makes them and the stale-binary - # guard on the real binaries. It takes 15 to 18 minutes on a free runner, - # and a busy runner pushed it past 20, so the limit is 30. + # guard on the real binaries. It took 15 to 18 minutes on a free runner and + # 23 on main on 2026-11-18, and the same tree on the geiresx fork ran past + # 30 twice, so the limit is 45: it stops a hung job, not a slow one. rust: runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 45 defaults: run: working-directory: rust @@ -848,10 +849,11 @@ jobs: # planted kind of difference must be reported, by oracle-diff.sh, by # og dev parity-compare and by bin/og's shadow mode), then every case of # tests/parity/cases.tsv with the oracle in its production image, then - # tests/test_fixitfelix.sh with og. + # tests/test_fixitfelix.sh with og. It took 19 minutes on main on + # 2026-11-18 and ran past 20 twice on the geiresx fork, so the limit is 35. parity: runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 35 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1