From 2142fd01802d90acf3ac28f11b6d43c66b2d6d58 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 20:55:28 +0100 Subject: [PATCH 1/5] test(docs): fail when a docs table puts an escaped pipe inside a code span The docs site prints a code span as written, so a table cell that escapes a pipe for GitHub shows the reader a backslash, and the copied command breaks. The check finds every such row under docs/ and proves itself on planted pages. Bead: K013 Bead: K020 --- tests/test_docs_table_pipes.sh | 131 +++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 tests/test_docs_table_pipes.sh diff --git a/tests/test_docs_table_pipes.sh b/tests/test_docs_table_pipes.sh new file mode 100644 index 00000000..588e61d4 --- /dev/null +++ b/tests/test_docs_table_pipes.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# A command inside a code span in a docs table must not hold an escaped pipe. +# +# GitHub needs `\|` to keep a pipe inside a table cell, but the docs site +# prints a code span as written, so the reader sees and copies the backslash: +# `curl -s https://get.nextflow.io \| NXF_VER=... bash` is not a pipeline. +# A command that needs a pipe goes in a code block under the table, and a +# table cell names a separator in words. +# +# Checks: +# 1. a planted page with such a row fails, and the scanner names its line; +# 2. a planted page with a pipe only where it renders right passes: a +# fenced block, prose, a table cell outside a code span, a code span in +# prose, and backticks that do not pair; +# 3. every docs/**/*.md page passes. +# +# Run: bash tests/test_docs_table_pipes.sh [DIR] (DIR defaults to docs/) +set -euo pipefail + +REPO=$(cd "$(dirname "$0")/.." && pwd) +DOCS=${1:-"${REPO}/docs"} +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +FAILS=0 +fail() { echo "FAIL: $*"; FAILS=$((FAILS + 1)); } +pass() { echo "ok: $*"; } + +# scan FILE...: prints FILE:LINE: TEXT for each table row whose code span holds +# a backslash and a pipe; returns 1 when there is one. A table row starts with +# a pipe. Lines inside ``` or ~~~ fences are skipped. A code span opens with a +# run of N backticks and closes at the next run of exactly N; a run with no +# partner is plain text. +scan() { + awk ' + function bad(line, rest, tmp, n, off, found, code) { + rest = line + while (match(rest, /`+/)) { + n = RLENGTH + rest = substr(rest, RSTART + n) + tmp = rest; off = 0; found = 0 + while (match(tmp, /`+/)) { + if (RLENGTH == n) { + code = substr(rest, 1, off + RSTART - 1) + rest = substr(tmp, RSTART + RLENGTH) + found = 1 + break + } + off += RSTART + RLENGTH - 1 + tmp = substr(tmp, RSTART + RLENGTH) + } + if (found && index(code, "\\|") > 0) return 1 + } + return 0 + } + FNR == 1 { fence = 0 } + /^[ \t]*(```|~~~)/ { fence = !fence; next } + fence { next } + /^[ \t]*\|/ && bad($0) { print FILENAME ":" FNR ": " $0; hits++ } + END { exit hits > 0 } + ' "$@" +} + +# 1. A planted row like the old Nextflow line fails, and the output names it. +cat > "${WORK}/bad.md" <<'MD' +# Planted + +| Software | Install | +|---|---| +| Docker | from docs.docker.com | +| Nextflow | `curl -s https://get.nextflow.io \| NXF_VER=1.0.0 bash` | +MD +if out=$(scan "${WORK}/bad.md"); then + fail "a planted table row with an escaped pipe in a code span passed" +elif grep -qF "bad.md:6: | Nextflow |" <<<"$out"; then + pass "a planted table row with an escaped pipe in a code span fails and is named" +else + fail "the planted row failed but its line was not named: ${out}" +fi + +# Double backticks open a span too. +printf '| a | ``x \\| y`` |\n' > "${WORK}/double.md" +if scan "${WORK}/double.md" >/dev/null; then + fail "an escaped pipe in a double-backtick span passed" +else + pass "an escaped pipe in a double-backtick span fails" +fi + +# 2. Pipes where they render right pass. +cat > "${WORK}/good.md" <<'MD' +# Planted + +Prose may say `a \| b` outside a table. + +| Column | Meaning | +|---|---| +| `readings` | `KEY=text`, separated by a vertical bar | +| choice | `left` \| `right` | +| stray | a lone `` run, then \| and `code` | + +```bash +dmesg | grep -i "oom\|killed" +| `not a table \| row` | +``` + +~~~ +| `still fenced \| here` | +~~~ +MD +if out=$(scan "${WORK}/good.md"); then + pass "pipes in fences, prose, cells outside code spans and unpaired backticks pass" +else + fail "a page with no bad row failed: ${out}" +fi + +# 3. The real docs. +mapfile -t pages < <(find "$DOCS" -name '*.md' -type f | sort) +if [ "${#pages[@]}" -eq 0 ]; then + fail "no Markdown pages under ${DOCS}" +elif out=$(scan "${pages[@]}"); then + pass "no docs table row has an escaped pipe in a code span (${#pages[@]} pages)" +else + echo "$out" + fail "docs table rows with an escaped pipe in a code span; move the command into a code block under the table" +fi + +if [ "$FAILS" -gt 0 ]; then + echo "${FAILS} check(s) failed" + exit 1 +fi +echo "All checks passed." From ec9079651b54ae60ab433fe35fb39c688234d732 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 20:57:45 +0100 Subject: [PATCH 2/5] fix(docs): make the Nextflow install and three other table commands copy and run as written The docs site printed the backslash of each escaped pipe inside a table code span, so the copied command was broken. The Nextflow row of Getting started points at the install command in Full run, the FAQ's BAM build check and the site table hash of Keeping versions up to date move into code blocks under their tables, and the cn-loci readings row names its separator in words. Bead: K013 Bead: K020 --- CHANGELOG.md | 1 + docs/cn-loci.md | 2 +- docs/faq.md | 8 +++++++- docs/getting-started.md | 2 +- docs/updating.md | 9 ++++++++- 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05c3388e..1fd8c9be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Four commands in docs tables now copy and run as written. The docs site printed the backslash that escaped each pipe for GitHub, so the Nextflow install line in Getting started read `curl -s https://get.nextflow.io \| ...`, which is not a pipeline. That row now points at the install command in Full run. The BAM build check of the FAQ and the site table hash of Keeping versions up to date moved into code blocks under their tables, and the `readings` row of Copy Number from Depth names its separator in words. `tests/test_docs_table_pipes.sh` fails when a table row under `docs/` holds an escaped pipe inside a code span. - The self-test of `scripts/ci/check-images.sh` copies the whole tree again once the list of tracked paths passes 128 KiB. It piped that list through `xargs` to `tar`, and GNU `xargs` split a list that long over two `tar` runs; the reading `tar` stopped after the first archive, so Guard's `images` job failed on a broken pipe (exit 123). One `tar` now reads the list itself (`tar --null -T -`). Nothing the check reports changes. - The phased family VCF is indexed by the Rust `og` binary. WhatsHap's step (step 38 and the Nextflow WHATSHAP_PED process) wrote `family.phased.vcf.gz.tbi` with a Python one-liner, `pysam.tabix_index`. It now runs `og vcf index -f`. The `.tbi` keeps its name and format, and its bytes, once inflated, are the ones htslib 1.24 writes (the htslib of the pysam and bcftools the pipeline pins): the same bins in the same order, the same chunks and the same linear index. Only the deflate stream can differ, as two deflate libraries differ. On a 5-million-record three-sample VCF it took 0.75 to 0.91 s and 31 MB at peak, where htslib's own `tabix` took 0.81 to 0.93 s and 19 MB. - As in htslib 1.24, a record ends at the largest of INFO/END, the REF length, the SVLEN of a ``, ``, `` or `` ALT, and a sample's LEN on a gVCF block. A line with no position to read, unsorted records, a sequence that starts again, a record past 2^29 and an empty file stop the index with a message, and nothing is written. Without `-f` it never writes over an index, even one that appears while it runs. diff --git a/docs/cn-loci.md b/docs/cn-loci.md index 43882cab..a2577271 100644 --- a/docs/cn-loci.md +++ b/docs/cn-loci.md @@ -45,7 +45,7 @@ Then the row's `call` column decides: | `mapq0_min`, `mapq0_max` | the MAPQ 0 share guards, or `.` | | `min_control_depth`, `max_control_ratio` | the control guards, or `.` | | `call`, `interval` | as above | -| `readings` | `KEY=text`, separated by `\|`: the text for a count, a class, or `more` for any count above the keys | +| `readings` | `KEY=text`, separated by a vertical bar: the text for a count, a class, or `more` for any count above the keys | | `input` | `bedcov` (this engine) or `cnvpytor-level` (the row records thresholds on CNVpytor's depth level, where 1.0 is two copies) | | `owner` | the engine that calls the locus (`cn-depth`, this one) | | `cross_check` | another engine that also reports the locus (`parascopy:RHCE`, `paraphase:hba`, `cnvpytor:cnvs` for step 18's CNVpytor calls) or `none` | diff --git a/docs/faq.md b/docs/faq.md index 030c6659..d8594c01 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -8,11 +8,17 @@ | "Permission denied" writing output | Container runs as non-root | Add `--user root` to `docker run` (already done in all scripts) | | VEP cache download fails/times out | 26 GB download over unreliable connection | Use `wget -c` (supports resume). See [docs/13-vep-annotation.md](13-vep-annotation.md) | | DeepVariant crashes on Mac | amd64 emulation + memory pressure | `THREADS=2 DV_MEM=12g ./scripts/03-deepvariant.sh `, the sex being `male` or `female`. Will be slow. See [troubleshooting](troubleshooting.md#step-3-deepvariant-crashes-on-mac-amd64-emulation). | -| Wrong number of variants (too few) | Genome build mismatch | Ensure your BAM is aligned to GRCh38 (hg38), not hg19/GRCh37. Check with `samtools view -H your.bam \| grep SN:chr1` | +| Wrong number of variants (too few) | Genome build mismatch | Ensure your BAM is aligned to GRCh38 (hg38), not hg19/GRCh37. Check the BAM header with the command under this table | | 0-byte output files | Missing input or wrong path | Check that all input files exist. Run the script with `bash -x` for debug output. | | "No such image" on `docker pull` | Image name/tag changed | Check the exact image name in the step's documentation. Biocontainer tags change frequently. | | Very slow on macOS | Rosetta 2 emulation overhead | Expected. Consider running on a Linux machine or cloud instance for heavy steps. | +To check the genome build of a BAM, print the chr1 line of its header. GRCh38 has `LN:248956422`, hg19 has `LN:249250621`, and GRCh37 names the contig `1`, so the command prints nothing: + +```bash +samtools view -H your.bam | grep -w SN:chr1 +``` + For detailed solutions, see: - [docs/chip-data-guide.md](chip-data-guide.md) — using 23andMe/MyHeritage/AncestryDNA data with this pipeline - [docs/troubleshooting.md](troubleshooting.md) — comprehensive troubleshooting guide organized by symptom diff --git a/docs/getting-started.md b/docs/getting-started.md index e7223978..744e4815 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -30,7 +30,7 @@ | Docker | 20.10+ | [docs.docker.com/get-docker](https://docs.docker.com/get-docker/) | | bash | 4.4+ | Pre-installed on Linux; macOS ships 3.2, install a newer one with `brew install bash` | | Java *(for a full run)* | 17+ | Any OpenJDK build, e.g. `apt install openjdk-17-jre-headless` or [Adoptium Temurin](https://adoptium.net/) | -| Nextflow *(for a full run)* | `NEXTFLOW_VERSION` in `versions.env` | `curl -s https://get.nextflow.io \| NXF_VER=26.10.5 bash`, see [Full run](#full-run) | +| Nextflow *(for a full run)* | `NEXTFLOW_VERSION` in `versions.env` | The pinned install command in [Full run](#full-run) | | wget or curl | Any | For downloading references | | python3 *(optional)* | 3.6+ | Used by long-read alignment (02b) for symlink resolution. Falls back to `readlink -f` on GNU/Linux if absent | diff --git a/docs/updating.md b/docs/updating.md index 46a888da..a1a9dd51 100644 --- a/docs/updating.md +++ b/docs/updating.md @@ -24,7 +24,14 @@ This page is for maintainers. It says how a pinned version gets bumped, what che | IPD-KIR release (step 8 with `KIR=true`) | `KIR_DB_RELEASE` in `versions.env` | nobody: the monthly freshness report says when a newer release exists | | Parascopy data (step 35) | `PARASCOPY_DATA_VERSION` in `versions.env`, with the Zenodo record and the md5s in `scripts/setup.sh` | nobody: the monthly freshness report says when the record has a newer version | | Cyrius lock (step 21) | `CYRIUS_VERSION` in `versions.env` and every wheel in `scripts/cyrius-constraints.txt` | nobody: resolved again by hand in `PYTHON_IMAGE`, as the lock's header says. The freshness report says when PyPI has a newer Cyrius | -| The site table of step 40: the 23andMe v5 site list commit, the dbSNP build and the table's hash | `CONSUMER_SITES_COMMIT`, `CONSUMER_DBSNP_BUILD`, their checksums and `CONSUMER_SITES_TABLE_SHA256` in `versions.env` | nobody: dbSNP releases a build every year or two, and the site list has not changed since 2019. A new build, a new commit or a change to `og consumer build-sites` needs one full build (`setup.sh --consumer-sites`, about 30 GB), then `gzip -dc \| sha256sum` into `CONSUMER_SITES_TABLE_SHA256`. CI never builds the full table. The monthly freshness report says when NCBI publishes a newer build | +| The site table of step 40: the 23andMe v5 site list commit, the dbSNP build and the table's hash | `CONSUMER_SITES_COMMIT`, `CONSUMER_DBSNP_BUILD`, their checksums and `CONSUMER_SITES_TABLE_SHA256` in `versions.env` | nobody: dbSNP releases a build every year or two, and the site list has not changed since 2019. A new build, a new commit or a change to `og consumer build-sites` needs one full build (`setup.sh --consumer-sites`, about 30 GB), then record the sha256 of the unpacked table in `CONSUMER_SITES_TABLE_SHA256` (the command is under this table). CI never builds the full table. The monthly freshness report says when NCBI publishes a newer build | + +The hash of the site table of step 40 is the sha256 of its unpacked content, because the gzip wrapper holds the time it was written. Run this from the repository root after the build: + +```bash +. ./versions.env +gzip -dc "${GENOME_DIR}/reference/consumer/23andme_v5_sites_dbsnp${CONSUMER_DBSNP_BUILD}.tsv.gz" | sha256sum +``` Three pins are floors, not versions that drift: Node 24 in `renovate-dry-run.yml`, Java 17 in the workflows that run Nextflow, and `manifest.nextflowVersion` in `nextflow.config`. Nothing proposes a bump for them, on purpose. From b777b7d494d85c6cb92a1313b65ea2774d0feb9a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 20:58:48 +0100 Subject: [PATCH 3/5] test(docs): plant the double-backtick row with a heredoc ShellCheck read the single-quoted printf as a missed expansion. Bead: K013 --- tests/test_docs_table_pipes.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_docs_table_pipes.sh b/tests/test_docs_table_pipes.sh index 588e61d4..eee9daf4 100644 --- a/tests/test_docs_table_pipes.sh +++ b/tests/test_docs_table_pipes.sh @@ -79,7 +79,9 @@ else fi # Double backticks open a span too. -printf '| a | ``x \\| y`` |\n' > "${WORK}/double.md" +cat > "${WORK}/double.md" <<'MD' +| a | ``x \| y`` | +MD if scan "${WORK}/double.md" >/dev/null; then fail "an escaped pipe in a double-backtick span passed" else From d5efc9b6b57f0c39608c81ebddf15c06294f09c4 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:01:24 +0100 Subject: [PATCH 4/5] docs(updating): set GENOME_DIR before hashing the step 40 site table --- docs/updating.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/updating.md b/docs/updating.md index a1a9dd51..03bc860e 100644 --- a/docs/updating.md +++ b/docs/updating.md @@ -26,9 +26,10 @@ This page is for maintainers. It says how a pinned version gets bumped, what che | Cyrius lock (step 21) | `CYRIUS_VERSION` in `versions.env` and every wheel in `scripts/cyrius-constraints.txt` | nobody: resolved again by hand in `PYTHON_IMAGE`, as the lock's header says. The freshness report says when PyPI has a newer Cyrius | | The site table of step 40: the 23andMe v5 site list commit, the dbSNP build and the table's hash | `CONSUMER_SITES_COMMIT`, `CONSUMER_DBSNP_BUILD`, their checksums and `CONSUMER_SITES_TABLE_SHA256` in `versions.env` | nobody: dbSNP releases a build every year or two, and the site list has not changed since 2019. A new build, a new commit or a change to `og consumer build-sites` needs one full build (`setup.sh --consumer-sites`, about 30 GB), then record the sha256 of the unpacked table in `CONSUMER_SITES_TABLE_SHA256` (the command is under this table). CI never builds the full table. The monthly freshness report says when NCBI publishes a newer build | -The hash of the site table of step 40 is the sha256 of its unpacked content, because the gzip wrapper holds the time it was written. Run this from the repository root after the build: +The hash of the site table of step 40 is the sha256 of its unpacked content, because the gzip wrapper holds the time it was written. Run this from the repository root after the build, with `GENOME_DIR` set to the folder you gave `setup.sh`: ```bash +export GENOME_DIR=/path/to/your/data . ./versions.env gzip -dc "${GENOME_DIR}/reference/consumer/23andme_v5_sites_dbsnp${CONSUMER_DBSNP_BUILD}.tsv.gz" | sha256sum ``` From 2dc458d801071cdcc0f7411a76eeac93ffe9f720 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:52:34 +0100 Subject: [PATCH 5/5] fix(docs): check borderless tables and real fence closers, and refuse an empty site table hash --- docs/updating.md | 7 +- tests/test_docs_table_pipes.sh | 128 ++++++++++++++++++++++++++------- 2 files changed, 105 insertions(+), 30 deletions(-) diff --git a/docs/updating.md b/docs/updating.md index 03bc860e..ec323751 100644 --- a/docs/updating.md +++ b/docs/updating.md @@ -24,14 +24,15 @@ This page is for maintainers. It says how a pinned version gets bumped, what che | IPD-KIR release (step 8 with `KIR=true`) | `KIR_DB_RELEASE` in `versions.env` | nobody: the monthly freshness report says when a newer release exists | | Parascopy data (step 35) | `PARASCOPY_DATA_VERSION` in `versions.env`, with the Zenodo record and the md5s in `scripts/setup.sh` | nobody: the monthly freshness report says when the record has a newer version | | Cyrius lock (step 21) | `CYRIUS_VERSION` in `versions.env` and every wheel in `scripts/cyrius-constraints.txt` | nobody: resolved again by hand in `PYTHON_IMAGE`, as the lock's header says. The freshness report says when PyPI has a newer Cyrius | -| The site table of step 40: the 23andMe v5 site list commit, the dbSNP build and the table's hash | `CONSUMER_SITES_COMMIT`, `CONSUMER_DBSNP_BUILD`, their checksums and `CONSUMER_SITES_TABLE_SHA256` in `versions.env` | nobody: dbSNP releases a build every year or two, and the site list has not changed since 2019. A new build, a new commit or a change to `og consumer build-sites` needs one full build (`setup.sh --consumer-sites`, about 30 GB), then record the sha256 of the unpacked table in `CONSUMER_SITES_TABLE_SHA256` (the command is under this table). CI never builds the full table. The monthly freshness report says when NCBI publishes a newer build | +| The site table of step 40: the 23andMe v5 site list commit, the dbSNP build and the table's hash | `CONSUMER_SITES_COMMIT`, `CONSUMER_DBSNP_BUILD`, their checksums and `CONSUMER_SITES_TABLE_SHA256` in `versions.env` | nobody: dbSNP releases a build every year or two, and the site list has not changed since 2019. A new build, a new commit or a change to `og consumer build-sites` needs one full build: remove the old table from `reference/consumer/` (`setup.sh` keeps a table it finds), run `setup.sh --consumer-sites` (about 30 GB), then record the sha256 of the unpacked table in `CONSUMER_SITES_TABLE_SHA256` (the command is under this table). CI never builds the full table. The monthly freshness report says when NCBI publishes a newer build | -The hash of the site table of step 40 is the sha256 of its unpacked content, because the gzip wrapper holds the time it was written. Run this from the repository root after the build, with `GENOME_DIR` set to the folder you gave `setup.sh`: +The hash of the site table of step 40 is the sha256 of its unpacked content, because the gzip wrapper holds the time it was written. Run this from the repository root after the build, with `GENOME_DIR` set to the folder you gave `setup.sh`. `gzip -t` stops it when the table is missing or broken, so it never prints the hash of an empty input: ```bash export GENOME_DIR=/path/to/your/data . ./versions.env -gzip -dc "${GENOME_DIR}/reference/consumer/23andme_v5_sites_dbsnp${CONSUMER_DBSNP_BUILD}.tsv.gz" | sha256sum +table="${GENOME_DIR}/reference/consumer/23andme_v5_sites_dbsnp${CONSUMER_DBSNP_BUILD}.tsv.gz" +gzip -t "$table" && gzip -dc "$table" | sha256sum ``` Three pins are floors, not versions that drift: Node 24 in `renovate-dry-run.yml`, Java 17 in the workflows that run Nextflow, and `manifest.nextflowVersion` in `nextflow.config`. Nothing proposes a bump for them, on purpose. diff --git a/tests/test_docs_table_pipes.sh b/tests/test_docs_table_pipes.sh index eee9daf4..16fe3fd4 100644 --- a/tests/test_docs_table_pipes.sh +++ b/tests/test_docs_table_pipes.sh @@ -8,7 +8,10 @@ # table cell names a separator in words. # # Checks: -# 1. a planted page with such a row fails, and the scanner names its line; +# 1. planted pages with such a row fail, and the scanner names the line: +# a bordered table, a double-backtick span, a table without border +# pipes, a span after an escaped backtick, and a table after a fenced +# block that holds a shorter fence line; # 2. a planted page with a pipe only where it renders right passes: a # fenced block, prose, a table cell outside a code span, a code span in # prose, and backticks that do not pair; @@ -27,17 +30,25 @@ fail() { echo "FAIL: $*"; FAILS=$((FAILS + 1)); } pass() { echo "ok: $*"; } # scan FILE...: prints FILE:LINE: TEXT for each table row whose code span holds -# a backslash and a pipe; returns 1 when there is one. A table row starts with -# a pipe. Lines inside ``` or ~~~ fences are skipped. A code span opens with a -# run of N backticks and closes at the next run of exactly N; a run with no -# partner is plain text. +# a backslash and a pipe; returns 1 when there is one. A table starts at a line +# with a pipe followed by a separator line (`|---|---|` or `--- | ---`, with or +# without border pipes) and ends at a blank line; its header and every row are +# checked. A fenced block opens with 3 or more backticks or tildes and closes +# only at a run of the same character at least as long with nothing after it; +# lines inside are skipped. A code span opens with a run of N backticks and +# closes at the next run of exactly N; a run with no partner is plain text, and +# a backtick after a backslash is a literal, not an opener. scan() { awk ' - function bad(line, rest, tmp, n, off, found, code) { + function runlen(s, c, i) { i = 0; while (substr(s, i + 1, 1) == c) i++; return i } + function bad(line, rest, tmp, n, s, k, off, found, code) { rest = line while (match(rest, /`+/)) { - n = RLENGTH - rest = substr(rest, RSTART + n) + n = RLENGTH; s = RSTART + k = 0; while (s - k > 1 && substr(rest, s - k - 1, 1) == "\\") k++ + if (k % 2) { s++; n-- } + rest = substr(rest, s + n) + if (n == 0) continue tmp = rest; off = 0; found = 0 while (match(tmp, /`+/)) { if (RLENGTH == n) { @@ -53,15 +64,47 @@ scan() { } return 0 } - FNR == 1 { fence = 0 } - /^[ \t]*(```|~~~)/ { fence = !fence; next } - fence { next } - /^[ \t]*\|/ && bad($0) { print FILENAME ":" FNR ": " $0; hits++ } + function hit(n, text) { print FILENAME ":" n ": " text; hits++ } + FNR == 1 { fence = 0; table = 0; prev = ""; prevn = 0 } + fence { + t = $0; sub(/^[ \t]*/, "", t); n = runlen(t, fch) + if (n >= flen && substr(t, n + 1) ~ /^[ \t]*$/) fence = 0 + next + } + /^[ \t]*(```|~~~)/ { + t = $0; sub(/^[ \t]*/, "", t); fch = substr(t, 1, 1); flen = runlen(t, fch) + fence = 1; table = 0; prev = "" + next + } + /^[ \t]*$/ { table = 0; prev = ""; next } + table { if (bad($0)) hit(FNR, $0); next } + /\|/ && /^[ \t]*\|?[ \t]*:?-+:?[ \t]*(\|[ \t]*:?-+:?[ \t]*)*\|?[ \t]*$/ && index(prev, "|") > 0 { + table = 1 + if (bad(prev)) hit(prevn, prev) + next + } + { prev = $0; prevn = FNR } END { exit hits > 0 } ' "$@" } -# 1. A planted row like the old Nextflow line fails, and the output names it. +# expect_hit NAME LINE WHAT: the planted page NAME must fail, and the scanner +# must name exactly one row, at line LINE. A scanner error that names no row +# is a failure too. +expect_hit() { + local name=$1 line=$2 what=$3 out rc=0 + out=$(scan "${WORK}/${name}" 2>&1) || rc=$? + if [ "$rc" -eq 0 ]; then + fail "${what} passed" + elif [ "$(grep -c "^${WORK}/${name}:" <<<"$out")" -eq 1 ] && grep -q "^${WORK}/${name}:${line}: " <<<"$out"; then + pass "${what} fails and is named" + else + fail "${what} failed, but the scanner did not name only line ${line}: ${out}" + fi +} + +# 1. Planted rows that must fail. +# A row like the old Nextflow line. cat > "${WORK}/bad.md" <<'MD' # Planted @@ -70,23 +113,46 @@ cat > "${WORK}/bad.md" <<'MD' | Docker | from docs.docker.com | | Nextflow | `curl -s https://get.nextflow.io \| NXF_VER=1.0.0 bash` | MD -if out=$(scan "${WORK}/bad.md"); then - fail "a planted table row with an escaped pipe in a code span passed" -elif grep -qF "bad.md:6: | Nextflow |" <<<"$out"; then - pass "a planted table row with an escaped pipe in a code span fails and is named" -else - fail "the planted row failed but its line was not named: ${out}" -fi +expect_hit bad.md 6 "a planted table row with an escaped pipe in a code span" # Double backticks open a span too. cat > "${WORK}/double.md" <<'MD' +| a | b | +|---|---| | a | ``x \| y`` | MD -if scan "${WORK}/double.md" >/dev/null; then - fail "an escaped pipe in a double-backtick span passed" -else - pass "an escaped pipe in a double-backtick span fails" -fi +expect_hit double.md 3 "an escaped pipe in a double-backtick span" + +# A table without border pipes is a table too. +cat > "${WORK}/borderless.md" <<'MD' +Tool | Install +--- | --- +Docker | from docs.docker.com +Nextflow | `curl -s https://get.nextflow.io \| bash` +MD +expect_hit borderless.md 4 "an escaped pipe in a code span of a table without border pipes" + +# An escaped backtick is a literal, so the span starts at the next backtick. +cat > "${WORK}/escaped.md" <<'MD' +| a | b | +|---|---| +| a | \`example `x \| y` | +MD +expect_hit escaped.md 3 "an escaped pipe in a code span after an escaped backtick" + +# A shorter fence line inside a four-backtick block does not close it, so the +# row inside stays skipped and the table after the block is checked. +cat > "${WORK}/fence.md" <<'MD' +````markdown +``` +| `inside the block \| here` | +```` + +| a | b | +|---|---| +| a | `x \| y` | +MD +expect_hit fence.md 8 "an escaped pipe in a table after a block with a shorter fence line" # 2. Pipes where they render right pass. cat > "${WORK}/good.md" <<'MD' @@ -100,6 +166,9 @@ Prose may say `a \| b` outside a table. | choice | `left` \| `right` | | stray | a lone `` run, then \| and `code` | +A line that starts with a pipe but has no separator under it is not a table: +| `not a table \| row` | + ```bash dmesg | grep -i "oom\|killed" | `not a table \| row` | @@ -115,8 +184,13 @@ else fail "a page with no bad row failed: ${out}" fi -# 3. The real docs. -mapfile -t pages < <(find "$DOCS" -name '*.md' -type f | sort) +# 3. The real docs. A find that cannot read part of the tree fails the test, +# so a partial list never passes as a clean scan. +if ! find "$DOCS" -name '*.md' -type f > "${WORK}/pages.raw"; then + fail "could not list the Markdown pages under ${DOCS}" +fi +pages=() +while IFS= read -r page; do pages+=("$page"); done < <(sort "${WORK}/pages.raw") if [ "${#pages[@]}" -eq 0 ]; then fail "no Markdown pages under ${DOCS}" elif out=$(scan "${pages[@]}"); then