diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e4bb4a..d184177 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,6 +69,8 @@ jobs: with: python-version: "3.12" - run: python scripts/check_file_length.py + # As the publish workflow does, so twine checks the README PyPI will get. + - run: python scripts/replace_readme_diagrams.py - run: python -m pip install build twine - run: python -m build - run: twine check --strict dist/* diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index a40c349..627f0cb 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -37,16 +37,39 @@ jobs: - run: mkdocs build --strict env: NO_MKDOCS_2_WARNING: "1" - - if: github.event_name != 'pull_request' + - if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 with: path: site + diagrams: + # The browser renders Mermaid, so the build cannot see a broken diagram; + # this parses each one in the runner's Chrome against the Mermaid release + # mkdocs.yml pins for the site (ADR 0014). A job of its own: + # the page runs a third-party script, which must not share a machine with + # the site artifact. + name: Mermaid diagrams parse + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + - run: python scripts/check_diagrams.py + deploy: name: publish to GitHub Pages - needs: build - # A pull request or a manual run from a branch must not replace the site. - if: github.event_name != 'pull_request' && github.ref == 'refs/heads/master' + # Waits for the diagrams too: a broken one must not reach the site, even + # through a merge that skipped the pull request checks. An unpkg.com + # outage then holds the deploy back loudly; re-run it when unpkg is up. + needs: [build, diagrams] + # Only a push to master or a manual run from it replaces the site; a pull + # request or a branch never does. + if: >- + github.ref == 'refs/heads/master' && + (github.event_name == 'push' || github.event_name == 'workflow_dispatch') # One deployment at a time, and a running one is never cancelled half-way. concurrency: group: pages diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 239ea21..31a2156 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -43,6 +43,23 @@ jobs: exit 1 } + # The PyPI page links the docs site, so the site shows this release + # before the package goes out (ADR 0013). The deploy job itself must have + # succeeded: a run whose deploy was skipped (a pull request, a manual run + # from the tag) still ends green and proves nothing. + - name: Require the docs site deployed from the tagged commit + env: + GH_TOKEN: ${{ github.token }} + run: | + deployed="" + for run in $(gh api "repos/$GITHUB_REPOSITORY/actions/workflows/docs.yml/runs?head_sha=$GITHUB_SHA" --jq '.workflow_runs[].id'); do + deployed+=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$run/jobs" --jq '.jobs[] | select(.name == "publish to GitHub Pages" and .conclusion == "success") | .id') + done + [ -n "$deployed" ] || { + echo "::error::no docs.yml deploy succeeded for $GITHUB_SHA; wait for the master run or re-run it from master" + exit 1 + } + - name: Setup Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: @@ -54,6 +71,12 @@ jobs: - name: Clean previous builds run: rm -rf dist build *.egg-info + # PyPI cannot render Mermaid: each diagram becomes a link to its section + # of the README on GitHub (ADR 0014). Before the pinning below, which + # pins those links to the tag too. + - name: Replace README diagrams with links + run: python scripts/replace_readme_diagrams.py + # Fails unless run from the tag v (gh workflow run ... --ref vX.Y.Z), # so the package and its README links come from the same release (ADR 0012). - name: Pin README links to the release tag diff --git a/CHANGELOG.md b/CHANGELOG.md index 39dbc16..84e6cca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ All notable changes to this project are documented here. The format follows ## [Unreleased] +## [3.0.2] - 2026-11-11 + +Documentation only: the library code behaves exactly as in 3.0.1. + ### Added - Documentation site at : the README as the user guide, an API reference generated from the source, the @@ -15,6 +19,17 @@ All notable changes to this project are documented here. The format follows - Docstrings on every public name and method, and on each `ErrorKind` member saying what it means. A test fails when a public name has no reference entry or no docstring. +- Diagrams: retries and magic-number scope in the README and user guide; + layers, ports, the request flow and concurrency in the architecture page + (ADR 0014). The site pins one Mermaid release with an integrity hash, and + CI renders every diagram with it. On PyPI, where Mermaid does not render, each one is replaced at + publish with a link to the same diagram in that release's README on GitHub. +- The README links the documentation site right after the description. + +### Changed +- The publish workflow refuses a release until the documentation site has + been deployed from the tagged commit, so the site PyPI links to already + describes the release. ## [3.0.1] - 2026-11-09 diff --git a/README.md b/README.md index e6bddbf..bc10d7f 100644 --- a/README.md +++ b/README.md @@ -224,6 +224,19 @@ magic **0**. Decision and reasoning: | `close_positions(ticket=t)` | Position `t`, whatever its magic: you named it. With `magic=` too, `t` closes only if its magic matches | | `positions()` / `pending_orders()` | Every magic by default: a read cannot close anything | +```mermaid +flowchart TD + call["close_positions() / cancel_pending_orders()"] --> ticket{"ticket= given?
close_positions only"} + ticket -- "yes, no magic=" --> named(["Position ticket, whatever its magic"]) + ticket -- "yes, with magic=" --> both(["Position ticket, only if its magic matches"]) + ticket -- no --> magic{"magic="} + magic -- "omitted or None" --> own(["Only the trader's own magic, from connect()"]) + magic -- "an integer, e.g. 7" --> seven(["Only magic 7"]) + magic -- ANY_MAGIC --> every(["Every magic, manual trades included"]) + own -. "own magic is 0
and something matched" .-> shared["Warning: manual trades
share magic 0"] + own -. "nothing matched while
other magics are open" .-> left["Warning: how many
were left open"] +``` + A magic must be an integer from 0 to 2\*\*64-1 (MT5's ulong). `"42"`, `True`, `1.5`, `-1` and `2**64` are refused as `INVALID_ARGUMENT` instead of matching nothing and reporting an empty success. @@ -337,6 +350,22 @@ position/order in a batch. | refused before sending: no quote yet (`NO_PRICE`, e.g. a symbol just added to Market Watch) | waited for and rebuilt up to `RetryPolicy.attempts`, then `NO_PRICE`; nothing was sent | | refused before sending: anything else | the matching `ErrorKind`, returned at once; nothing was sent | +```mermaid +flowchart LR + build["Build the request
from a fresh quote"] --> sent{"Built
and sent?"} + sent -- "refused before leaving
the process" --> refused(["Err, nothing sent"]) + sent -- "no quote yet" --> left{"Attempts
left?"} + sent -- "lost in IPC" --> unknown(["Err OUTCOME_UNKNOWN
check positions() first"]) + sent -- yes --> answer{"Server
answer"} + answer -- "done, or pending
order placed" --> ok(["Ok(receipt)"]) + answer -- "market deal
partly filled" --> partial(["Err PARTIAL_FILL"]) + answer -- "timeout, no connection,
market deal only queued" --> unknown + answer -- other --> failed(["Err with its ErrorKind"]) + answer -- "requote and alike" --> left + left -- "yes, after delay_seconds" --> build + left -- no --> exhausted(["Err NO_PRICE or
RETRIES_EXHAUSTED"]) +``` + ```python from PythonMetaTrader5 import RetryPolicy, connect @@ -441,6 +470,7 @@ to MetaTrader 5. Details: [docs/architecture.md](https://github.com/Akinzou/Meta | [0011](https://github.com/Akinzou/MetaTrader5-Python/blob/master/docs/adr/0011-close-all-defaults-to-own-magic.md) | Close-all and cancel-all default to the trader's own magic | | [0012](https://github.com/Akinzou/MetaTrader5-Python/blob/master/docs/adr/0012-pin-readme-links-at-publish.md) | Publish from the tag and pin the README's links there | | [0013](https://github.com/Akinzou/MetaTrader5-Python/blob/master/docs/adr/0013-documentation-site.md) | A documentation site built from the README and the source | +| [0014](https://github.com/Akinzou/MetaTrader5-Python/blob/master/docs/adr/0014-mermaid-diagrams.md) | Diagrams as Mermaid, checked in CI and replaced on PyPI | ## Development @@ -459,6 +489,7 @@ The documentation site (ADR 0013) includes this README as its user guide: pip install -e ".[docs]" mkdocs serve # preview at http://127.0.0.1:8000 mkdocs build --strict # what CI runs: broken links and anchors fail +python scripts/check_diagrams.py # every Mermaid diagram parses (needs Chrome or Edge) ``` `tests/contract` compares the unit-test fakes with the installed `MetaTrader5` diff --git a/docs/adr/0013-documentation-site.md b/docs/adr/0013-documentation-site.md index 5d4d1ed..56a579f 100644 --- a/docs/adr/0013-documentation-site.md +++ b/docs/adr/0013-documentation-site.md @@ -1,6 +1,6 @@ # ADR 0013: A documentation site built from the README and the source -- Status: accepted (after 3.0.1, unreleased). Asked for by the package owner on 2026-11-11. +- Status: accepted (3.0.2). Asked for by the package owner on 2026-11-11. ## Context diff --git a/docs/adr/0014-mermaid-diagrams.md b/docs/adr/0014-mermaid-diagrams.md new file mode 100644 index 0000000..1f4b61e --- /dev/null +++ b/docs/adr/0014-mermaid-diagrams.md @@ -0,0 +1,72 @@ +# ADR 0014: Diagrams as Mermaid, checked in CI and replaced on PyPI + +- Status: accepted (3.0.2). Asked for by the package owner on 2026-11-11. + +## Context + +The owner asked for diagrams in the documentation and the README. The README +appears in three places: GitHub, the docs site (ADR 0013) and PyPI, where each +release keeps its README forever (ADR 0012). GitHub and Material for MkDocs +render Mermaid; PyPI does not, and would show a diagram's source. + +Material renders Mermaid in the reader's browser, so `mkdocs build --strict` +never sees a diagram that fails to parse: the live page would show its source +in silence. A `;` inside a sequence diagram label did exactly that while this +record was being written. And by default Material loads whatever Mermaid 11.x +is newest, so a diagram that parsed yesterday can break without a commit. + +## Decision + +- **Diagrams are Mermaid fences in Markdown**, next to the text they explain: + the README (retries, magic-number scope) and `docs/architecture.md` (layers, + ports, request flow, concurrency). The site's palette themes them; there is + no image to regenerate when the code changes. +- **The site pins one Mermaid release with its integrity hash** in + `overrides/main.html` (11.17.2 from unpkg.com, Subresource Integrity as OWASP + ASVS requires for externally hosted scripts). A template, because + `extra_javascript` cannot carry an integrity attribute. Material uses an + already loaded Mermaid instead of fetching the newest 11.x, so a diagram + changes only with a commit, and a changed file on unpkg is refused. +- **`scripts/check_diagrams.py` renders every diagram** in the README, the + changelog, the migration guide and `docs/` with that same file and hash, + read from the template, in headless Chrome with its sandbox on. Rendering, + not only parsing: some errors appear only when a diagram is laid out. + `docs.yml` runs it on every pull request, in a job of its own, apart from + the site artifact, and the deploy waits for it. It judges every Mermaid fence where it starts, and + fails when one does not parse, when one is written in a form it cannot + extract (`~~~`, indented), and when Mermaid never loaded or fewer verdicts + than diagrams came back, because then nothing was checked. +- **`scripts/replace_readme_diagrams.py` swaps each diagram for a link to its + section of the README on GitHub** in the publish workflow. It runs before + the links are pinned (ADR 0012), so the link names the release tag and shows + the diagram of that release, not the site's, which follows master. It fails + on a Mermaid fence it cannot swap, and on a diagram whose match ran past its + own closing fence, rather than ship a README with source in it or text + missing. The CI hygiene job runs it before `twine check`, so the check covers + the README that PyPI receives. +- **`scripts/mermaid_fences.py` holds the one definition** of a diagram and of + a code block that both scripts use, so CI cannot pass a README the publish + then refuses. + +We chose Mermaid over committed SVG or PNG files because an image drifts from +the code it shows, and the regeneration step would need its own check. The +cost is a browser in CI. + +We chose a pinned URL over a copy of Mermaid in the repository because it pins +the version without a vendored file of about 3 MB. The cost: readers' browsers +still fetch it from unpkg.com. + +We chose replacing the diagrams for PyPI over leaving the source there because +a release's README cannot be fixed once it is on PyPI. + +## Consequences + +- Moving to a newer Mermaid means a new URL and hash in the template, checked + like any other pull request: a wrong hash fails the diagram check. +- The diagram check depends on unpkg.com being reachable from CI. When it is + not, the check fails rather than passing unchecked, and the deploy, which + waits for it, does not run: a docs fix waits until unpkg is back and the + run is repeated. We chose that over deploying unchecked diagrams, because + merges can skip the pull request checks (`--admin`). +- On PyPI each diagram is a one-line link to the same diagram in that + release's README on GitHub. diff --git a/docs/architecture.md b/docs/architecture.md index 11eab54..904ddbd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -2,11 +2,17 @@ Clean Architecture with Ports & Adapters. Dependencies point inwards only: -``` -legacy/ (Broker facade) ─┐ -connection.py (connect) ─┼─► application/ ─► domain/ ─► result.py -adapters/ ───────────────┘ ▲ - (implement the ports) ─────────┘ +```mermaid +flowchart LR + connect["connect()"] --> trader + broker["Broker
2.x facade"] --> trader + trader["application/
Trader · execute()"] --> domain["domain/
trading rules"] + domain --> result["result.py
Ok · Err"] + trader --> ports{{"Ports
Terminal · Clock"}} + adapters["adapters/
MetaTrader5Terminal
SystemClock"] -. implement .-> ports + adapters --> mt5[("terminal64.exe")] + connect -. wires .-> adapters + broker -. "default
adapters" .-> adapters ``` | Layer | Package | Knows about | I/O | @@ -25,6 +31,29 @@ There are exactly two, one per process boundary - `Terminal`: the MetaTrader 5 terminal (IPC to `terminal64.exe`). - `Clock`: waiting between retries. +```mermaid +classDiagram + direction LR + class Terminal { + <> + connect(credentials) Result + shutdown() + symbol(name) Result~SymbolSpec~ + tick(name) Result~Tick~ + send(request) Result~TradeReceipt~ + positions(symbol, ticket) Result + pending_orders(symbol) Result + } + class Clock { + <> + sleep(seconds) + } + Terminal <|.. MetaTrader5Terminal : production + Terminal <|.. FakeTerminal : unit tests + Clock <|.. SystemClock : production + Clock <|.. FakeClock : unit tests +``` + Pure logic (price normalisation, stop resolution, retcode classification, volume checks) is plain functions and methods on domain objects, not ports. @@ -40,6 +69,44 @@ volume checks) is plain functions and methods on domain objects, not ports. ([ADR 0003](adr/0003-bounded-retries.md)): success → `Ok`; retryable → sleep and rebuild with a fresh quote; anything else → `Err` with the receipt. +```mermaid +sequenceDiagram + autonumber + actor Bot + participant T as Trader + participant D as Domain rules + participant P as Terminal port + participant S as Trade server + participant C as Clock port + Bot->>T: open_position(...) + T->>D: check the caller's values + T->>P: symbol(name) + T->>D: check volume and price levels against the spec + alt refused + T-->>Bot: Err, nothing sent + else accepted + loop at most RetryPolicy.attempts + T->>P: tick(name) + T->>D: build the request, point stops from this quote + alt request built + T->>P: send(request) + P->>S: order_send + S-->>P: retcode + P-->>T: TradeReceipt + T->>D: classify(retcode) + else no quote yet + Note over T,D: nothing sent, retried below + else other refusal while building + T-->>Bot: Err, nothing sent + end + opt retryable answer, or no quote yet + T->>C: sleep(delay_seconds) + end + end + T-->>Bot: Ok(receipt) or Err(TradeError) + end +``` + ## Concurrency `last_error()` in the `MetaTrader5` package is process-global, so the adapter @@ -48,6 +115,23 @@ explains it ([ADR 0009](adr/0009-one-lock-per-process.md)). The lock and the single `MetaTrader5` import live in `adapters/mt5_session.py`, shared with the lazy 2.x namespace re-export ([ADR 0010](adr/0010-keep-the-metatrader5-re-export.md)). +```mermaid +sequenceDiagram + participant A as Webhook thread A + participant B as Webhook thread B + participant L as Process-wide lock + participant M as MetaTrader5 module + A->>L: acquire + B-->>L: acquire, waits + A->>M: order_send(request) + A->>M: last_error() + Note over A,M: the error read belongs to A's call + A->>L: release + L-->>B: acquired + B->>M: positions_get() + B->>L: release +``` + ## Errors Expected failures travel as `Result` values diff --git a/mkdocs.yml b/mkdocs.yml index c9f3ae7..00dd525 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -13,6 +13,8 @@ copyright: Copyright © Wiktor Jelen · CC BY-NC 4.0 · Trading carries risk theme: name: material + # The pinned Mermaid with its integrity hash (ADR 0014). + custom_dir: overrides icon: repo: fontawesome/brands/github logo: material/finance @@ -86,7 +88,14 @@ markdown_extensions: - toc: permalink: true - pymdownx.details - - pymdownx.superfences + # Mermaid as rendered by Material, themed from the site palette. GitHub + # renders the same blocks; for PyPI, which cannot, the publish workflow + # swaps each one for a link to it on GitHub (ADR 0014). + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format - pymdownx.inlinehilite - pymdownx.highlight: anchor_linenums: true @@ -134,5 +143,6 @@ nav: - adr/0011-close-all-defaults-to-own-magic.md - adr/0012-pin-readme-links-at-publish.md - adr/0013-documentation-site.md + - adr/0014-mermaid-diagrams.md - Migrating from 2.x: MIGRATION.md - Changelog: CHANGELOG.md diff --git a/overrides/main.html b/overrides/main.html new file mode 100644 index 0000000..7e296f2 --- /dev/null +++ b/overrides/main.html @@ -0,0 +1,15 @@ +{#- + The one Mermaid release the site loads, with its Subresource Integrity hash + (ADR 0014, OWASP ASVS): a changed file on unpkg.com is refused by the + browser instead of run on every page. A template, not extra_javascript in + mkdocs.yml, because extra_javascript cannot carry an integrity attribute. + Before Material's bundle: Material then uses this Mermaid instead of + fetching the newest 11.x. scripts/check_diagrams.py reads this tag, so CI + parses the diagrams with the same file and hash. +-#} +{% extends "base.html" %} + +{% block scripts %} + + {{ super() }} +{% endblock %} diff --git a/pyproject.toml b/pyproject.toml index e4f3336..ce3c131 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "PythonMetaTrader5" -version = "3.0.1" +version = "3.0.2" description = "Safe, typed trading layer over the official MetaTrader5 Python API" readme = "README.md" requires-python = ">=3.10" @@ -63,6 +63,9 @@ PythonMetaTrader5 = ["py.typed"] [tool.pytest.ini_options] testpaths = ["tests"] addopts = "-ra --strict-markers" +# Plain imports over per-test importlib loaders: the scripts import each other +# (mermaid_fences, pin_readme_links), which a file loader would not resolve. +pythonpath = ["scripts"] markers = ["integration: needs a running MetaTrader 5 terminal and MT5_* credentials"] [tool.coverage.run] diff --git a/scripts/check_diagrams.py b/scripts/check_diagrams.py new file mode 100644 index 0000000..5b19f69 --- /dev/null +++ b/scripts/check_diagrams.py @@ -0,0 +1,195 @@ +"""Fail when a Mermaid diagram in the docs does not parse (ADR 0014). + +`mkdocs build --strict` cannot catch it: the browser renders the diagrams, and a +broken one shows its source on the live site in silence. This parses each one +with the Mermaid file and hash that overrides/main.html pins for the site, in a headless +Chromium-based browser: Chrome in CI, Edge on Windows. +""" + +from __future__ import annotations + +import functools +import html +import http.server +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import threading +from pathlib import Path + +from mermaid_fences import ANY_FENCE, DIAGRAM + +# The tag the site loads (overrides/main.html): CI parses with the same file, +# under the same integrity hash, so a changed file fails here as it would there. +PINNED = re.compile( + r'" would end the script early. + data = json.dumps(found).replace("<", "\\u003c") + script = SCRIPT.replace("DIAGRAMS", data) + return ( + '
pending
' + f'" + ) + + +def failures(dom: str, expected: int) -> list[str]: + match = OUT.search(dom) + text = html.unescape(match.group(1)) if match else "" + try: + lines = json.loads(text) + except json.JSONDecodeError: + lines = [text] + # Anything but one verdict per diagram (the page left "pending" because + # Mermaid never loaded, a crash half-way) means nothing was proven. + verdicts = isinstance(lines, list) and all( + isinstance(line, str) and line.startswith(("OK ", "FAIL ")) for line in lines + ) + if not verdicts or len(lines) != expected: + return [f"expected {expected} results, got: {lines!r}"] + return [line for line in lines if line.startswith("FAIL ")] + + +class _QuietHandler(http.server.SimpleHTTPRequestHandler): + def log_message(self, format: str, *args: object) -> None: + """One request per run; the verdict is the output that matters.""" + + +def browser() -> str | None: + named = os.environ.get("DIAGRAM_BROWSER") or next(filter(shutil.which, BROWSERS), None) + if named: + return shutil.which(named) or named + return str(EDGE) if EDGE.is_file() else None + + +def render(executable: str, content: str) -> subprocess.CompletedProcess[str]: + with tempfile.TemporaryDirectory() as directory: + Path(directory, "index.html").write_text(content, encoding="utf-8") + # Served over HTTP, not file://, where the browser may refuse the script. + handler = functools.partial(_QuietHandler, directory=directory) + with http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) as server: + threading.Thread(target=server.serve_forever, daemon=True).start() + url = f"http://127.0.0.1:{server.server_address[1]}/index.html" + # The browser sandbox stays on: the page runs a third-party script. + command = [ + executable, + "--headless=new", + "--disable-gpu", + f"--virtual-time-budget={BUDGET_MS}", + "--dump-dom", + url, + ] + try: + # The verdict is the page, not the exit code: a browser that + # failed to start leaves no verdicts, which failures() reports. + # S603: the arguments are this script's own, no untrusted input. + return subprocess.run( # noqa: S603 + command, + check=False, + capture_output=True, + # The DOM is UTF-8 whatever the locale, cp1252 on Windows. + encoding="utf-8", + errors="replace", + timeout=120, + ) + finally: + server.shutdown() + + +def main(argv: list[str]) -> int: + paths = documents(Path()) + skipped = unchecked(paths) + if skipped: + print("\n".join([*skipped, "write each diagram as a ```mermaid fence at the line start"])) + return 1 + found = diagrams(paths) + mermaid = site_mermaid(Path("overrides/main.html")) + if mermaid is None: + print("overrides/main.html: no pinned Mermaid with an integrity hash to check with") + return 1 + executable = browser() + if executable is None: + print("no Chrome, Chromium or Edge found; set DIAGRAM_BROWSER") + return 1 + try: + done = render(executable, page(found, mermaid)) + except (OSError, subprocess.TimeoutExpired) as error: + print(f"could not run {executable}: {error}") + return 1 + problems = failures(done.stdout, expected=len(found)) + if problems and not any(problem.startswith("FAIL ") for problem in problems): + # No verdicts at all: the cause is in the browser's own output. + problems.append(f"browser exit {done.returncode}: {done.stderr[-2000:]}") + print("\n".join(problems) or f"all {len(found)} Mermaid diagrams parse") + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/mermaid_fences.py b/scripts/mermaid_fences.py new file mode 100644 index 0000000..4283185 --- /dev/null +++ b/scripts/mermaid_fences.py @@ -0,0 +1,50 @@ +"""What counts as a Mermaid diagram, and as code, in the docs (ADR 0014). + +One definition for check_diagrams.py (CI) and replace_readme_diagrams.py +(publish): if they disagreed, CI would pass a README that the publish then +refuses on release day, or that reaches PyPI with diagram source in it. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterator + +# The one form both scripts extract: a ``` fence at the start of a line. The +# closing fence may be indented up to three spaces and followed by spaces, and +# may end the file (CommonMark); anything stricter would let the lazy match run +# on to a later fence and take the text in between with it. +DIAGRAM = re.compile( + r"^```mermaid[ \t]*\n(.*?)^ {0,3}`{3,}[ \t]*(?:\n|\Z)", re.MULTILINE | re.DOTALL +) +# Every form GitHub or Material would still render as a diagram: a tilde or a +# longer fence, or one indented inside an admonition or a tab. `[ \t]`, not +# `\s`: a match must not run on to "mermaid" at the start of a later line. +ANY_FENCE = re.compile(r"^[ \t]*(?:`{3,}|~{3,})[ \t]*mermaid\b", re.MULTILINE) +# Any line that opens or closes a fenced code block. +FENCE = re.compile(r"^ {0,3}(`{3,}|~{3,})(.*)$") + + +def code_flags(lines: list[str]) -> Iterator[bool]: + """For each line, whether it belongs to a fenced code block, fences included. + + A block closes only on a fence of its own character, at least as long and + with nothing after it, so a ``` example inside a ```` block stays code. + """ + opener: str | None = None + for line in lines: + fence = FENCE.match(line) + if opener is None: + if fence: + opener = fence.group(1) + yield fence is not None + continue + closes = ( + fence is not None + and fence.group(1)[0] == opener[0] + and len(fence.group(1)) >= len(opener) + and not fence.group(2).strip() + ) + yield True + if closes: + opener = None diff --git a/scripts/replace_readme_diagrams.py b/scripts/replace_readme_diagrams.py new file mode 100644 index 0000000..fae43aa --- /dev/null +++ b/scripts/replace_readme_diagrams.py @@ -0,0 +1,80 @@ +"""Swap the README's Mermaid diagrams for links to them on GitHub at publish (ADR 0014). + +GitHub and the docs site render Mermaid; PyPI does not, and would show each +diagram as its source. A fence this script cannot swap fails the publish +instead of shipping that source, because PyPI keeps a release's README forever. + +The links say `blob/master` and point at the diagram's own section. The publish +workflow runs this before pin_readme_links.py, which pins them to the release +tag, so a release on PyPI links the diagrams of that release. The docs site +follows master and could show a later one. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +from mermaid_fences import ANY_FENCE, DIAGRAM, FENCE, code_flags +from pin_readme_links import REPO + +# From the pinning step, which must recognise these links to pin them. +README = f"{REPO}/blob/master/README.md" +HEADING = re.compile(r"#{1,6} +(.+?)$") + + +def anchor(heading: str) -> str: + """GitHub's anchor for a heading, without the suffix of a repeated one.""" + text = re.sub(r"[ \t]+#+[ \t]*$", "", heading.strip()) # closing hashes + text = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", text) # a link keeps its text + kept = re.sub(r"[^\w\- ]", "", text.lower()) + return kept.replace(" ", "-") + + +def section(text: str, end: int) -> str | None: + """Anchor of the last heading before `end`; a repeated one gets GitHub's `-1`.""" + # Headings only outside code: a `# comment` in a bash or python block + # would give a link to a section that does not exist. + lines = text[:end].splitlines() + seen: dict[str, int] = {} + found = None + for line, in_code in zip(lines, code_flags(lines), strict=True): + heading = None if in_code else HEADING.match(line) + if heading: + base = anchor(heading.group(1)) + repeat = seen.get(base, 0) + seen[base] = repeat + 1 + found = f"{base}-{repeat}" if repeat else base + return found + + +def link(text: str, match: re.Match[str]) -> str: + # A fence inside the diagram means the match ran past its own closing + # fence and took other text with it; that text would vanish from PyPI. + if any(FENCE.match(line) for line in match.group(1).splitlines()): + raise ValueError(f"a diagram swallowed a code block near: {match.group(0)[:80]!r}") + anchor_ = section(text, match.start()) + target = f"{README}#{anchor_}" if anchor_ else README + return f"*This diagram is shown in the [README on GitHub]({target}).*\n" + + +def main(argv: list[str]) -> int: + readme = Path("README.md") + # Bytes: text mode would turn LF into CRLF when run on Windows. + text = readme.read_bytes().decode("utf-8") + try: + replaced, count = DIAGRAM.subn(lambda match: link(text, match), text) + except ValueError as error: + print(f"README.md: {error}") + return 1 + if ANY_FENCE.search(replaced): + print("README.md: a Mermaid fence this script cannot replace; use ```mermaid") + return 1 + readme.write_bytes(replaced.encode("utf-8")) + print(f"README.md: {count} diagram(s) replaced with a link to GitHub") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tests/unit/test_check_diagrams.py b/tests/unit/test_check_diagrams.py new file mode 100644 index 0000000..7dad283 --- /dev/null +++ b/tests/unit/test_check_diagrams.py @@ -0,0 +1,136 @@ +"""Every Mermaid diagram must parse: the docs build cannot see a broken one.""" + +from __future__ import annotations + +from pathlib import Path + +import check_diagrams as check +import pytest + +ROOT = Path(__file__).resolve().parents[2] +URL = "https://unpkg.com/mermaid@11.17.2/dist/mermaid.min.js" +TAG = f'' + + +def test_diagrams_are_found_with_their_file_and_position(tmp_path: Path) -> None: + page = tmp_path / "page.md" + page.write_text( + "# T\n\n```mermaid\nflowchart LR\n a --> b\n```\n\n```python\nx = 1\n```\n" + "\n```mermaid\nsequenceDiagram\n A->>B: hi\n```\n", + encoding="utf-8", + ) + found = check.diagrams([page]) + assert [d["id"] for d in found] == [f"{page}#1", f"{page}#2"] + assert found[0]["src"] == "flowchart LR\n a --> b\n" + + +def test_the_repository_documents_carry_diagrams() -> None: + found = check.diagrams(check.documents(ROOT)) + assert any(d["id"].startswith(str(ROOT / "README.md")) for d in found) + assert any("architecture.md" in d["id"] for d in found) + + +def test_every_document_the_site_includes_is_scanned() -> None: + # The site includes the changelog and migration guide from the root. + # docs/CHANGELOG.md and docs/MIGRATION.md are one-line stubs; the text, + # and any diagram in it, lives in the files at the root. + scanned = set(check.documents(ROOT)) + expected = {ROOT / name for name in ("README.md", "CHANGELOG.md", "MIGRATION.md")} + assert expected | {ROOT / "docs" / "architecture.md"} <= scanned + + +def test_a_diagram_the_checker_cannot_extract_is_reported(tmp_path: Path) -> None: + # Material renders a tilde or an indented fence too; skipping one would + # leave it unchecked while the run reports that every diagram parses. + page = tmp_path / "page.md" + page.write_text( + "```mermaid\nflowchart LR\n a --> b\n```\n\n" + "!!! note\n\n ```mermaid\n flowchart LR\n c --> d\n ```\n\n" + "~~~mermaid\nflowchart LR\n e --> f\n~~~\n", + encoding="utf-8", + ) + assert check.unchecked([page]) == [ + f"{page}:8: a Mermaid fence written in a form not checked", + f"{page}:13: a Mermaid fence written in a form not checked", + ] + + +def test_a_document_with_only_checkable_fences_is_not_reported(tmp_path: Path) -> None: + page = tmp_path / "page.md" + page.write_text("```mermaid\nflowchart LR\n a --> b\n```\n", encoding="utf-8") + assert check.unchecked([page]) == [] + + +def test_a_multi_line_parse_failure_is_reported_whole() -> None: + dom = '
["OK   a#1", "FAIL b#2: Parse error\\n->>"]
' + assert check.failures(dom, expected=2) == ["FAIL b#2: Parse error\n->>"] + + +def test_all_parsed_is_no_failure() -> None: + dom = '
["OK   a#1", "OK   b#2"]
' + assert check.failures(dom, expected=2) == [] + + +def test_mermaid_that_never_loaded_is_a_failure_not_a_pass() -> None: + # An unreachable CDN leaves the page as it was served: nothing was checked. + dom = '
pending
' + assert check.failures(dom, expected=2) != [] + + +def test_fewer_results_than_diagrams_is_a_failure() -> None: + dom = '
["OK   a#1"]
' + assert check.failures(dom, expected=2) != [] + + +def test_prose_starting_with_mermaid_after_a_fence_is_no_diagram(tmp_path: Path) -> None: + page = tmp_path / "page.md" + page.write_text("```\ncode\n```\n\nmermaid-cli is not needed.\n", encoding="utf-8") + assert check.unchecked([page]) == [] + + +def test_a_browser_that_cannot_start_is_reported_not_raised( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + for name in ("README.md", "CHANGELOG.md", "MIGRATION.md"): + (tmp_path / name).write_text("```mermaid\nflowchart LR\n a --> b\n```\n", encoding="utf-8") + (tmp_path / "docs").mkdir() + (tmp_path / "overrides").mkdir() + (tmp_path / "overrides" / "main.html").write_text(TAG, encoding="utf-8") + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("DIAGRAM_BROWSER", str(tmp_path / "no-such-browser")) + assert check.main([]) == 1 + assert "could not run" in capsys.readouterr().out + + +def test_an_extra_extracted_fence_cannot_hide_an_unextractable_one(tmp_path: Path) -> None: + # Counting fences would pair the example inside ```` with the indented + # diagram and report nothing; each fence is checked where it starts. + page = tmp_path / "page.md" + page.write_text( + "````markdown\n```mermaid\nflowchart LR\n a --> b\n```\n````\n\n" + "!!! note\n\n ```mermaid\n flowchart LR\n c --> d\n ```\n", + encoding="utf-8", + ) + assert check.unchecked([page]) == [f"{page}:10: a Mermaid fence written in a form not checked"] + + +def test_a_closing_script_tag_in_a_label_cannot_break_the_page() -> None: + diagram = {"id": "a#1", "src": 'flowchart LR\n a[""] --> b\n'} + html = check.page([diagram], (URL, "sha384-abc")) + assert html.count("") == 2 + + +def test_the_checked_mermaid_is_the_one_the_site_loads() -> None: + pinned = check.site_mermaid(ROOT / "overrides" / "main.html") + assert pinned is not None + url, integrity = pinned + assert url.startswith("https://unpkg.com/mermaid@11.") + assert url.endswith("/dist/mermaid.min.js") + assert integrity.startswith("sha384-") + + +def test_the_check_loads_mermaid_under_the_same_integrity_hash() -> None: + # A changed file on unpkg is then refused in CI as it is on the site, and + # the check fails instead of parsing with code nobody reviewed. + html = check.page([], (URL, "sha384-abc")) + assert f'