diff --git a/.claude/skills/review-open-issues/SKILL.md b/.claude/skills/review-open-issues/SKILL.md index 5bf4e48..b7fb442 100644 --- a/.claude/skills/review-open-issues/SKILL.md +++ b/.claude/skills/review-open-issues/SKILL.md @@ -216,8 +216,12 @@ pull request, so it can be red without blocking anything: python scripts/fleet/check_cards.py # card headline figures vs each Mech's site ``` -A drifted card therefore shows up as a failed scheduled run, not a failed PR -check. Look at the latest scheduled run before crediting the cards as current. +A stale card therefore shows up as a failed scheduled run, not a failed PR +check. A card behind a fast Mech only warns ("grew") for 14 days after the +refresh's pins, and only while its site is at most half as large again as the +card; past either limit it fails as STALE (#263). So a green run does not mean +every card equals its site: read the run's log, not just its colour, before +crediting the cards as current. An issue asserting a defect that one of these already blocks is P2 unless it shows the gate is porous — and they have been porous: a test can pass because diff --git a/.claude/skills/update-xmech-page/SKILL.md b/.claude/skills/update-xmech-page/SKILL.md index 1f35615..339f511 100644 --- a/.claude/skills/update-xmech-page/SKILL.md +++ b/.claude/skills/update-xmech-page/SKILL.md @@ -232,8 +232,14 @@ Rewrite `_fleet/data/site_audit.json` for the run: per repository the pinned test compares a value with itself, #125) and its commit date, the URL each card figure is read from, the figure, the sha256 of the fetched HTML and of any data file, merged PRs, and short notes on how the site figure relates to the repo count. Set `checked_at_utc`, -`local_date`, `pinned_at_utc` and `scope`. Derive the mechanical fields rather -than typing them: the figure through `check_cards.published()`, merged PRs from +`local_date`, `pinned_at_utc` (ISO, with its offset) and `scope`. Record +`figure_at_pin` for every source with a committed copy: read the copy at the pin +with `check_cards.figure()`, the nightly's own parser with `REGIONS` applied, +never from the template. The provenance tests require it to equal each card, so +a mistyped card fails on the PR, and the nightly reports WRONG from it (#231). +ProteinTraitsMech's data file is built in CI and has none. Derive the other +mechanical fields rather than typing them: the live figure through +`check_cards.read_source()`, merged PRs from `mech_stats.json`, SHAs and commit dates from the pins, and assert that the pins equal the stats' `source_revision` before writing. Hash the served page as committed at the pin too (`git show :pages/index.html`, or `docs/`), record @@ -253,7 +259,7 @@ python3 scripts/fleet/assemble_page.py python3 -m unittest discover -s tests -v python3 scripts/fleet/assemble_page.py --check python3 scripts/fleet/refresh_manifest.py --claw-root "$SNAP/claw" --check -python3 scripts/fleet/check_cards.py # 0 drifted, except sites that moved past their pin (below) +python3 scripts/fleet/check_cards.py # exit 0; "grew" lines are sites that moved past their pin (below) ``` Rerun `check_cards.py` immediately before opening the PR and again before any @@ -264,8 +270,12 @@ When a site has moved past its pin, do not re-pin that one Mech: the census and overlaps are computed across Mechs, so a single re-pin is a partial rerun, and a fast Mech moves again before the rerun finishes. Keep the page a consistent snapshot at the pins, record the live figure as `site_figure_at_check` in that -Mech's `site_audit.json` entry, and say in the PR which cards will show as -drifted. Re-pin everything only if the drift is large enough to mislead. +Mech's `site_audit.json` entry, and say in the PR which cards the check reports +as grown. That stays a warning for `GRACE_DAYS` (14) after `pinned_at_utc` while +the site is at most half as large again as the card; past either limit the +check reports STALE and fails, and the page is due a full refresh. A card that +differs from its audit's `figure_at_pin` fails as WRONG however far the site has +moved (step 7). Emoji headings render with a leading hyphen in their id on GitHub Pages. Verify anchors against the deployed HTML, not a local kramdown. diff --git a/.github/workflows/fleet-page.yml b/.github/workflows/fleet-page.yml index ee98a95..972170c 100644 --- a/.github/workflows/fleet-page.yml +++ b/.github/workflows/fleet-page.yml @@ -33,5 +33,8 @@ jobs: # are hand-curated from ten sites that publish on their own cadence, so a # blocking check would make an unrelated docs fix unmergeable whenever a # Mech shipped records overnight. Nightly red is the right signal. - - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + # !cancelled() runs it even when an earlier step failed: a mistyped card + # fails the provenance tests first, and the card report, with its WRONG + # or AUDIT line and every other card's verdict, must still print (#239). + - if: ${{ !cancelled() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} run: python scripts/fleet/check_cards.py diff --git a/_fleet/README.md b/_fleet/README.md index f5f7622..5a70da4 100644 --- a/_fleet/README.md +++ b/_fleet/README.md @@ -37,13 +37,44 @@ against the page it cites and is the one script here that needs the network: python3 scripts/fleet/check_cards.py ``` -It exits 1 on a figure that differs from the site and only warns on a page it -could not read, and `SOURCES` at its top pins where each Mech publishes its -count. It runs on the workflow's nightly schedule, not on pull requests, so a -Mech shipping records overnight does not block an unrelated change; a nightly -red means the card figures in `mechs_template.md` and the `MECHS` block in -`fleet_fragment.html` need refreshing together. A new card needs a `SOURCES` -entry; a test enforces that. +`SOURCES` at its top pins where each Mech publishes its count, and `REGIONS` +restricts a source to the part that states it where the same words appear +elsewhere (CultureMech's generated README block). The page is a snapshot at a +refresh's pins, so a site ahead of its card only warns ("grew") for +`GRACE_DAYS` (14) after the pins in `site_audit.json`, and only while the site is +at most `MAX_LEAD` (50%) ahead. Past either limit it fails as STALE. It also +fails when a card differs from `figure_at_pin`, the figure `site_audit.json` +records its source stating at the pin (the card was never right), when a site is +behind its card, when a source answers a 4xx other than a throttle or no longer +states a figure the parser can read, when a card lacks exactly one headline +figure or has no `SOURCES` entry, when the audit is missing or malformed, its +pin time is missing, unreadable or in the future, or a source lacks a +whole-number `figure_at_pin` (all but ProteinTraitsMech, whose file is built in +CI), and +when more than half the sources could not be fetched. One site's outage or +throttle only warns (#148, #115, #176, #217-#220, #231, #232). It runs on the +workflow's nightly schedule, not on pull requests, so a Mech shipping records +overnight does not block an unrelated change. The run's closing line names the +remedy for each failing verdict (#235): + +- STALE or SHRANK: the card figures in `mechs_template.md` and the `MECHS` block + in `fleet_fragment.html` need a full refresh, since the page is a snapshot. +- WRONG: a card, or the audit's `figure_at_pin`, was mistyped. Correct the card + and every other occurrence of its figure, found by grepping the tree for it as + step 6 of the update skill does: the MECHS `records:` and `extra:` text in + `fleet_fragment.html`, cross-references, `card_records` in `site_audit.json` + and the pages that repeat it. Or correct `figure_at_pin` if that is what was + wrong. Then rerun `assemble_page.py`; no re-pin (#248, #258). The unit tests catch this on the + PR and in the nightly, which still runs the card check after a failed test + step so its report prints (#239, #240). +- GONE or CHANGED: a `SOURCES` entry needs repointing. +- MARKUP, UNCARDED or AUDIT: fix the card markup, `SOURCES` or `site_audit.json`. +- UNCHECKED: most sites could not be reached; rerun before changing anything. + +A new card needs a `SOURCES` entry; a test enforces that. The cards +are read by `scripts/fleet/card_markup.py`, the one parser the assembler, this +check and the tests share, and the assembler refuses a card without exactly one +headline figure (#114, #218). The `Fleet page` workflow checks pull requests, pushes and the live CLAW manifest daily. It detects changes to membership, capability declarations (including @@ -180,7 +211,10 @@ its site lists 422 communities, while its record glob also takes four isolate records, so the census and `mech_stats.json` count 426. CellStructureMech and TraitMech published new records after the pins were taken; their cards keep the pinned figures, and `site_audit.json` records what the two sites showed when it -was written. `check_cards.py` will report both as drifted until the next refresh. +was written. `check_cards.py` reports both as grown, a warning, until 14 days +after the pins or until a site is half as large again as its card, whichever +comes first; CellStructureMech, adding about two records an hour, reaches the +second within a week. NaturalProductMech's landing page and MediaIngredientMech's data file also changed after the pins without changing their figures; the audit records each live hash beside the hash of the committed copy at the pin. diff --git a/_fleet/data/site_audit.json b/_fleet/data/site_audit.json index 0ac5081..9488317 100644 --- a/_fleet/data/site_audit.json +++ b/_fleet/data/site_audit.json @@ -10,6 +10,7 @@ "commit_date": "2026-09-25T01:30:31Z", "readme_url": "https://github.com/CultureBotAI/AntibioticMech/blob/66d68c2a8d99230ab729267e6cd8ae1051b85225/README.md", "card_records": 2939, + "figure_at_pin": 2939, "merged_prs": 422, "site": "https://culturebotai.github.io/AntibioticMech/pages/index.html", "site_html_sha256": "426d439768e46f5e3c65b0241762359d06dffb3b217a3bba01e2552b65991ae9", @@ -22,6 +23,7 @@ "commit_date": "2026-09-25T01:58:24Z", "readme_url": "https://github.com/CultureBotAI/CellStructureMech/blob/42cc23b90b6c48c49efd94b7ee6efb808dded7e6/README.md", "card_records": 542, + "figure_at_pin": 542, "merged_prs": 611, "site": "https://culturebotai.github.io/CellStructureMech/pages/index.html", "site_html_sha256": "aa3a2b8d51d6cd1bbac33c472b0b40279538779c514f384437f818b3c665d679", @@ -35,6 +37,7 @@ "commit_date": "2026-09-24T14:00:15Z", "readme_url": "https://github.com/CultureBotAI/CommunityMech/blob/8505a56d644b02fe87f776be9d667db4cf3f5c5d/README.md", "card_records": 422, + "figure_at_pin": 422, "merged_prs": 578, "site": "https://culturebotai.github.io/CommunityMech/", "site_html_sha256": "5c7aba370a7e7bd50590eb16005c4e3bdd31b1f17d933b8106de7506d67d07dd", @@ -54,6 +57,7 @@ "commit_date": "2026-09-22T06:24:22Z", "readme_url": "https://github.com/CultureBotAI/CultureMech/blob/faaf033b8c5386aaf2cb6f28fc527678d00cf83b/README.md", "card_records": 6288, + "figure_at_pin": 6288, "merged_prs": 253, "site": "https://raw.githubusercontent.com/CultureBotAI/CultureMech/main/README.md", "site_html_sha256": "9bc06d232a67fa55cdef539fec2f52fa2644f7cf50e2820e223cc26106ee6f51", @@ -66,6 +70,7 @@ "commit_date": "2026-09-25T01:40:32Z", "readme_url": "https://github.com/CultureBotAI/HabitatMech/blob/b16e3099478a7ea55b82142836e9a118ae062cf6/README.md", "card_records": 3206, + "figure_at_pin": 3206, "merged_prs": 417, "site": "https://culturebotai.github.io/HabitatMech/pages/index.html", "site_html_sha256": "53564300e6d0aaf2a5b1144fceab088024586941f197113c57c42c17eefbc8e9", @@ -78,6 +83,7 @@ "commit_date": "2026-09-24T08:38:41Z", "readme_url": "https://github.com/CultureBotAI/MediaIngredientMech/blob/dfce1c9342ca7aec41b50d0f6db8adb950bb72fe/README.md", "card_records": 2953, + "figure_at_pin": 2953, "merged_prs": 379, "site": "https://culturebotai.github.io/MediaIngredientMech/", "site_html_sha256": "e98a1ac81f8faafad3aae291782b32e22e78e8f20cd822effc4cba4c22ce7b09", @@ -92,6 +98,7 @@ "commit_date": "2026-09-24T07:48:18Z", "readme_url": "https://github.com/CultureBotAI/NaturalProductMech/blob/36662aa8fcdcac9d3ddf4acba0f430e39bc2da2e/README.md", "card_records": 3115, + "figure_at_pin": 3115, "merged_prs": 84, "site": "https://culturebotai.github.io/NaturalProductMech/pages/index.html", "site_html_sha256": "28d335df6bfeb065940fad9e5ae783d6cd3ed7eacabd8c46c94ee4a9aecdbabd", @@ -117,6 +124,7 @@ "commit_date": "2026-09-21T08:37:02Z", "readme_url": "https://github.com/CultureBotAI/TaxonMech/blob/972fbd7b85a052c6b95bf1437f672168bd6d8970/README.md", "card_records": 625960, + "figure_at_pin": 625960, "merged_prs": 18, "site": "https://culturebotai.github.io/TaxonMech/pages/index.html", "site_html_sha256": "53aab681e40bafb9326b320858660a4273c7df8b9a19ad561d4730b5b32c6a89", @@ -129,6 +137,7 @@ "commit_date": "2026-09-22T15:30:43Z", "readme_url": "https://github.com/CultureBotAI/TraitMech/blob/a21c5aa34a876c43ea1e8053a58bc63d53be30a1/README.md", "card_records": 763, + "figure_at_pin": 763, "merged_prs": 656, "site": "https://culturebotai.github.io/TraitMech/pages/index.html", "site_html_sha256": "21fd928ae51ddaca7a9f1e5cafce81a37b055c97ab8a83442ba19021acc2daeb", diff --git a/scripts/fleet/assemble_page.py b/scripts/fleet/assemble_page.py index be60cfd..4333968 100644 --- a/scripts/fleet/assemble_page.py +++ b/scripts/fleet/assemble_page.py @@ -6,6 +6,7 @@ from pathlib import Path import re +from card_markup import card_figures, card_names, markup_problems from refresh_manifest import validate REPO = Path(__file__).resolve().parents[2] @@ -50,11 +51,8 @@ def number_word(value: int) -> str: return WORDS[value] if 0 <= value < len(WORDS) else f"{value:,}" -CARD_RECORDS = re.compile(r'
([\d,]+)') - - def fleet_records(template): - """What the Mech cards add up to. + """What the Mech cards add up to, one figure per card. The tile used to carry its own typed figure and drifted away from the cards it was meant to total: it read 448,724 while the ten cards summed to @@ -68,11 +66,21 @@ def fleet_records(template): The ten are not ten counts of the same thing: the cards call theirs taxon records, published recipes, natural product structures and so on. The tile says "curated entries" rather than "records" for that reason (#82). + + Keyed by Mech and read card by card through card_markup, the parser + check_cards.py also uses, so the total and the nightly check cannot read the + markup differently (#114). """ - counts = [int(n.replace(",", "")) for n in CARD_RECORDS.findall(template)] - if not counts: + # Exactly one figure per card, and none outside the cards: a second stat + # tile used to replace a card's headline in the total without failing (#218). + problems = markup_problems(template) + if problems: + raise ValueError("Every Mech card must carry a record count, exactly once: " + + "; ".join(f"{mech}: {why}" for mech, why in problems)) + figures = card_figures(template) + if not figures: raise ValueError("No Mech card record counts found") - return counts + return figures def assemble(template, fragment, data, snapshot, stats, census): @@ -81,7 +89,7 @@ def assemble(template, fragment, data, snapshot, stats, census): badges = re.findall(r"", template) if len(badges) != len(names) or set(badges) != names: raise ValueError("Mech cards must match canonical fleet membership exactly") - cards = re.findall(r']*\bdata-mech="([^"]+)"', template) + cards = card_names(template) if len(cards) != len(names) or set(cards) != names: raise ValueError("Actual Mech cards must match canonical fleet membership exactly") metadata = fragment.split("var MECHS = {", 1)[1].split("\n };", 1)[0] @@ -114,7 +122,7 @@ def assemble(template, fragment, data, snapshot, stats, census): if counted != names: raise ValueError("Mech stats must cover canonical fleet membership exactly") counts = fleet_records(template) - if len(counts) != len(names): + if set(counts) != names: raise ValueError("Every Mech card must carry a record count") # The census is a dated scan, so its vocabulary tally is labelled with its own # run date rather than as current, and its coverage is stated below. @@ -127,7 +135,7 @@ def assemble(template, fragment, data, snapshot, stats, census): tokens = { "": str(len(names)), "": number_word(len(names)), - "": f"{sum(counts):,}", + "": f"{sum(counts.values()):,}", "": f"{len(vocabularies):,}", # "all ten Mechs" once the census reaches every member, which it has # since TaxonMech was added (#87); "nine of the ten Mechs" otherwise. diff --git a/scripts/fleet/card_markup.py b/scripts/fleet/card_markup.py new file mode 100644 index 0000000..10a3e15 --- /dev/null +++ b/scripts/fleet/card_markup.py @@ -0,0 +1,68 @@ +"""The one parser for the Mech cards in _fleet/mechs_template.md. + +assemble_page.py, check_cards.py and the tests used to read the card markup with +three separate regexes, and check_cards.py paired each Mech with the *next* +figure in the file, so a card missing its figure silently took its neighbour's +(#114). Everything now reads the cards here, one
at a time. +""" +from __future__ import annotations + +import collections +import re + +ARTICLE = re.compile(r']*\bdata-mech="([^"]+)"[^>]*>(.*?)
', re.S) +# A card's headline tile, counted whatever it holds, so a malformed second tile +# cannot slip past the one-per-card rule (#261). +TILE = re.compile(r'
') +# The figure in a tile: plain digits, or digits grouped in threes by commas, so +# "," is unreadable rather than int("") (#261) and "32,06" is unreadable +# rather than 3206 (#264). +FIGURE = re.compile(r'
(\d{1,3}(?:,\d{3})+|\d+)') + + +def card_names(template: str) -> list[str]: + """Every card's Mech, in page order, duplicates kept.""" + return [mech for mech, _ in ARTICLE.findall(template)] + + +def markup_problems(template: str) -> list[tuple[str, str]]: + """(Mech, what is wrong) for every card that does not state exactly one figure. + + A card with two figures used to be read as its first, and a figure outside + every card was ignored, so a second stat tile changed the fleet total without + failing anything (#218). "-" stands for a figure that belongs to no card. + """ + problems = [] + for mech, body in ARTICLE.findall(template): + tiles = len(TILE.findall(body)) + if tiles == 0: + problems.append((mech, "card has no headline figure")) + elif tiles > 1: + problems.append((mech, f"card has {tiles} headline figures; it must have exactly one")) + elif not FIGURE.search(body): + problems.append((mech, "card's headline figure is unreadable")) + for mech, count in collections.Counter(card_names(template)).items(): + if count > 1: + # The nightly would check only one of them (#262). + problems.append((mech, f"{count} cards for one Mech")) + outside = len(TILE.findall(ARTICLE.sub("", template))) + if outside: + problems.append(("-", f"{outside} headline figure(s) outside any card")) + return problems + + +def card_figures(template: str) -> dict[str, int]: + """Each card's headline figure, keyed by Mech. + + Read inside the card's own
, so a card without a figure is absent + rather than borrowing the next card's. A card with more than one headline + tile, an unreadable one, or a second card for the same Mech is absent too; + markup_problems() says which cards those are. + """ + names = collections.Counter(card_names(template)) + figures: dict[str, int] = {} + for mech, body in ARTICLE.findall(template): + hits = FIGURE.findall(body) + if names[mech] == 1 and len(TILE.findall(body)) == 1 and len(hits) == 1: + figures[mech] = int(hits[0].replace(",", "")) + return figures diff --git a/scripts/fleet/check_cards.py b/scripts/fleet/check_cards.py index cf8c064..b4439f3 100644 --- a/scripts/fleet/check_cards.py +++ b/scripts/fleet/check_cards.py @@ -12,13 +12,45 @@ that a blocking check would make an unrelated docs fix unmergeable because some Mech published records overnight. -Two failure kinds, kept apart on purpose. A fetch that does not arrive is a -warning, because the network is not the site's fault. A fetch that arrives and -does not match is a failure, because either the number moved or the markup did, -and both need a person. +What fails and what only warns (#148, #115, #176, #217): + + ok the site states the card's figure. + grew the site is ahead of the card, the refresh that pinned the card is + at most GRACE_DAYS old, and the site is at most MAX_LEAD ahead. A + warning: the page is a snapshot at the refresh's pins, and fast + Mechs publish within hours of them. + STALE the site is ahead and the refresh is older than that, or the site + leads the card by more than MAX_LEAD of the card (the card then + understates it by over a third). Refresh the page. + WRONG the card differs from figure_at_pin, the figure the audit read from + the source's own committed copy at the pin with figure() below. The + card was never right, however the site has moved since (#217, #231). + SHRANK the site states fewer than the card. Records are not normally + withdrawn in bulk, so either the card is wrong or the site regressed. + GONE the source answered a 4xx other than a throttle. The page the card + cites has moved or been deleted, which is how #175 began. + CHANGED the source arrived but no figure could be read from it: the markup or + the wording moved, and the card is no longer being checked at all. + MARKUP a card in the template does not carry exactly one headline figure. + UNCARDED a card with no SOURCES entry, or an entry with no card. + AUDIT site_audit.json is missing or malformed, its pinned_at_utc is + missing, cannot be read as a time, or is in the future, or a + source's figure_at_pin is missing or not a whole number. + unread the fetch did not arrive: DNS, timeout, a dropped connection, a + 5xx, or a 408, 425 or 429 throttle. A warning, because the network is + not the site's fault, unless more than half the sources are unread, + when the run has verified too little to call itself a pass and fails + as UNCHECKED. + +The pin time and each source's figure at the pin come from +_fleet/data/site_audit.json, which the refresh writes (update-xmech-page, step 7). +ProteinTraitsMech's data file is built in CI rather than committed, so it has no +copy at the pin and no figure_at_pin; its card is checked only against the site. """ from __future__ import annotations +import datetime +import http.client import json import re import sys @@ -26,8 +58,11 @@ import urllib.request from pathlib import Path +from card_markup import card_figures, card_names, markup_problems + REPO = Path(__file__).resolve().parents[2] TEMPLATE = REPO / "_fleet/mechs_template.md" +AUDIT = REPO / "_fleet/data/site_audit.json" SITE = "https://culturebotai.github.io/" # Where each card's headline actually comes from. Pinned here rather than @@ -60,12 +95,31 @@ "MediaIngredientMech": ("json", "MediaIngredientMech/data/ingredients.json", "ingredients"), } -CARD = re.compile(r'data-mech="([A-Za-z]+)".*?
([\d,]+)', re.S) +# Where inside a source the figure must be read, when the source states the same +# label elsewhere too. CultureMech's README says "merged records" in its prose; +# only the block its generator writes, and its CI keeps in sync with the data, +# is the figure (#176). Markers missing is a shape change, not a pass. +REGIONS: dict[str, tuple[str, str]] = { + "CultureMech": ("", ""), +} + +# How long a card may trail a growing site before the page counts as stale, and +# how far it may trail within that time. A fixed percentage alone did not hold: +# CellStructureMech adds about two records an hour, 9% a day on a card of 542, +# so a 10% allowance went red a day after every refresh (#217). The time limit +# asks for a refresh a fortnight after the pins while any Mech grows; the lead +# limit asks sooner once a site is more than half as large again as its card +# (the card then understates it by over a third), which CellStructureMech +# reaches in under a week. +GRACE_DAYS = 14 +MAX_LEAD = 0.5 +# Sources with no committed copy at the pin, so no figure_at_pin: the served +# file is built in CI. Every other source must have one, or WRONG is off (#260). +NO_PIN_COPY = ("ProteinTraitsMech",) -def cards(template: str) -> dict[str, int]: - """The headline figure each card states, keyed by Mech.""" - return {m.group(1): int(m.group(2).replace(",", "")) for m in CARD.finditer(template)} +# 4xx answers that mean "not now" rather than "not here" (#219). +THROTTLES = (408, 425, 429) def source_url(path: str) -> str: @@ -78,6 +132,16 @@ def fetch(url: str, timeout: int = 30) -> str: return urllib.request.urlopen(request, timeout=timeout).read().decode("utf-8", "replace") +def region(body: str, markers: tuple[str, str]) -> str | None: + """The text between two markers, or None when either is missing.""" + start, end = markers + _, found, rest = body.partition(start) + if not found: + return None + inside, found, _ = rest.partition(end) + return inside if found else None + + def published(kind: str, body: str, selector: str) -> int | None: """The figure the site publishes, or None when the shape has changed.""" if kind == "json": @@ -111,52 +175,207 @@ def published(kind: str, body: str, selector: str) -> int | None: return int(hit.group(1).replace(",", "")) if hit else None -def main() -> int: - stated = cards(TEMPLATE.read_text()) - missing = sorted(set(stated) - set(SOURCES)) - drifted, unreadable, matched = [], [], [] +def figure(mech: str, kind: str, body: str, selector: str) -> int | None: + """The figure a source's body states, read the way the nightly reads it. + + Applies the Mech's REGIONS first, so the refresh can read the committed copy + at the pin with exactly the rules used on the live site (#221). None when no + figure can be read; a body of the wrong type raises, as published() does. + """ + if mech in REGIONS: + body = region(body, REGIONS[mech]) + if body is None: + return None + return published(kind, body, selector) + + +def read_source(mech: str, kind: str, path: str, selector: str, fetcher=None) -> tuple[str, int | str]: + """The figure a source publishes, or why there is none. + + Returns ("value", N), or a status and the reason: "GONE" for a 4xx other + than a throttle, "unread" for a fetch that did not arrive, "CHANGED" for a + body with no figure in it. + """ + fetcher = fetcher or fetch + try: + body = fetcher(source_url(path)) + except urllib.error.HTTPError as error: + # HTTPError is a URLError, so it is caught first. A 4xx is the source + # telling us it is not there, unless it is asking us to come back later. + gone = 400 <= error.code < 500 and error.code not in THROTTLES + return ("GONE" if gone else "unread"), f"fetch failed: {error}" + except (urllib.error.URLError, http.client.HTTPException, TimeoutError, OSError) as error: + # HTTPException covers a body cut short (IncompleteRead) and a garbled + # status line, which used to end the run with a traceback (#220). + return "unread", f"fetch failed: {error}" + try: + value = figure(mech, kind, body, selector) + except (ValueError, TypeError, AttributeError) as error: + # json.JSONDecodeError is a ValueError. The other two are what a + # body of an unexpected type raises when it is walked. + return "CHANGED", f"unparseable: {error}" + if value is None: + # #247: this used to say the missing block was "found". + where = ("the generated block is missing or states no figure" if mech in REGIONS + else f"no {selector!r} found") + return "CHANGED", f"{kind} shape changed; {where}" + return "value", value + + +def pin_time(audit: dict) -> datetime.datetime: + """The audit's pin time as an aware datetime; a time with no offset is UTC. + + Raises ValueError when the field is missing or is not an ISO time (#232). + """ + value = audit.get("pinned_at_utc") + if not isinstance(value, str): + raise ValueError(f"pinned_at_utc is {value!r}") + moment = datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) + return moment if moment.tzinfo else moment.replace(tzinfo=datetime.timezone.utc) + + +def classify(card: int, site: int, age: datetime.timedelta | None) -> str: + """How a published figure relates to the card that states it.""" + if site == card: + return "ok" + if site < card: + return "SHRANK" + if age is None or age > datetime.timedelta(days=GRACE_DAYS): + return "STALE" + return "grew" if site - card <= card * MAX_LEAD else "STALE" + +def span(age: datetime.timedelta) -> str: + """A pin's age in the unit a reader wants: hours on the first day, then days.""" + if age.days >= 1: + return f"{age.days} day{'s' if age.days != 1 else ''}" + hours = int(age.total_seconds() // 3600) + return f"{hours} hour{'s' if hours != 1 else ''}" + + +FAILURES = ("STALE", "WRONG", "SHRANK", "GONE", "CHANGED", "MARKUP", "UNCARDED", "AUDIT", "UNCHECKED") + + +def audit_entries(audit) -> tuple[dict, str | None]: + """The audit's repositories keyed by lower-cased name, or why they cannot be read.""" + if not isinstance(audit, dict): + return {}, f"site_audit.json holds a {type(audit).__name__}, not an object" + if "repositories" not in audit: + return {}, "site_audit.json has no repositories list" # #257 + repositories = audit["repositories"] + if not isinstance(repositories, list) or not all( + isinstance(row, dict) and isinstance(row.get("repo"), str) for row in repositories): + return {}, "site_audit.json: repositories must be a list of objects, each with a repo name" + return {row["repo"].lower(): row for row in repositories}, None + + +def check(template: str, fetcher=None, audit=None, now: datetime.datetime | None = None, + audit_error: str | None = None) -> list[tuple[str, str, str]]: + """One (status, mech, detail) row per card problem and per source, plus the run-level verdicts. + + A malformed audit is an AUDIT row, not an exception, so the other rows still + print (#250); audit_error carries a problem found while reading the file. + """ + now = now or datetime.datetime.now(datetime.timezone.utc) + rows = [("MARKUP", mech, why) for mech, why in markup_problems(template)] + if audit_error: + rows.append(("AUDIT", "-", audit_error)) + entries, problem = audit_entries(audit) if audit is not None else ({}, None) + if problem: + rows.append(("AUDIT", "-", problem)) + audit = None + age = None + if audit is not None: # {} too, so a missing pin time is reported (#257) + try: + age = now - pin_time(audit) + except ValueError as error: + # Growth then counts as STALE, which fails anyway; say why. + rows.append(("AUDIT", "-", f"site_audit.json: {error}")) + else: + if age < datetime.timedelta(0): + # A pin in the future would hold off the grace limit until the + # clock caught up with it (#242). + rows.append(("AUDIT", "-", f"site_audit.json: pinned_at_utc {audit['pinned_at_utc']} is in the future")) + age = None + stated = card_figures(template) + names = set(card_names(template)) + for mech in sorted(names - set(SOURCES)): + rows.append(("UNCARDED", mech, "card with no entry in SOURCES")) for mech, (kind, path, selector) in sorted(SOURCES.items()): - if mech not in stated: - unreadable.append((mech, "no card in the template")) + if mech not in names: + rows.append(("UNCARDED", mech, "SOURCES entry with no card in the template")) continue - try: - body = fetch(source_url(path)) - except (urllib.error.URLError, TimeoutError, OSError) as error: - unreadable.append((mech, f"fetch failed: {error}")) + if mech not in stated: + continue # its card's markup is already reported above + card = stated[mech] + entry = entries.get(mech.lower()) + at_pin = entry.get("figure_at_pin") if entry else None + whole = isinstance(at_pin, int) and not isinstance(at_pin, bool) + if audit is not None: + # A missing or mistyped figure_at_pin would switch WRONG off without + # a word (#260). + if entry is None: + rows.append(("AUDIT", mech, "no entry in site_audit.json")) + elif at_pin is None and mech not in NO_PIN_COPY: + rows.append(("AUDIT", mech, "site_audit.json has no figure_at_pin for it")) + elif at_pin is not None and not whole: + rows.append(("AUDIT", mech, f"figure_at_pin is {at_pin!r}, not a whole number")) + if whole and at_pin != card: + rows.append(("WRONG", mech, f"card {card:,}, but the source stated {at_pin:,} at the pin")) continue - try: - value = published(kind, body, selector) - except (ValueError, TypeError, AttributeError) as error: - # json.JSONDecodeError is a ValueError. The other two are what a - # body of an unexpected type raises when it is walked. - unreadable.append((mech, f"unparseable: {error}")) + status, result = read_source(mech, kind, path, selector, fetcher) + if status != "value": + rows.append((status, mech, result)) continue - if value is None: - unreadable.append((mech, f"{kind} shape changed; no {selector!r} found")) - elif value != stated[mech]: - drifted.append((mech, stated[mech], value)) - else: - matched.append(mech) - - for mech in matched: - print(f" ok {mech:<20} {stated[mech]:>9,}") - for mech, card, site in drifted: - print(f" DRIFTED {mech:<20} card {card:,}, site {site:,}") - for mech, why in unreadable: - print(f" unread {mech:<20} {why}") - if missing: - print(f" MISSING cards with no entry in SOURCES: {', '.join(missing)}") - - print(f"\n{len(matched)} match, {len(drifted)} drifted, {len(unreadable)} unreadable.") - if drifted or missing: - print("Refresh the card figures in _fleet/mechs_template.md and the MECHS block " - "in _fleet/fleet_fragment.html, then rerun assemble_page.py.") + verdict = classify(card, result, age) + detail = f"{card:>9,}" if verdict == "ok" else f"card {card:,}, site {result:,}" + if verdict in ("grew", "STALE") and age is not None: + detail += f" (+{result - card:,} in the {span(age)} since the pins)" + rows.append((verdict, mech, detail)) + unread = sum(1 for status, _, _ in rows if status == "unread") + if unread * 2 > len(SOURCES): + # One site down is someone else's outage. Most of them down is this run + # having checked nothing, which must not read as a pass (#115). + rows.append(("UNCHECKED", "-", f"{unread} of {len(SOURCES)} sources could not be read")) + return rows + + +def main() -> int: + audit, audit_error = None, None + try: + audit = json.loads(AUDIT.read_text()) + if not isinstance(audit, dict): + # check() reads None as "no audit supplied", so a null file would + # otherwise pass silently (#257). + audit_error = f"site_audit.json holds {type(audit).__name__}, not an object" + audit = None + except FileNotFoundError: + audit_error = "site_audit.json is missing" + except (OSError, ValueError) as error: + audit_error = f"site_audit.json cannot be read: {error}" + rows = check(TEMPLATE.read_text(), audit=audit, audit_error=audit_error) + for status, mech, detail in rows: + print(f" {status:<9} {mech:<20} {detail}") + tally: dict[str, int] = {} + for status, _, _ in rows: + tally[status] = tally.get(status, 0) + 1 + print("\n" + ", ".join(f"{count} {status.lower()}" for status, count in sorted(tally.items())) + ".") + if any(status in FAILURES for status, _, _ in rows): + print("Failing. STALE or SHRANK: refresh the card figures in _fleet/mechs_template.md " + "and the MECHS block in _fleet/fleet_fragment.html (update-xmech-page), then rerun " + "assemble_page.py. WRONG: correct the card and every other occurrence of its " + "figure, found by grepping the tree for it as update-xmech-page step 6 does (the MECHS " + "records: and extra: text in _fleet/fleet_fragment.html, cross-references, card_records " + "in site_audit.json, the pages that repeat it), or the audit's figure_at_pin if that is " + "what was mistyped, then rerun assemble_page.py; no re-pin. GONE or CHANGED: repoint that Mech's SOURCES entry. MARKUP or " + "UNCARDED: fix the card or its SOURCES entry. AUDIT: fix site_audit.json. " + "UNCHECKED: the run could not reach most sites; rerun before changing anything.") return 1 - if unreadable: - # A site that cannot be reached is not the same as a wrong number, and a - # nightly red for someone else's outage teaches people to ignore it. - print("Nothing drifted, but some sites could not be read; see above.") + if any(status in ("grew", "unread") for status, _, _ in rows): + # A site a little ahead of a recently pinned card, or one that could not + # be reached, is not a wrong number, and a nightly red for either + # teaches people to ignore it. + print("Passing with warnings; see above.") return 0 diff --git a/tests/test_fleet_page.py b/tests/test_fleet_page.py index 6b02746..d64b102 100644 --- a/tests/test_fleet_page.py +++ b/tests/test_fleet_page.py @@ -1,6 +1,8 @@ """Regression coverage for fleet admission, capability drift and generated output.""" from copy import deepcopy import contextlib +import datetime +import io import json import os from pathlib import Path @@ -13,7 +15,8 @@ ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "scripts/fleet")) -from assemble_page import CARD_RECORDS, assemble, capability_rows, number_word, script_json +from assemble_page import assemble, capability_rows, number_word, script_json +from card_markup import card_figures, card_names from refresh_manifest import ARTIFACT_PATH, MANIFEST_PATH, read_canonical, semantic, validate import roots @@ -35,7 +38,7 @@ def test_rendering_twice_does_not_consume_the_census(self): def test_records_tile_equals_the_sum_of_the_cards(self): page = self.render() - total = sum(int(n.replace(",", "")) for n in CARD_RECORDS.findall(self.template)) + total = sum(card_figures(self.template).values()) self.assertIn(f"
{total:,}curated entries across the fleet
", page) def test_a_card_without_a_record_count_cannot_be_left_out_of_the_total(self): @@ -43,6 +46,14 @@ def test_a_card_without_a_record_count_cannot_be_left_out_of_the_total(self): with self.assertRaisesRegex(ValueError, "record count"): self.render() + def test_a_figure_outside_every_card_cannot_join_the_total(self): + # #218, #234: a stray figure used to be ignored, and a second one in a + # card used to replace its headline in the total. + stray = self.template.replace("", '
7
', 1) + self.template = stray + with self.assertRaisesRegex(ValueError, "outside any card"): + self.render() + def test_card_stats_must_cover_every_fleet_member(self): self.stats["mechs"] = [m for m in self.stats["mechs"] if m["mech"] != "TaxonMech"] with self.assertRaisesRegex(ValueError, "Mech stats"): @@ -400,7 +411,7 @@ def test_every_card_has_a_published_source(self): # would otherwise get a card and be silently exempt from checking, # which is the failure this whole issue is about. import check_cards - stated = check_cards.cards((ROOT / "_fleet/mechs_template.md").read_text()) + stated = card_figures((ROOT / "_fleet/mechs_template.md").read_text()) self.assertEqual(sorted(set(stated) - set(check_cards.SOURCES)), [], "card with no entry in check_cards.SOURCES") self.assertEqual(sorted(set(check_cards.SOURCES) - set(stated)), [], @@ -430,9 +441,356 @@ def test_a_relative_source_is_read_from_the_pages_host(self): def test_the_card_parser_reads_every_member(self): snapshot = json.loads((ROOT / "_fleet/data/manifest.json").read_text()) + template = (ROOT / "_fleet/mechs_template.md").read_text() + self.assertEqual(sorted(card_figures(template)), sorted(snapshot["mechs"])) + self.assertEqual(sorted(card_names(template)), sorted(snapshot["mechs"])) + + def test_a_card_without_a_figure_does_not_borrow_its_neighbours(self): + # The old check_cards regex paired a name with the next figure in the + # file, so the first card here would have been read as 20 (#114). + template = ('

A

\n' + '
20
') + self.assertEqual(card_figures(template), {"BMech": 20}) + self.assertEqual(card_names(template), ["AMech", "BMech"]) + + +class CardCheckTests(unittest.TestCase): + """What the nightly card check fails on and what it only warns about.""" + + NAMES = ("AMech", "BMech", "CMech") + + def setUp(self): import check_cards - stated = check_cards.cards((ROOT / "_fleet/mechs_template.md").read_text()) - self.assertEqual(sorted(stated), sorted(snapshot["mechs"])) + from unittest import mock + self.check_cards = check_cards + self.real_sources = dict(check_cards.SOURCES) + self.sources = mock.patch.dict(check_cards.SOURCES, { + m: ("html", f"{m}/pages/index.html", f"{m[0].lower()} records") for m in self.NAMES + }, clear=True) + self.sources.start() + self.addCleanup(self.sources.stop) + self.cards = {m: 1000 for m in self.NAMES} + self.site = {m: 1000 for m in self.NAMES} + self.pinned_at = datetime.datetime(2026, 9, 25, 2, 0, tzinfo=datetime.timezone.utc) + self.now = self.pinned_at + datetime.timedelta(hours=18) + # The figure each source stated at the pin, as the audit records it. + self.at_pin = {m: 1000 for m in self.NAMES} + + def body(self, mech, value): + return f"
{value:,}{mech[0].lower()} records
" + + def fetch(self, url): + mech = url.split("/")[3] + value = self.site[mech] + if isinstance(value, Exception): + raise value + return self.body(mech, value) if isinstance(value, int) else value + + def template(self): + return "".join(f'
{n:,}
' + for m, n in self.cards.items()) + + def audit(self): + entries = [] + for m in self.check_cards.SOURCES: + if m in self.at_pin: + entry = {"repo": m} + if self.at_pin[m] is not None: + entry["figure_at_pin"] = self.at_pin[m] + entries.append(entry) + return {"pinned_at_utc": self.pinned_at.isoformat(), "repositories": entries} + + def rows(self, template=None): + return self.check_cards.check(template or self.template(), self.fetch, self.audit(), self.now) + + def statuses(self): + return {mech: status for status, mech, _ in self.rows()} + + def failed(self, template=None): + return sorted(status for status, _, _ in self.rows(template) if status in self.check_cards.FAILURES) + + def test_matching_figures_pass(self): + self.assertEqual(set(self.statuses().values()), {"ok"}) + self.assertEqual(self.failed(), []) + + def test_a_site_ahead_of_a_recent_pin_only_warns(self): + # #148, #217: CellStructureMech adds about 9% a day; within the grace + # period that is growth, not a wrong card. + self.site["AMech"] = 1400 + self.assertEqual(self.statuses()["AMech"], "grew") + self.assertEqual(self.failed(), []) + + def test_growth_past_the_grace_period_fails(self): + self.assertEqual(self.check_cards.GRACE_DAYS, 14) # #243: the documented figure + self.site["AMech"] = 1001 + self.now = self.pinned_at + datetime.timedelta(days=self.check_cards.GRACE_DAYS, hours=1) + self.assertEqual(self.failed(), ["STALE"]) + self.now = self.pinned_at + datetime.timedelta(days=self.check_cards.GRACE_DAYS) + self.assertEqual(self.failed(), []) + + def test_a_card_that_understates_its_site_by_more_than_the_lead_fails_early(self): + self.site["AMech"] = 1500 + self.assertEqual(self.failed(), []) + self.site["AMech"] = 1501 + self.assertEqual(self.failed(), ["STALE"]) + + def test_growth_without_an_audit_fails(self): + self.site["AMech"] = 1001 + rows = self.check_cards.check(self.template(), self.fetch, None, self.now) + self.assertIn(("STALE", "AMech"), [(s, m) for s, m, _ in rows]) + + def test_a_mistyped_card_fails_however_the_site_has_moved(self): + # #217, #231: 3,026 typed for 3,206 used to pass as "grew", and a hash + # comparison caught it only while the source was unchanged since the pin. + self.cards["AMech"] = 3026 + self.at_pin["AMech"] = 3206 + for live in (3206, 3300): + self.site["AMech"] = live + self.assertEqual(self.statuses()["AMech"], "WRONG", live) + self.assertEqual(self.failed(), ["WRONG"]) + # #249: overstated too, with the site ahead of both. + self.cards["AMech"] = 3260 + self.site["AMech"] = 3300 + self.assertEqual(self.statuses()["AMech"], "WRONG") + + def test_a_source_with_no_figure_at_the_pin_is_checked_against_the_site_only(self): + # ProteinTraitsMech's data file is built in CI, so the audit has no copy. + from unittest import mock + self.at_pin["AMech"] = None + self.site["AMech"] = 1100 + with mock.patch.object(self.check_cards, "NO_PIN_COPY", ("AMech",)): + self.assertEqual(self.statuses()["AMech"], "grew") + self.assertEqual(self.failed(), []) + + def test_a_missing_or_mistyped_figure_at_the_pin_is_an_audit_failure(self): + # #260: each of these used to switch WRONG off without a word, so a + # mistyped card passed as "grew". + self.cards["AMech"] = 3026 + self.site["AMech"] = 3206 + for at_pin in ("3206", 3206.0, True, None): + self.at_pin["AMech"] = at_pin + rows = self.rows() + self.assertIn(("AUDIT", "AMech"), [(s, m) for s, m, _ in rows], at_pin) + self.assertIn("AUDIT", self.failed(), at_pin) + + def test_a_pin_time_without_an_offset_is_utc_and_an_unreadable_one_fails(self): + # #232: both used to end the run with a traceback before any row printed. + audit = self.audit() + audit["pinned_at_utc"] = self.pinned_at.replace(tzinfo=None).isoformat() + self.site["AMech"] = 1100 + rows = self.check_cards.check(self.template(), self.fetch, audit, self.now) + self.assertIn(("grew", "AMech"), [(s, m) for s, m, _ in rows]) + audit["pinned_at_utc"] = "25 September 2026" + rows = self.check_cards.check(self.template(), self.fetch, audit, self.now) + statuses = [s for s, _, _ in rows] + self.assertIn("AUDIT", statuses) + self.assertIn(("STALE", "AMech"), [(s, m) for s, m, _ in rows]) + + def audit_failures(self, audit): + return sorted(s for s, _, _ in self.check_cards.check(self.template(), self.fetch, audit, self.now) + if s in self.check_cards.FAILURES) + + def test_a_missing_future_or_unreadable_pin_time_fails_on_its_own(self): + # #242, #243: with every card equal to its site, the audit alone fails. + audit = self.audit() + del audit["pinned_at_utc"] + self.assertEqual(self.audit_failures(audit), ["AUDIT"]) + audit["pinned_at_utc"] = "not a time" + self.assertEqual(self.audit_failures(audit), ["AUDIT"]) + for ahead in (datetime.timedelta(minutes=1), datetime.timedelta(days=365)): # #249 + audit["pinned_at_utc"] = (self.now + ahead).isoformat() + self.assertEqual(self.audit_failures(audit), ["AUDIT"], ahead) + # And a future pin must not hold off the grace limit for a grown card. + self.site["AMech"] = 1001 + self.assertIn("STALE", self.audit_failures(audit)) + + def test_a_malformed_audit_is_an_audit_row_not_a_traceback(self): + # #250: the other rows must still print. + pinned = self.pinned_at.isoformat() + for audit in ([], [1], {}, {"pinned_at_utc": pinned}, # #257, #259 + {"pinned_at_utc": pinned, "repositories": [{"figure_at_pin": 1}]}, + {"pinned_at_utc": pinned, "repositories": "x"}, + {"pinned_at_utc": pinned, "repositories": 5}): + rows = self.check_cards.check(self.template(), self.fetch, audit, self.now) + self.assertIn("AUDIT", [s for s, _, _ in rows], audit) + self.assertEqual(sorted(m for s, m, _ in rows if s == "ok"), list(self.NAMES), audit) + rows = self.check_cards.check(self.template(), self.fetch, None, self.now, + audit_error="site_audit.json is missing") + self.assertEqual(sorted(s for s, _, _ in rows if s in self.check_cards.FAILURES), ["AUDIT"]) + + def test_a_pin_time_without_an_offset_is_read_as_utc_not_local_time(self): + # #243: in a UTC runner, local time and UTC agree, so pin a zone that differs. + import time + saved = os.environ.get("TZ") + os.environ["TZ"] = "America/Los_Angeles" + time.tzset() + try: + moment = self.check_cards.pin_time({"pinned_at_utc": "2026-09-25T02:06:03"}) + finally: + if saved is None: + os.environ.pop("TZ", None) + else: + os.environ["TZ"] = saved + time.tzset() + self.assertEqual(moment, datetime.datetime(2026, 9, 25, 2, 6, 3, tzinfo=datetime.timezone.utc)) + self.assertEqual(moment.utcoffset(), datetime.timedelta(0)) + + def test_a_site_behind_its_card_fails(self): + self.site["AMech"] = 999 + self.assertEqual(self.failed(), ["SHRANK"]) + + def test_a_missing_page_fails_but_an_outage_or_throttle_only_warns(self): + # #176: a 404 is the cited page gone, which is how #175 began. #219: a + # 429 is the host asking us to come back later. + import urllib.error + self.site["AMech"] = urllib.error.HTTPError("u", 404, "Not Found", {}, None) + self.site["BMech"] = urllib.error.HTTPError("u", 429, "Too Many Requests", {}, None) + statuses = self.statuses() + self.assertEqual((statuses["AMech"], statuses["BMech"]), ("GONE", "unread")) + self.assertEqual(self.failed(), ["GONE"]) + for code in (408, 425, 500, 503): + self.site["BMech"] = urllib.error.HTTPError("u", code, "x", {}, None) + self.assertEqual(self.statuses()["BMech"], "unread", code) + for code in (400, 403, 410, 499): # #234, #243: any other 4xx + self.site["AMech"] = urllib.error.HTTPError("u", code, "x", {}, None) + self.assertEqual(self.statuses()["AMech"], "GONE", code) + + def test_a_body_cut_short_is_unread_not_a_traceback(self): + # #220 + import http.client + self.site["AMech"] = http.client.IncompleteRead(b"
", 500) + self.assertEqual(self.statuses()["AMech"], "unread") + self.assertEqual(self.failed(), []) + + def test_a_page_with_no_figure_fails(self): + self.site["AMech"] = "
1,000entries
" + self.assertEqual(self.failed(), ["CHANGED"]) + + def test_a_json_source_that_is_not_json_fails(self): + # #222: an HTML shell served with 200 where the data file used to be. + self.check_cards.SOURCES["AMech"] = ("json", "AMech/data/ingredients.json", "ingredients") + self.site["AMech"] = "" + self.assertEqual(self.statuses()["AMech"], "CHANGED") + + def test_more_than_half_unread_fails_and_half_does_not(self): + # #115: a run that read nothing used to exit 0. #222: ten sources, so + # "more than half" can be told from "at least half". + import urllib.error + names = [f"M{i}Mech" for i in range(10)] + self.check_cards.SOURCES.clear() + self.check_cards.SOURCES.update({m: ("html", f"{m}/pages/index.html", "m records") for m in names}) + self.cards = {m: 1000 for m in names} + self.site = {m: 1000 for m in names} + self.at_pin = {m: 1000 for m in names} + for m in names[:5]: + self.site[m] = urllib.error.URLError("no route") + self.assertEqual(self.failed(), []) + self.site[names[5]] = TimeoutError("timed out") + self.assertEqual(self.failed(), ["UNCHECKED"]) + + def test_a_card_and_its_source_must_both_exist(self): + del self.cards["CMech"] + self.cards["DMech"] = 5 + self.assertEqual(self.failed(), ["UNCARDED", "UNCARDED"]) + + def test_a_card_without_exactly_one_figure_is_reported_as_markup(self): + # #218: not as a missing card, and a second figure is not silently dropped. + template = self.template() + unreadable = template.replace('
1,000
', + '
1,000
', 1) + rows = self.rows(unreadable) + self.assertIn(("MARKUP", "AMech"), [(s, m) for s, m, _ in rows]) + self.assertNotIn("UNCARDED", [s for s, _, _ in rows]) + doubled = template.replace('
', + '
7
', 1) + self.assertEqual(self.failed(doubled), ["MARKUP"]) + stray = template + '
7
' + self.assertEqual(self.failed(stray), ["MARKUP"]) + # #261: an unreadable figure is MARKUP, not int("") ending the run, and + # a malformed second tile, in a card or outside one, still counts. + for bad in (",", "10,00", "1000,", "1,0000"): # #261, #264 + self.assertEqual(self.failed(template.replace("1,000", bad, 1)), ["MARKUP"], bad) + from card_markup import card_figures as figures + for good, value in (("1000", 1000), ("625,960", 625960), ("7", 7)): + self.assertEqual(figures(template.replace("1,000", good, 1))["AMech"], value, good) + spaced = template.replace('
', + '
7
', 1) + self.assertEqual(self.failed(spaced), ["MARKUP"]) + self.assertEqual(self.failed(template + '
7
'), ["MARKUP"]) + # #262: a second card for one Mech would have been checked instead of the first. + twice = '
3,026
' + template + self.assertEqual(self.failed(twice), ["MARKUP"]) + + def test_the_culturemech_figure_is_read_only_from_its_generated_block(self): + # #176: the regex takes the first match, and the README's prose could + # state an older "N merged records" above the generated block. + kind, path, selector = self.real_sources["CultureMech"] + begin, end = self.check_cards.REGIONS["CultureMech"] + line = ("The tracked corpus currently contains **15,878 normalized records** and " + "**6,288 merged records**.") + readme = "Release 2 added 1,024 merged records.\n" + begin + "\n" + line + "\n" + end + "\n" + read = self.check_cards.read_source + self.assertEqual(read("CultureMech", kind, path, selector, lambda url: readme)[:2], ("value", 6288)) + for broken in ("The corpus has 6,288 merged records.", begin + "\n" + line + "\n", + begin + "\nno figure here\n" + end): + status, detail = read("CultureMech", kind, path, selector, lambda url: broken) + self.assertEqual(status, "CHANGED") + # #247: it used to say the missing block was "found". + self.assertIn("missing or states no figure", detail) + + def test_main_exits_by_the_same_rule(self): + from unittest import mock + import urllib.error + with tempfile.TemporaryDirectory() as tmp: + template, audit = Path(tmp) / "mechs_template.md", Path(tmp) / "site_audit.json" + template.write_text(self.template()) + self.pinned_at = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(hours=1) + audit.write_text(json.dumps(self.audit())) + with mock.patch.object(self.check_cards, "TEMPLATE", template), \ + mock.patch.object(self.check_cards, "AUDIT", audit), \ + mock.patch.object(self.check_cards, "fetch", self.fetch), \ + contextlib.redirect_stdout(io.StringIO()): + self.site["AMech"] = 1050 + self.site["BMech"] = urllib.error.URLError("no route") + self.assertEqual(self.check_cards.main(), 0) + self.site["AMech"] = 900 + self.assertEqual(self.check_cards.main(), 1) + # #234: every failing verdict fails the run, not just SHRANK. + self.site["AMech"] = 1600 # STALE + self.assertEqual(self.check_cards.main(), 1) + self.site["AMech"] = 1000 + self.at_pin["AMech"] = 1001 # WRONG + audit.write_text(json.dumps(self.audit())) + self.assertEqual(self.check_cards.main(), 1) + self.at_pin["AMech"] = 1000 + audit.write_text(json.dumps(self.audit())) + self.site["AMech"] = self.site["CMech"] = TimeoutError("t") # UNCHECKED + self.assertEqual(self.check_cards.main(), 1) + self.site = {m: 1000 for m in self.NAMES} + # #249: the remedy WRONG prints is a correction, not a refresh. + self.at_pin["AMech"] = 1001 + audit.write_text(json.dumps(self.audit())) + out = io.StringIO() + with contextlib.redirect_stdout(out): + self.assertEqual(self.check_cards.main(), 1) + remedy = out.getvalue().split("WRONG:", 1)[1].split(" GONE or CHANGED:", 1)[0] + for named in ("every other occurrence", "grepping the tree", "fleet_fragment.html", + "extra:", "card_records", "no re-pin"): # #248, #258, #259 + self.assertIn(named, remedy) + self.assertNotIn("update-xmech-page)", remedy) + self.at_pin["AMech"] = 1000 + # AUDIT alone fails the run, and an unreadable file is AUDIT, not a traceback (#250). + audit.write_text("{not json") + self.assertEqual(self.check_cards.main(), 1) + for body in ("null", "{}", "[1]"): # #257: none may pass as "no audit" + audit.write_text(body) + out = io.StringIO() + with contextlib.redirect_stdout(out): + self.assertEqual(self.check_cards.main(), 1, body) + self.assertIn("AUDIT", out.getvalue(), body) + audit.unlink() + self.assertEqual(self.check_cards.main(), 1) + class RefreshProvenanceTests(unittest.TestCase): """The derived numbers must all come from one set of checkouts (#85). @@ -477,11 +835,44 @@ def test_the_audit_pins_the_revisions_the_stats_counted(self): self.assertIn(entry["repo"], self.audit, "Mech missing from site_audit.json") self.assertEqual(self.audit[entry["repo"]]["sha"], entry["source_revision"]) + def test_every_graph_panel_states_its_card_figure(self): + # #248: the MECHS block repeats each card's figure as records:, and a + # correction to one used to leave the other behind unnoticed. + fragment = (ROOT / "_fleet/fleet_fragment.html").read_text() + metadata = fragment.split("var MECHS = {", 1)[1].split("\n };", 1)[0] + panels = {m: int(n) for m, n in re.findall(r"^\s+([A-Za-z]+Mech):\s*\{.*?\brecords:\s*(\d+)", metadata, re.M)} + self.assertEqual(panels, card_figures((ROOT / "_fleet/mechs_template.md").read_text())) + + def test_every_card_equals_the_figure_its_source_stated_at_the_pin(self): + # #231: the audit builder read card_records from the template, so the + # audit could only repeat a mistyped card. figure_at_pin is read from the + # source's committed copy at the pin with check_cards.figure(), and the + # nightly's WRONG verdict depends on it being there. + import check_cards + audit = json.loads((ROOT / "_fleet/data/site_audit.json").read_text()) + pinned = check_cards.pin_time(audit) # #232: must parse + # #242: and must fall between the newest pinned commit and the check. + newest = max(datetime.datetime.fromisoformat(r["commit_date"].replace("Z", "+00:00")) + for r in audit["repositories"]) + checked = datetime.datetime.fromisoformat(audit["checked_at_utc"]) + self.assertLessEqual(newest, pinned) + self.assertLessEqual(pinned, checked) + entries = {r["repo"].lower(): r for r in audit["repositories"]} + cards = card_figures((ROOT / "_fleet/mechs_template.md").read_text()) + for mech in check_cards.SOURCES: + with self.subTest(mech=mech): + entry = entries[mech.lower()] + if mech in check_cards.NO_PIN_COPY: + # Built in CI, so there is no committed copy at the pin. + self.assertNotIn("figure_at_pin", entry) + continue + self.assertIs(type(entry.get("figure_at_pin")), int) # #260: not 3206.0 + self.assertEqual(entry["figure_at_pin"], cards[mech]) + def test_the_audit_records_this_refresh_and_nothing_else(self): # The audit's other fields had no gate at all (#128): its merged PRs, # the card figure it read, the CLAW pin, and which repositories it lists. - import check_cards - cards = check_cards.cards((ROOT / "_fleet/mechs_template.md").read_text()) + cards = card_figures((ROOT / "_fleet/mechs_template.md").read_text()) manifest = json.loads((ROOT / "_fleet/data/manifest.json").read_text()) self.assertEqual(set(self.audit), {m["repo"] for m in self.stats.values()} | {"culturebotai-claw"}) self.assertEqual(self.audit["culturebotai-claw"]["sha"], manifest["source"]["revision"])