From 0c8ff8120a3c8fd08d12d6478aebe98a5838c448 Mon Sep 17 00:00:00 2001 From: "marcin p. joachimiak" <4625870+realmarcin@users.noreply.github.com> Date: Fri, 25 Sep 2026 13:09:06 -0700 Subject: [PATCH 1/8] Let the card check tell a site that grew from a wrong card The nightly check failed on every card a fast Mech had outgrown since the refresh's pins, and passed on a source that returned 404, lost its figure, or could not be reached at all. It now warns on a site up to 10% ahead of its card and fails past that, on a site behind its card, on a 4xx, on a source with no readable figure, on an unpaired card or SOURCES entry, and when more than half the sources are unread (#148, #113, #115, #176). CultureMech's figure is read only from the README block its generator writes (#176). The cards are now parsed once, card by card, in card_markup.py, which the assembler, the check and the tests share, so a card without a figure is missing rather than reading its neighbour's (#114). Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/skills/review-open-issues/SKILL.md | 6 +- .claude/skills/update-xmech-page/SKILL.md | 7 +- _fleet/README.md | 24 ++- scripts/fleet/assemble_page.py | 22 +-- scripts/fleet/card_markup.py | 32 ++++ scripts/fleet/check_cards.py | 180 +++++++++++++++------ tests/test_fleet_page.py | 132 ++++++++++++++- 7 files changed, 323 insertions(+), 80 deletions(-) create mode 100644 scripts/fleet/card_markup.py diff --git a/.claude/skills/review-open-issues/SKILL.md b/.claude/skills/review-open-issues/SKILL.md index 5bf4e48..1fbeaa1 100644 --- a/.claude/skills/review-open-issues/SKILL.md +++ b/.claude/skills/review-open-issues/SKILL.md @@ -216,8 +216,10 @@ pull request, so it can be red without blocking anything: python scripts/fleet/check_cards.py # card headline figures vs each Mech's site ``` -A drifted card therefore shows up as a failed scheduled run, not a failed PR -check. Look at the latest scheduled run before crediting the cards as current. +A stale card therefore shows up as a failed scheduled run, not a failed PR +check. A card a little behind a fast Mech only warns ("grew", within 10%), so a +green run does not mean every card equals its site: read the run's log, not +just its colour, before crediting the cards as current. An issue asserting a defect that one of these already blocks is P2 unless it shows the gate is porous — and they have been porous: a test can pass because diff --git a/.claude/skills/update-xmech-page/SKILL.md b/.claude/skills/update-xmech-page/SKILL.md index 1f35615..9d5520b 100644 --- a/.claude/skills/update-xmech-page/SKILL.md +++ b/.claude/skills/update-xmech-page/SKILL.md @@ -253,7 +253,7 @@ python3 scripts/fleet/assemble_page.py python3 -m unittest discover -s tests -v python3 scripts/fleet/assemble_page.py --check python3 scripts/fleet/refresh_manifest.py --claw-root "$SNAP/claw" --check -python3 scripts/fleet/check_cards.py # 0 drifted, except sites that moved past their pin (below) +python3 scripts/fleet/check_cards.py # exit 0; "grew" lines are sites that moved past their pin (below) ``` Rerun `check_cards.py` immediately before opening the PR and again before any @@ -264,8 +264,9 @@ When a site has moved past its pin, do not re-pin that one Mech: the census and overlaps are computed across Mechs, so a single re-pin is a partial rerun, and a fast Mech moves again before the rerun finishes. Keep the page a consistent snapshot at the pins, record the live figure as `site_figure_at_check` in that -Mech's `site_audit.json` entry, and say in the PR which cards will show as -drifted. Re-pin everything only if the drift is large enough to mislead. +Mech's `site_audit.json` entry, and say in the PR which cards the check reports +as grown. Within `GROWTH_TOLERANCE` (10%) that is a warning; past it the check +reports STALE and fails, and the refresh should re-pin everything. Emoji headings render with a leading hyphen in their id on GitHub Pages. Verify anchors against the deployed HTML, not a local kramdown. diff --git a/_fleet/README.md b/_fleet/README.md index f5f7622..c2f6e17 100644 --- a/_fleet/README.md +++ b/_fleet/README.md @@ -37,13 +37,21 @@ against the page it cites and is the one script here that needs the network: python3 scripts/fleet/check_cards.py ``` -It exits 1 on a figure that differs from the site and only warns on a page it -could not read, and `SOURCES` at its top pins where each Mech publishes its -count. It runs on the workflow's nightly schedule, not on pull requests, so a -Mech shipping records overnight does not block an unrelated change; a nightly -red means the card figures in `mechs_template.md` and the `MECHS` block in -`fleet_fragment.html` need refreshing together. A new card needs a `SOURCES` -entry; a test enforces that. +`SOURCES` at its top pins where each Mech publishes its count, and `REGIONS` +restricts a source to the part that states it where the same words appear +elsewhere (CultureMech's generated README block). The page is a snapshot at a +refresh's pins, so a site up to 10% ahead of its card (`GROWTH_TOLERANCE`) only +warns. It fails when a site is further ahead than that or behind its card, when +a source answers 4xx or no longer states a figure the parser can read, when a +card and a `SOURCES` entry do not pair up, and when more than half the sources +could not be fetched at all. One site's outage only warns (#148, #115, #176). +It runs on the workflow's nightly schedule, not on pull requests, so a Mech +shipping records overnight does not block an unrelated change; a nightly red +means either the card figures in `mechs_template.md` and the `MECHS` block in +`fleet_fragment.html` need refreshing together, or a `SOURCES` entry needs +repointing. A new card needs a `SOURCES` entry; a test enforces that. The cards +are read by `scripts/fleet/card_markup.py`, the one parser the assembler, this +check and the tests share (#114). The `Fleet page` workflow checks pull requests, pushes and the live CLAW manifest daily. It detects changes to membership, capability declarations (including @@ -180,7 +188,7 @@ its site lists 422 communities, while its record glob also takes four isolate records, so the census and `mech_stats.json` count 426. CellStructureMech and TraitMech published new records after the pins were taken; their cards keep the pinned figures, and `site_audit.json` records what the two sites showed when it -was written. `check_cards.py` will report both as drifted until the next refresh. +was written. `check_cards.py` reports both as grown, a warning, until the next refresh. NaturalProductMech's landing page and MediaIngredientMech's data file also changed after the pins without changing their figures; the audit records each live hash beside the hash of the committed copy at the pin. diff --git a/scripts/fleet/assemble_page.py b/scripts/fleet/assemble_page.py index be60cfd..6ed8e57 100644 --- a/scripts/fleet/assemble_page.py +++ b/scripts/fleet/assemble_page.py @@ -6,6 +6,7 @@ from pathlib import Path import re +from card_markup import card_figures, card_names from refresh_manifest import validate REPO = Path(__file__).resolve().parents[2] @@ -50,11 +51,8 @@ def number_word(value: int) -> str: return WORDS[value] if 0 <= value < len(WORDS) else f"{value:,}" -CARD_RECORDS = re.compile(r'
([\d,]+)') - - def fleet_records(template): - """What the Mech cards add up to. + """What the Mech cards add up to, one figure per card. The tile used to carry its own typed figure and drifted away from the cards it was meant to total: it read 448,724 while the ten cards summed to @@ -68,11 +66,15 @@ def fleet_records(template): The ten are not ten counts of the same thing: the cards call theirs taxon records, published recipes, natural product structures and so on. The tile says "curated entries" rather than "records" for that reason (#82). + + Keyed by Mech and read card by card through card_markup, the parser + check_cards.py also uses, so the total and the nightly check cannot read the + markup differently (#114). """ - counts = [int(n.replace(",", "")) for n in CARD_RECORDS.findall(template)] - if not counts: + figures = card_figures(template) + if not figures: raise ValueError("No Mech card record counts found") - return counts + return figures def assemble(template, fragment, data, snapshot, stats, census): @@ -81,7 +83,7 @@ def assemble(template, fragment, data, snapshot, stats, census): badges = re.findall(r"", template) if len(badges) != len(names) or set(badges) != names: raise ValueError("Mech cards must match canonical fleet membership exactly") - cards = re.findall(r']*\bdata-mech="([^"]+)"', template) + cards = card_names(template) if len(cards) != len(names) or set(cards) != names: raise ValueError("Actual Mech cards must match canonical fleet membership exactly") metadata = fragment.split("var MECHS = {", 1)[1].split("\n };", 1)[0] @@ -114,7 +116,7 @@ def assemble(template, fragment, data, snapshot, stats, census): if counted != names: raise ValueError("Mech stats must cover canonical fleet membership exactly") counts = fleet_records(template) - if len(counts) != len(names): + if set(counts) != names: raise ValueError("Every Mech card must carry a record count") # The census is a dated scan, so its vocabulary tally is labelled with its own # run date rather than as current, and its coverage is stated below. @@ -127,7 +129,7 @@ def assemble(template, fragment, data, snapshot, stats, census): tokens = { "": str(len(names)), "": number_word(len(names)), - "": f"{sum(counts):,}", + "": f"{sum(counts.values()):,}", "": f"{len(vocabularies):,}", # "all ten Mechs" once the census reaches every member, which it has # since TaxonMech was added (#87); "nine of the ten Mechs" otherwise. diff --git a/scripts/fleet/card_markup.py b/scripts/fleet/card_markup.py new file mode 100644 index 0000000..2b85f09 --- /dev/null +++ b/scripts/fleet/card_markup.py @@ -0,0 +1,32 @@ +"""The one parser for the Mech cards in _fleet/mechs_template.md. + +assemble_page.py, check_cards.py and the tests used to read the card markup with +three separate regexes, and check_cards.py paired each Mech with the *next* +figure in the file, so a card missing its figure silently took its neighbour's +(#114). Everything now reads the cards here, one
at a time. +""" +from __future__ import annotations + +import re + +ARTICLE = re.compile(r']*\bdata-mech="([^"]+)"[^>]*>(.*?)
', re.S) +FIGURE = re.compile(r'
([\d,]+)') + + +def card_names(template: str) -> list[str]: + """Every card's Mech, in page order, duplicates kept.""" + return [mech for mech, _ in ARTICLE.findall(template)] + + +def card_figures(template: str) -> dict[str, int]: + """Each card's headline figure, keyed by Mech. + + Read inside the card's own
, so a card without a figure is absent + rather than borrowing the next card's. + """ + figures: dict[str, int] = {} + for mech, body in ARTICLE.findall(template): + hit = FIGURE.search(body) + if hit: + figures[mech] = int(hit.group(1).replace(",", "")) + return figures diff --git a/scripts/fleet/check_cards.py b/scripts/fleet/check_cards.py index cf8c064..14e53cf 100644 --- a/scripts/fleet/check_cards.py +++ b/scripts/fleet/check_cards.py @@ -12,10 +12,27 @@ that a blocking check would make an unrelated docs fix unmergeable because some Mech published records overnight. -Two failure kinds, kept apart on purpose. A fetch that does not arrive is a -warning, because the network is not the site's fault. A fetch that arrives and -does not match is a failure, because either the number moved or the markup did, -and both need a person. +What fails and what only warns (#148, #115, #176): + + ok the site states the card's figure. + grew the site is ahead of the card by at most GROWTH_TOLERANCE. A warning: + the page is a snapshot at a refresh's pins and fast Mechs publish + within hours of them, so a small lead is the expected state, not a + wrong card. site_audit.json records the lead the refresh saw. + STALE the site is further ahead than that. The card no longer describes the + Mech; refresh it. + SHRANK the site states fewer than the card. Records are not normally + withdrawn in bulk, so either the card is wrong or the site regressed. + GONE the source answered 4xx. The page the card cites has moved or been + deleted, which is how #175 began; an outage does not look like this. + CHANGED the source arrived but no figure could be read from it: the markup or + the wording moved, and the card is no longer being checked at all. + unread the fetch did not arrive (DNS, timeout, 5xx). A warning, because the + network is not the site's fault, unless more than half the sources + are unread, when the run has verified too little to call itself a + pass and fails as UNCHECKED. + +A card with no SOURCES entry, or an entry with no card, also fails. """ from __future__ import annotations @@ -26,6 +43,8 @@ import urllib.request from pathlib import Path +from card_markup import card_figures + REPO = Path(__file__).resolve().parents[2] TEMPLATE = REPO / "_fleet/mechs_template.md" SITE = "https://culturebotai.github.io/" @@ -60,12 +79,19 @@ "MediaIngredientMech": ("json", "MediaIngredientMech/data/ingredients.json", "ingredients"), } -CARD = re.compile(r'data-mech="([A-Za-z]+)".*?
([\d,]+)', re.S) - +# Where inside a source the figure must be read, when the source states the same +# label elsewhere too. CultureMech's README says "merged records" in its prose; +# only the block its generator writes, and its CI keeps in sync with the data, +# is the figure (#176). Markers missing is a shape change, not a pass. +REGIONS: dict[str, tuple[str, str]] = { + "CultureMech": ("", ""), +} -def cards(template: str) -> dict[str, int]: - """The headline figure each card states, keyed by Mech.""" - return {m.group(1): int(m.group(2).replace(",", "")) for m in CARD.finditer(template)} +# How far a site may be ahead of its card before the card counts as stale. At +# the 2026-09-24 refresh the two fastest Mechs were 0.7% and 6.5% ahead of their +# pins when its audit was written; a mistyped or misattributed card is rarely +# that close (#148). +GROWTH_TOLERANCE = 0.10 def source_url(path: str) -> str: @@ -78,6 +104,16 @@ def fetch(url: str, timeout: int = 30) -> str: return urllib.request.urlopen(request, timeout=timeout).read().decode("utf-8", "replace") +def region(body: str, markers: tuple[str, str]) -> str | None: + """The text between two markers, or None when either is missing.""" + start, end = markers + _, found, rest = body.partition(start) + if not found: + return None + inside, found, _ = rest.partition(end) + return inside if found else None + + def published(kind: str, body: str, selector: str) -> int | None: """The figure the site publishes, or None when the shape has changed.""" if kind == "json": @@ -111,52 +147,96 @@ def published(kind: str, body: str, selector: str) -> int | None: return int(hit.group(1).replace(",", "")) if hit else None -def main() -> int: - stated = cards(TEMPLATE.read_text()) - missing = sorted(set(stated) - set(SOURCES)) - drifted, unreadable, matched = [], [], [] - +def classify(card: int, site: int) -> str: + """How a published figure relates to the card that states it.""" + if site == card: + return "ok" + if site < card: + return "SHRANK" + return "grew" if site - card <= card * GROWTH_TOLERANCE else "STALE" + + +def read_source(mech: str, kind: str, path: str, selector: str, fetcher=None) -> tuple[str, int | str]: + """The figure a source publishes, or why there is none. + + Returns ("value", N), or a status and the reason: "GONE" for a 4xx, "unread" + for a fetch that did not arrive, "CHANGED" for a body with no figure in it. + """ + fetcher = fetcher or fetch + try: + body = fetcher(source_url(path)) + except urllib.error.HTTPError as error: + # HTTPError is a URLError, so it is caught first. A 4xx is the source + # telling us it is not there; a 5xx is the host having a bad night. + status = "GONE" if 400 <= error.code < 500 else "unread" + return status, f"fetch failed: {error}" + except (urllib.error.URLError, TimeoutError, OSError) as error: + return "unread", f"fetch failed: {error}" + if mech in REGIONS: + body = region(body, REGIONS[mech]) + if body is None: + return "CHANGED", "the generated block that states the figure is gone" + try: + value = published(kind, body, selector) + except (ValueError, TypeError, AttributeError) as error: + # json.JSONDecodeError is a ValueError. The other two are what a + # body of an unexpected type raises when it is walked. + return "CHANGED", f"unparseable: {error}" + if value is None: + return "CHANGED", f"{kind} shape changed; no {selector!r} found" + return "value", value + + +FAILURES = ("STALE", "SHRANK", "GONE", "CHANGED", "UNCARDED", "UNCHECKED") + + +def check(stated: dict[str, int], fetcher=None) -> list[tuple[str, str, str]]: + """One (status, mech, detail) row per source, plus the run-level verdicts.""" + rows = [] + for mech in sorted(set(stated) - set(SOURCES)): + rows.append(("UNCARDED", mech, "card with no entry in SOURCES")) for mech, (kind, path, selector) in sorted(SOURCES.items()): if mech not in stated: - unreadable.append((mech, "no card in the template")) - continue - try: - body = fetch(source_url(path)) - except (urllib.error.URLError, TimeoutError, OSError) as error: - unreadable.append((mech, f"fetch failed: {error}")) + rows.append(("UNCARDED", mech, "SOURCES entry with no card in the template")) continue - try: - value = published(kind, body, selector) - except (ValueError, TypeError, AttributeError) as error: - # json.JSONDecodeError is a ValueError. The other two are what a - # body of an unexpected type raises when it is walked. - unreadable.append((mech, f"unparseable: {error}")) + status, result = read_source(mech, kind, path, selector, fetcher) + if status != "value": + rows.append((status, mech, result)) continue - if value is None: - unreadable.append((mech, f"{kind} shape changed; no {selector!r} found")) - elif value != stated[mech]: - drifted.append((mech, stated[mech], value)) - else: - matched.append(mech) - - for mech in matched: - print(f" ok {mech:<20} {stated[mech]:>9,}") - for mech, card, site in drifted: - print(f" DRIFTED {mech:<20} card {card:,}, site {site:,}") - for mech, why in unreadable: - print(f" unread {mech:<20} {why}") - if missing: - print(f" MISSING cards with no entry in SOURCES: {', '.join(missing)}") - - print(f"\n{len(matched)} match, {len(drifted)} drifted, {len(unreadable)} unreadable.") - if drifted or missing: - print("Refresh the card figures in _fleet/mechs_template.md and the MECHS block " - "in _fleet/fleet_fragment.html, then rerun assemble_page.py.") + card = stated[mech] + verdict = classify(card, result) + detail = f"{card:>9,}" if verdict == "ok" else f"card {card:,}, site {result:,}" + if verdict == "grew": + detail += f" (+{(result - card) / card:.1%}, within {GROWTH_TOLERANCE:.0%})" + rows.append((verdict, mech, detail)) + unread = sum(1 for status, _, _ in rows if status == "unread") + if unread * 2 > len(SOURCES): + # One site down is someone else's outage. Most of them down is this run + # having checked nothing, which must not read as a pass (#115). + rows.append(("UNCHECKED", "-", f"{unread} of {len(SOURCES)} sources could not be read")) + return rows + + +def main() -> int: + rows = check(card_figures(TEMPLATE.read_text())) + for status, mech, detail in rows: + print(f" {status:<9} {mech:<20} {detail}") + tally: dict[str, int] = {} + for status, _, _ in rows: + tally[status] = tally.get(status, 0) + 1 + print("\n" + ", ".join(f"{count} {status.lower()}" for status, count in sorted(tally.items())) + ".") + failed = [status for status, _, _ in rows if status in FAILURES] + if failed: + print("Failing. STALE or SHRANK: refresh the card figures in _fleet/mechs_template.md " + "and the MECHS block in _fleet/fleet_fragment.html, then rerun assemble_page.py. " + "GONE or CHANGED: repoint that Mech's SOURCES entry. UNCARDED: add the missing " + "card or SOURCES entry. UNCHECKED: the run could not reach most sites.") return 1 - if unreadable: - # A site that cannot be reached is not the same as a wrong number, and a - # nightly red for someone else's outage teaches people to ignore it. - print("Nothing drifted, but some sites could not be read; see above.") + if any(status in ("grew", "unread") for status, _, _ in rows): + # A site a little ahead of its pinned card, or one that could not be + # reached, is not a wrong number, and a nightly red for either teaches + # people to ignore it. + print("Passing with warnings; see above.") return 0 diff --git a/tests/test_fleet_page.py b/tests/test_fleet_page.py index 6b02746..a7774d6 100644 --- a/tests/test_fleet_page.py +++ b/tests/test_fleet_page.py @@ -1,6 +1,7 @@ """Regression coverage for fleet admission, capability drift and generated output.""" from copy import deepcopy import contextlib +import io import json import os from pathlib import Path @@ -13,7 +14,8 @@ ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "scripts/fleet")) -from assemble_page import CARD_RECORDS, assemble, capability_rows, number_word, script_json +from assemble_page import assemble, capability_rows, number_word, script_json +from card_markup import card_figures, card_names from refresh_manifest import ARTIFACT_PATH, MANIFEST_PATH, read_canonical, semantic, validate import roots @@ -35,7 +37,7 @@ def test_rendering_twice_does_not_consume_the_census(self): def test_records_tile_equals_the_sum_of_the_cards(self): page = self.render() - total = sum(int(n.replace(",", "")) for n in CARD_RECORDS.findall(self.template)) + total = sum(card_figures(self.template).values()) self.assertIn(f"
{total:,}curated entries across the fleet
", page) def test_a_card_without_a_record_count_cannot_be_left_out_of_the_total(self): @@ -400,7 +402,7 @@ def test_every_card_has_a_published_source(self): # would otherwise get a card and be silently exempt from checking, # which is the failure this whole issue is about. import check_cards - stated = check_cards.cards((ROOT / "_fleet/mechs_template.md").read_text()) + stated = card_figures((ROOT / "_fleet/mechs_template.md").read_text()) self.assertEqual(sorted(set(stated) - set(check_cards.SOURCES)), [], "card with no entry in check_cards.SOURCES") self.assertEqual(sorted(set(check_cards.SOURCES) - set(stated)), [], @@ -430,9 +432,126 @@ def test_a_relative_source_is_read_from_the_pages_host(self): def test_the_card_parser_reads_every_member(self): snapshot = json.loads((ROOT / "_fleet/data/manifest.json").read_text()) + template = (ROOT / "_fleet/mechs_template.md").read_text() + self.assertEqual(sorted(card_figures(template)), sorted(snapshot["mechs"])) + self.assertEqual(sorted(card_names(template)), sorted(snapshot["mechs"])) + + def test_a_card_without_a_figure_does_not_borrow_its_neighbours(self): + # The old check_cards regex paired a name with the next figure in the + # file, so the first card here would have been read as 20 (#114). + template = ('

A

\n' + '
20
') + self.assertEqual(card_figures(template), {"BMech": 20}) + self.assertEqual(card_names(template), ["AMech", "BMech"]) + + +class CardCheckTests(unittest.TestCase): + """What the nightly card check fails on and what it only warns about.""" + + def setUp(self): import check_cards - stated = check_cards.cards((ROOT / "_fleet/mechs_template.md").read_text()) - self.assertEqual(sorted(stated), sorted(snapshot["mechs"])) + from unittest import mock + self.check_cards = check_cards + self.real_sources = dict(check_cards.SOURCES) + self.sources = mock.patch.dict(check_cards.SOURCES, { + "AMech": ("html", "AMech/pages/index.html", "a records"), + "BMech": ("html", "BMech/pages/index.html", "b records"), + "CMech": ("html", "CMech/pages/index.html", "c records"), + }, clear=True) + self.sources.start() + self.addCleanup(self.sources.stop) + self.cards = {"AMech": 1000, "BMech": 1000, "CMech": 1000} + self.site = {"AMech": 1000, "BMech": 1000, "CMech": 1000} + + def fetch(self, url): + mech = url.split("/")[3] + value = self.site[mech] + if isinstance(value, int): + return f"
{value:,}{mech[0].lower()} records
" + if isinstance(value, Exception): + raise value + return value + + def statuses(self): + return {mech: status for status, mech, _ in self.check_cards.check(self.cards, self.fetch)} + + def failed(self): + return [status for status, _, _ in self.check_cards.check(self.cards, self.fetch) + if status in self.check_cards.FAILURES] + + def test_matching_figures_pass(self): + self.assertEqual(set(self.statuses().values()), {"ok"}) + self.assertEqual(self.failed(), []) + + def test_a_site_a_little_ahead_of_its_pinned_card_only_warns(self): + # #148: fast Mechs publish within hours of a refresh's pins. + self.site["AMech"] = 1100 + self.assertEqual(self.statuses()["AMech"], "grew") + self.assertEqual(self.failed(), []) + + def test_a_site_far_ahead_of_its_card_fails(self): + self.site["AMech"] = 1101 + self.assertEqual(self.statuses()["AMech"], "STALE") + self.assertEqual(self.failed(), ["STALE"]) + + def test_a_site_behind_its_card_fails(self): + self.site["AMech"] = 999 + self.assertEqual(self.failed(), ["SHRANK"]) + + def test_a_missing_page_fails_but_an_outage_only_warns(self): + # #176: a 404 is the cited page gone, which is how #175 began. + import urllib.error + self.site["AMech"] = urllib.error.HTTPError("u", 404, "Not Found", {}, None) + self.site["BMech"] = urllib.error.HTTPError("u", 503, "Unavailable", {}, None) + statuses = self.statuses() + self.assertEqual((statuses["AMech"], statuses["BMech"]), ("GONE", "unread")) + self.assertEqual(self.failed(), ["GONE"]) + + def test_a_page_with_no_figure_fails(self): + self.site["AMech"] = "
1,000entries
" + self.assertEqual(self.failed(), ["CHANGED"]) + + def test_one_unreachable_site_warns_but_most_unreachable_fails(self): + # #115: a run that read nothing used to exit 0. + import urllib.error + self.site["AMech"] = urllib.error.URLError("no route") + self.assertEqual(self.failed(), []) + self.site["BMech"] = TimeoutError("timed out") + self.assertEqual(self.failed(), ["UNCHECKED"]) + + def test_a_card_and_its_source_must_both_exist(self): + del self.cards["CMech"] + self.cards["DMech"] = 5 + self.assertEqual(sorted(self.failed()), ["UNCARDED", "UNCARDED"]) + + def test_the_culturemech_figure_is_read_only_from_its_generated_block(self): + # #176: the regex takes the first match, and the README's prose could + # state an older "N merged records" above the generated block. + kind, path, selector = self.real_sources["CultureMech"] + begin, end = self.check_cards.REGIONS["CultureMech"] + readme = ("Release 2 added 1,024 merged records.\n" + begin + + "\nThe tracked corpus currently contains **15,878 normalized records** and " + "**6,288 merged records**.\n" + end + "\n") + read = self.check_cards.read_source + self.assertEqual(read("CultureMech", kind, path, selector, lambda url: readme), ("value", 6288)) + status, _ = read("CultureMech", kind, path, selector, + lambda url: "The corpus has 6,288 merged records.") + self.assertEqual(status, "CHANGED") + + def test_main_exits_by_the_same_rule(self): + from unittest import mock + template = "".join(f'
{n:,}
' + for m, n in self.cards.items()) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "mechs_template.md" + path.write_text(template) + with mock.patch.object(self.check_cards, "TEMPLATE", path), \ + mock.patch.object(self.check_cards, "fetch", self.fetch), \ + contextlib.redirect_stdout(io.StringIO()): + self.site["AMech"] = 1050 + self.assertEqual(self.check_cards.main(), 0) + self.site["AMech"] = 1500 + self.assertEqual(self.check_cards.main(), 1) class RefreshProvenanceTests(unittest.TestCase): """The derived numbers must all come from one set of checkouts (#85). @@ -480,8 +599,7 @@ def test_the_audit_pins_the_revisions_the_stats_counted(self): def test_the_audit_records_this_refresh_and_nothing_else(self): # The audit's other fields had no gate at all (#128): its merged PRs, # the card figure it read, the CLAW pin, and which repositories it lists. - import check_cards - cards = check_cards.cards((ROOT / "_fleet/mechs_template.md").read_text()) + cards = card_figures((ROOT / "_fleet/mechs_template.md").read_text()) manifest = json.loads((ROOT / "_fleet/data/manifest.json").read_text()) self.assertEqual(set(self.audit), {m["repo"] for m in self.stats.values()} | {"culturebotai-claw"}) self.assertEqual(self.audit["culturebotai-claw"]["sha"], manifest["source"]["revision"]) From 2b25441b7126b158972e22f1abaf6498f560283f Mon Sep 17 00:00:00 2001 From: "marcin p. joachimiak" <4625870+realmarcin@users.noreply.github.com> Date: Fri, 25 Sep 2026 13:27:17 -0700 Subject: [PATCH 2/8] Judge a card against its audit, not a fixed percentage A 10% allowance went red a day after every refresh, because CellStructureMech adds about 9% a day, and it let a card mistyped inside 10% pass (#217). Growth now warns for 14 days after the pins in site_audit.json while the site is at most half as large again as the card, and a card that differs from a source still byte-identical to its pin fails as WRONG. Throttles (408, 425, 429) and a connection dropped mid-body are unread, not GONE or a traceback (#219, #220). A card without exactly one headline figure, or a figure outside every card, is a MARKUP failure here and an error in the assembler (#218). The update skill derives audit figures through read_source(), which applies REGIONS (#221), and the tests cover the parse-exception path, the more-than-half boundary and a half-marked region (#222). Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/skills/review-open-issues/SKILL.md | 6 +- .claude/skills/update-xmech-page/SKILL.md | 10 +- _fleet/README.md | 31 ++-- scripts/fleet/assemble_page.py | 8 +- scripts/fleet/card_markup.py | 29 +++- scripts/fleet/check_cards.py | 189 ++++++++++++++------- tests/test_fleet_page.py | 161 +++++++++++++----- 7 files changed, 310 insertions(+), 124 deletions(-) diff --git a/.claude/skills/review-open-issues/SKILL.md b/.claude/skills/review-open-issues/SKILL.md index 1fbeaa1..6e7d26f 100644 --- a/.claude/skills/review-open-issues/SKILL.md +++ b/.claude/skills/review-open-issues/SKILL.md @@ -217,9 +217,9 @@ python scripts/fleet/check_cards.py # card headline figures vs each Mech's sit ``` A stale card therefore shows up as a failed scheduled run, not a failed PR -check. A card a little behind a fast Mech only warns ("grew", within 10%), so a -green run does not mean every card equals its site: read the run's log, not -just its colour, before crediting the cards as current. +check. A card behind a fast Mech only warns ("grew") for 14 days after the +refresh's pins, so a green run does not mean every card equals its site: read +the run's log, not just its colour, before crediting the cards as current. An issue asserting a defect that one of these already blocks is P2 unless it shows the gate is porous — and they have been porous: a test can pass because diff --git a/.claude/skills/update-xmech-page/SKILL.md b/.claude/skills/update-xmech-page/SKILL.md index 9d5520b..fcee27e 100644 --- a/.claude/skills/update-xmech-page/SKILL.md +++ b/.claude/skills/update-xmech-page/SKILL.md @@ -233,7 +233,7 @@ test compares a value with itself, #125) and its commit date, the URL each card sha256 of the fetched HTML and of any data file, merged PRs, and short notes on how the site figure relates to the repo count. Set `checked_at_utc`, `local_date`, `pinned_at_utc` and `scope`. Derive the mechanical fields rather -than typing them: the figure through `check_cards.published()`, merged PRs from +than typing them: the figure through `check_cards.read_source()`, which applies `REGIONS`, merged PRs from `mech_stats.json`, SHAs and commit dates from the pins, and assert that the pins equal the stats' `source_revision` before writing. Hash the served page as committed at the pin too (`git show :pages/index.html`, or `docs/`), record @@ -265,8 +265,12 @@ overlaps are computed across Mechs, so a single re-pin is a partial rerun, and a fast Mech moves again before the rerun finishes. Keep the page a consistent snapshot at the pins, record the live figure as `site_figure_at_check` in that Mech's `site_audit.json` entry, and say in the PR which cards the check reports -as grown. Within `GROWTH_TOLERANCE` (10%) that is a warning; past it the check -reports STALE and fails, and the refresh should re-pin everything. +as grown. That stays a warning for `GRACE_DAYS` (14) after `pinned_at_utc` while +the site is at most half as large again as the card; past either limit the +check reports STALE and fails, and the page is due a full refresh. The check +also reads each `*_sha256_at_pin` in the audit, so record them for every source +the check reads: a card that differs from a source still byte-identical to its +pin fails as WRONG. Emoji headings render with a leading hyphen in their id on GitHub Pages. Verify anchors against the deployed HTML, not a local kramdown. diff --git a/_fleet/README.md b/_fleet/README.md index c2f6e17..baa1f4b 100644 --- a/_fleet/README.md +++ b/_fleet/README.md @@ -40,18 +40,24 @@ python3 scripts/fleet/check_cards.py `SOURCES` at its top pins where each Mech publishes its count, and `REGIONS` restricts a source to the part that states it where the same words appear elsewhere (CultureMech's generated README block). The page is a snapshot at a -refresh's pins, so a site up to 10% ahead of its card (`GROWTH_TOLERANCE`) only -warns. It fails when a site is further ahead than that or behind its card, when -a source answers 4xx or no longer states a figure the parser can read, when a -card and a `SOURCES` entry do not pair up, and when more than half the sources -could not be fetched at all. One site's outage only warns (#148, #115, #176). -It runs on the workflow's nightly schedule, not on pull requests, so a Mech -shipping records overnight does not block an unrelated change; a nightly red -means either the card figures in `mechs_template.md` and the `MECHS` block in -`fleet_fragment.html` need refreshing together, or a `SOURCES` entry needs +refresh's pins, so a site ahead of its card only warns ("grew") for +`GRACE_DAYS` (14) after the pins in `site_audit.json`, and only while the site is +at most `MAX_LEAD` (50%) ahead. Past either limit it fails as STALE. It also +fails when a source is byte-identical to its copy at the pin but states a +different figure (the card was never right), when a site is behind its card, +when a source answers a 4xx other than a throttle or no longer states a figure +the parser can read, when a card lacks exactly one headline figure or has no +`SOURCES` entry, and when more than half the sources could not be fetched. One +site's outage or throttle only warns (#148, #115, #176, #217-#220). It runs on +the workflow's nightly schedule, not on pull requests, so a Mech shipping +records overnight does not block an unrelated change. A nightly red means one +of three things: the card figures in `mechs_template.md` and the `MECHS` block +in `fleet_fragment.html` need refreshing together (a full refresh, since the +page is a snapshot), a card figure is wrong, or a `SOURCES` entry needs repointing. A new card needs a `SOURCES` entry; a test enforces that. The cards are read by `scripts/fleet/card_markup.py`, the one parser the assembler, this -check and the tests share (#114). +check and the tests share, and the assembler refuses a card without exactly one +headline figure (#114, #218). The `Fleet page` workflow checks pull requests, pushes and the live CLAW manifest daily. It detects changes to membership, capability declarations (including @@ -188,7 +194,10 @@ its site lists 422 communities, while its record glob also takes four isolate records, so the census and `mech_stats.json` count 426. CellStructureMech and TraitMech published new records after the pins were taken; their cards keep the pinned figures, and `site_audit.json` records what the two sites showed when it -was written. `check_cards.py` reports both as grown, a warning, until the next refresh. +was written. `check_cards.py` reports both as grown, a warning, until 14 days +after the pins or until a site is half as large again as its card, whichever +comes first; CellStructureMech, adding about two records an hour, reaches the +second within a week. NaturalProductMech's landing page and MediaIngredientMech's data file also changed after the pins without changing their figures; the audit records each live hash beside the hash of the committed copy at the pin. diff --git a/scripts/fleet/assemble_page.py b/scripts/fleet/assemble_page.py index 6ed8e57..4333968 100644 --- a/scripts/fleet/assemble_page.py +++ b/scripts/fleet/assemble_page.py @@ -6,7 +6,7 @@ from pathlib import Path import re -from card_markup import card_figures, card_names +from card_markup import card_figures, card_names, markup_problems from refresh_manifest import validate REPO = Path(__file__).resolve().parents[2] @@ -71,6 +71,12 @@ def fleet_records(template): check_cards.py also uses, so the total and the nightly check cannot read the markup differently (#114). """ + # Exactly one figure per card, and none outside the cards: a second stat + # tile used to replace a card's headline in the total without failing (#218). + problems = markup_problems(template) + if problems: + raise ValueError("Every Mech card must carry a record count, exactly once: " + + "; ".join(f"{mech}: {why}" for mech, why in problems)) figures = card_figures(template) if not figures: raise ValueError("No Mech card record counts found") diff --git a/scripts/fleet/card_markup.py b/scripts/fleet/card_markup.py index 2b85f09..f27c348 100644 --- a/scripts/fleet/card_markup.py +++ b/scripts/fleet/card_markup.py @@ -18,15 +18,36 @@ def card_names(template: str) -> list[str]: return [mech for mech, _ in ARTICLE.findall(template)] +def markup_problems(template: str) -> list[tuple[str, str]]: + """(Mech, what is wrong) for every card that does not state exactly one figure. + + A card with two figures used to be read as its first, and a figure outside + every card was ignored, so a second stat tile changed the fleet total without + failing anything (#218). "-" stands for a figure that belongs to no card. + """ + problems = [] + for mech, body in ARTICLE.findall(template): + count = len(FIGURE.findall(body)) + if count == 0: + problems.append((mech, "card has no readable headline figure")) + elif count > 1: + problems.append((mech, f"card has {count} headline figures; it must have exactly one")) + outside = len(FIGURE.findall(ARTICLE.sub("", template))) + if outside: + problems.append(("-", f"{outside} headline figure(s) outside any card")) + return problems + + def card_figures(template: str) -> dict[str, int]: """Each card's headline figure, keyed by Mech. Read inside the card's own
, so a card without a figure is absent - rather than borrowing the next card's. + rather than borrowing the next card's. A card with more than one figure is + absent too; markup_problems() says which cards those are. """ figures: dict[str, int] = {} for mech, body in ARTICLE.findall(template): - hit = FIGURE.search(body) - if hit: - figures[mech] = int(hit.group(1).replace(",", "")) + hits = FIGURE.findall(body) + if len(hits) == 1: + figures[mech] = int(hits[0].replace(",", "")) return figures diff --git a/scripts/fleet/check_cards.py b/scripts/fleet/check_cards.py index 14e53cf..ff77ce3 100644 --- a/scripts/fleet/check_cards.py +++ b/scripts/fleet/check_cards.py @@ -12,30 +12,40 @@ that a blocking check would make an unrelated docs fix unmergeable because some Mech published records overnight. -What fails and what only warns (#148, #115, #176): +What fails and what only warns (#148, #115, #176, #217): ok the site states the card's figure. - grew the site is ahead of the card by at most GROWTH_TOLERANCE. A warning: - the page is a snapshot at a refresh's pins and fast Mechs publish - within hours of them, so a small lead is the expected state, not a - wrong card. site_audit.json records the lead the refresh saw. - STALE the site is further ahead than that. The card no longer describes the - Mech; refresh it. + grew the site is ahead of the card, the refresh that pinned the card is + at most GRACE_DAYS old, and the site is at most MAX_LEAD ahead. A + warning: the page is a snapshot at the refresh's pins, and fast + Mechs publish within hours of them. + STALE the site is ahead and the refresh is older than that, or the card + understates the site by more than MAX_LEAD. Refresh the page. + WRONG the source is byte-identical to its copy at the pin, which the + audit recorded, yet states a different figure. The site has not + moved, so the card was never right (#217). SHRANK the site states fewer than the card. Records are not normally withdrawn in bulk, so either the card is wrong or the site regressed. - GONE the source answered 4xx. The page the card cites has moved or been - deleted, which is how #175 began; an outage does not look like this. + GONE the source answered a 4xx other than a throttle. The page the card + cites has moved or been deleted, which is how #175 began. CHANGED the source arrived but no figure could be read from it: the markup or the wording moved, and the card is no longer being checked at all. - unread the fetch did not arrive (DNS, timeout, 5xx). A warning, because the - network is not the site's fault, unless more than half the sources - are unread, when the run has verified too little to call itself a - pass and fails as UNCHECKED. - -A card with no SOURCES entry, or an entry with no card, also fails. + MARKUP a card in the template does not carry exactly one headline figure. + UNCARDED a card with no SOURCES entry, or an entry with no card. + unread the fetch did not arrive: DNS, timeout, a dropped connection, a + 5xx, or a 408, 425 or 429 throttle. A warning, because the network is + not the site's fault, unless more than half the sources are unread, + when the run has verified too little to call itself a pass and fails + as UNCHECKED. + +The pins, the pin time and the hash of each source at its pin come from +_fleet/data/site_audit.json, which the refresh writes (update-xmech-page, step 7). """ from __future__ import annotations +import datetime +import hashlib +import http.client import json import re import sys @@ -43,10 +53,11 @@ import urllib.request from pathlib import Path -from card_markup import card_figures +from card_markup import card_figures, card_names, markup_problems REPO = Path(__file__).resolve().parents[2] TEMPLATE = REPO / "_fleet/mechs_template.md" +AUDIT = REPO / "_fleet/data/site_audit.json" SITE = "https://culturebotai.github.io/" # Where each card's headline actually comes from. Pinned here rather than @@ -87,11 +98,19 @@ "CultureMech": ("", ""), } -# How far a site may be ahead of its card before the card counts as stale. At -# the 2026-09-24 refresh the two fastest Mechs were 0.7% and 6.5% ahead of their -# pins when its audit was written; a mistyped or misattributed card is rarely -# that close (#148). -GROWTH_TOLERANCE = 0.10 +# How long a card may trail a growing site before the page counts as stale, and +# how far it may trail within that time. A fixed percentage alone did not hold: +# CellStructureMech adds about two records an hour, 9% a day on a card of 542, +# so a 10% allowance went red a day after every refresh (#217). The time limit +# asks for a refresh a fortnight after the pins while any Mech grows; the lead +# limit asks sooner once a site is more than half as large again as its card +# (the card then understates it by over a third), which CellStructureMech +# reaches in under a week. +GRACE_DAYS = 14 +MAX_LEAD = 0.5 + +# 4xx answers that mean "not now" rather than "not here" (#219). +THROTTLES = (408, 425, 429) def source_url(path: str) -> str: @@ -147,67 +166,110 @@ def published(kind: str, body: str, selector: str) -> int | None: return int(hit.group(1).replace(",", "")) if hit else None -def classify(card: int, site: int) -> str: - """How a published figure relates to the card that states it.""" - if site == card: - return "ok" - if site < card: - return "SHRANK" - return "grew" if site - card <= card * GROWTH_TOLERANCE else "STALE" - - -def read_source(mech: str, kind: str, path: str, selector: str, fetcher=None) -> tuple[str, int | str]: - """The figure a source publishes, or why there is none. +def read_source(mech: str, kind: str, path: str, selector: str, fetcher=None) -> tuple[str, int | str, str | None]: + """The figure a source publishes, or why there is none, and the source's hash. - Returns ("value", N), or a status and the reason: "GONE" for a 4xx, "unread" - for a fetch that did not arrive, "CHANGED" for a body with no figure in it. + Returns ("value", N, sha256 of the body), or a status, the reason and None: + "GONE" for a 4xx other than a throttle, "unread" for a fetch that did not + arrive, "CHANGED" for a body with no figure in it. """ fetcher = fetcher or fetch try: body = fetcher(source_url(path)) except urllib.error.HTTPError as error: # HTTPError is a URLError, so it is caught first. A 4xx is the source - # telling us it is not there; a 5xx is the host having a bad night. - status = "GONE" if 400 <= error.code < 500 else "unread" - return status, f"fetch failed: {error}" - except (urllib.error.URLError, TimeoutError, OSError) as error: - return "unread", f"fetch failed: {error}" + # telling us it is not there, unless it is asking us to come back later. + gone = 400 <= error.code < 500 and error.code not in THROTTLES + return ("GONE" if gone else "unread"), f"fetch failed: {error}", None + except (urllib.error.URLError, http.client.HTTPException, TimeoutError, OSError) as error: + # HTTPException covers a body cut short (IncompleteRead) and a garbled + # status line, which used to end the run with a traceback (#220). + return "unread", f"fetch failed: {error}", None + digest = hashlib.sha256(body.encode("utf-8")).hexdigest() if mech in REGIONS: body = region(body, REGIONS[mech]) if body is None: - return "CHANGED", "the generated block that states the figure is gone" + return "CHANGED", "the generated block that states the figure is gone", digest try: value = published(kind, body, selector) except (ValueError, TypeError, AttributeError) as error: # json.JSONDecodeError is a ValueError. The other two are what a # body of an unexpected type raises when it is walked. - return "CHANGED", f"unparseable: {error}" + return "CHANGED", f"unparseable: {error}", digest if value is None: - return "CHANGED", f"{kind} shape changed; no {selector!r} found" - return "value", value + return "CHANGED", f"{kind} shape changed; no {selector!r} found", digest + return "value", value, digest -FAILURES = ("STALE", "SHRANK", "GONE", "CHANGED", "UNCARDED", "UNCHECKED") +def pinned_hash(entry: dict, url: str) -> str | None: + """The audit's hash of this source at the pin, when the audit has one. + + ProteinTraitsMech's data file is built in CI rather than committed, so there + is no copy at the pin to hash, and its card cannot be shown WRONG this way. + """ + if entry.get("data_url") == url: + return entry.get("data_sha256_at_pin") + if entry.get("site") == url: + return entry.get("site_html_sha256_at_pin") + return None -def check(stated: dict[str, int], fetcher=None) -> list[tuple[str, str, str]]: - """One (status, mech, detail) row per source, plus the run-level verdicts.""" - rows = [] - for mech in sorted(set(stated) - set(SOURCES)): +def classify(card: int, site: int, digest: str | None, pinned: str | None, + age: datetime.timedelta | None) -> str: + """How a published figure relates to the card that states it.""" + if site == card: + return "ok" + if digest and pinned and digest == pinned: + return "WRONG" + if site < card: + return "SHRANK" + if age is None or age > datetime.timedelta(days=GRACE_DAYS): + return "STALE" + return "grew" if site - card <= card * MAX_LEAD else "STALE" + + +def span(age: datetime.timedelta) -> str: + """A pin's age in the unit a reader wants: hours on the first day, then days.""" + if age.days >= 1: + return f"{age.days} day{'s' if age.days != 1 else ''}" + hours = int(age.total_seconds() // 3600) + return f"{hours} hour{'s' if hours != 1 else ''}" + + +FAILURES = ("STALE", "WRONG", "SHRANK", "GONE", "CHANGED", "MARKUP", "UNCARDED", "UNCHECKED") + + +def check(template: str, fetcher=None, audit: dict | None = None, + now: datetime.datetime | None = None) -> list[tuple[str, str, str]]: + """One (status, mech, detail) row per card problem and per source, plus the run-level verdict.""" + now = now or datetime.datetime.now(datetime.timezone.utc) + audit = audit or {} + entries = {row["repo"].lower(): row for row in audit.get("repositories", [])} + pinned_at = audit.get("pinned_at_utc") + age = now - datetime.datetime.fromisoformat(pinned_at) if pinned_at else None + stated = card_figures(template) + names = set(card_names(template)) + rows = [("MARKUP", mech, why) for mech, why in markup_problems(template)] + for mech in sorted(names - set(SOURCES)): rows.append(("UNCARDED", mech, "card with no entry in SOURCES")) for mech, (kind, path, selector) in sorted(SOURCES.items()): - if mech not in stated: + if mech not in names: rows.append(("UNCARDED", mech, "SOURCES entry with no card in the template")) continue - status, result = read_source(mech, kind, path, selector, fetcher) + if mech not in stated: + continue # its card's markup is already reported above + status, result, digest = read_source(mech, kind, path, selector, fetcher) if status != "value": rows.append((status, mech, result)) continue card = stated[mech] - verdict = classify(card, result) + pinned = pinned_hash(entries.get(mech.lower(), {}), source_url(path)) + verdict = classify(card, result, digest, pinned, age) detail = f"{card:>9,}" if verdict == "ok" else f"card {card:,}, site {result:,}" - if verdict == "grew": - detail += f" (+{(result - card) / card:.1%}, within {GROWTH_TOLERANCE:.0%})" + if verdict == "WRONG": + detail += ", and the source is unchanged since the pin" + elif verdict in ("grew", "STALE") and age is not None: + detail += f" (+{result - card:,} in the {span(age)} since the pins)" rows.append((verdict, mech, detail)) unread = sum(1 for status, _, _ in rows if status == "unread") if unread * 2 > len(SOURCES): @@ -218,24 +280,25 @@ def check(stated: dict[str, int], fetcher=None) -> list[tuple[str, str, str]]: def main() -> int: - rows = check(card_figures(TEMPLATE.read_text())) + audit = json.loads(AUDIT.read_text()) if AUDIT.exists() else None + rows = check(TEMPLATE.read_text(), audit=audit) for status, mech, detail in rows: print(f" {status:<9} {mech:<20} {detail}") tally: dict[str, int] = {} for status, _, _ in rows: tally[status] = tally.get(status, 0) + 1 print("\n" + ", ".join(f"{count} {status.lower()}" for status, count in sorted(tally.items())) + ".") - failed = [status for status, _, _ in rows if status in FAILURES] - if failed: - print("Failing. STALE or SHRANK: refresh the card figures in _fleet/mechs_template.md " - "and the MECHS block in _fleet/fleet_fragment.html, then rerun assemble_page.py. " - "GONE or CHANGED: repoint that Mech's SOURCES entry. UNCARDED: add the missing " - "card or SOURCES entry. UNCHECKED: the run could not reach most sites.") + if any(status in FAILURES for status, _, _ in rows): + print("Failing. STALE, WRONG or SHRANK: refresh the card figures in _fleet/mechs_template.md " + "and the MECHS block in _fleet/fleet_fragment.html (update-xmech-page), then rerun " + "assemble_page.py. GONE or CHANGED: repoint that Mech's SOURCES entry. MARKUP or " + "UNCARDED: fix the card or its SOURCES entry. UNCHECKED: the run could not reach " + "most sites.") return 1 if any(status in ("grew", "unread") for status, _, _ in rows): - # A site a little ahead of its pinned card, or one that could not be - # reached, is not a wrong number, and a nightly red for either teaches - # people to ignore it. + # A site a little ahead of a recently pinned card, or one that could not + # be reached, is not a wrong number, and a nightly red for either + # teaches people to ignore it. print("Passing with warnings; see above.") return 0 diff --git a/tests/test_fleet_page.py b/tests/test_fleet_page.py index a7774d6..b27f39b 100644 --- a/tests/test_fleet_page.py +++ b/tests/test_fleet_page.py @@ -1,6 +1,7 @@ """Regression coverage for fleet admission, capability drift and generated output.""" from copy import deepcopy import contextlib +import datetime import io import json import os @@ -448,111 +449,193 @@ def test_a_card_without_a_figure_does_not_borrow_its_neighbours(self): class CardCheckTests(unittest.TestCase): """What the nightly card check fails on and what it only warns about.""" + NAMES = ("AMech", "BMech", "CMech") + def setUp(self): import check_cards from unittest import mock self.check_cards = check_cards self.real_sources = dict(check_cards.SOURCES) self.sources = mock.patch.dict(check_cards.SOURCES, { - "AMech": ("html", "AMech/pages/index.html", "a records"), - "BMech": ("html", "BMech/pages/index.html", "b records"), - "CMech": ("html", "CMech/pages/index.html", "c records"), + m: ("html", f"{m}/pages/index.html", f"{m[0].lower()} records") for m in self.NAMES }, clear=True) self.sources.start() self.addCleanup(self.sources.stop) - self.cards = {"AMech": 1000, "BMech": 1000, "CMech": 1000} - self.site = {"AMech": 1000, "BMech": 1000, "CMech": 1000} + self.cards = {m: 1000 for m in self.NAMES} + self.site = {m: 1000 for m in self.NAMES} + self.pinned_at = datetime.datetime(2026, 9, 25, 2, 0, tzinfo=datetime.timezone.utc) + self.now = self.pinned_at + datetime.timedelta(hours=18) + # By default each source has moved since its pin, so a lead reads as growth. + self.pinned = {m: "0" * 64 for m in self.NAMES} + + def body(self, mech, value): + return f"
{value:,}{mech[0].lower()} records
" def fetch(self, url): mech = url.split("/")[3] value = self.site[mech] - if isinstance(value, int): - return f"
{value:,}{mech[0].lower()} records
" if isinstance(value, Exception): raise value - return value + return self.body(mech, value) if isinstance(value, int) else value + + def template(self): + return "".join(f'
{n:,}
' + for m, n in self.cards.items()) + + def audit(self): + return {"pinned_at_utc": self.pinned_at.isoformat(), "repositories": [ + {"repo": m, "site": self.check_cards.source_url(self.check_cards.SOURCES[m][1]), + "site_html_sha256_at_pin": self.pinned[m]} + for m in self.check_cards.SOURCES]} + + def rows(self, template=None): + return self.check_cards.check(template or self.template(), self.fetch, self.audit(), self.now) def statuses(self): - return {mech: status for status, mech, _ in self.check_cards.check(self.cards, self.fetch)} + return {mech: status for status, mech, _ in self.rows()} - def failed(self): - return [status for status, _, _ in self.check_cards.check(self.cards, self.fetch) - if status in self.check_cards.FAILURES] + def failed(self, template=None): + return sorted(status for status, _, _ in self.rows(template) if status in self.check_cards.FAILURES) def test_matching_figures_pass(self): self.assertEqual(set(self.statuses().values()), {"ok"}) self.assertEqual(self.failed(), []) - def test_a_site_a_little_ahead_of_its_pinned_card_only_warns(self): - # #148: fast Mechs publish within hours of a refresh's pins. - self.site["AMech"] = 1100 + def test_a_site_ahead_of_a_recent_pin_only_warns(self): + # #148, #217: CellStructureMech adds about 9% a day; within the grace + # period that is growth, not a wrong card. + self.site["AMech"] = 1400 self.assertEqual(self.statuses()["AMech"], "grew") self.assertEqual(self.failed(), []) - def test_a_site_far_ahead_of_its_card_fails(self): - self.site["AMech"] = 1101 - self.assertEqual(self.statuses()["AMech"], "STALE") + def test_growth_past_the_grace_period_fails(self): + self.site["AMech"] = 1001 + self.now = self.pinned_at + datetime.timedelta(days=self.check_cards.GRACE_DAYS, hours=1) self.assertEqual(self.failed(), ["STALE"]) + self.now = self.pinned_at + datetime.timedelta(days=self.check_cards.GRACE_DAYS) + self.assertEqual(self.failed(), []) + + def test_a_card_that_understates_its_site_by_more_than_the_lead_fails_early(self): + self.site["AMech"] = 1500 + self.assertEqual(self.failed(), []) + self.site["AMech"] = 1501 + self.assertEqual(self.failed(), ["STALE"]) + + def test_growth_without_an_audit_fails(self): + self.site["AMech"] = 1001 + rows = self.check_cards.check(self.template(), self.fetch, None, self.now) + self.assertIn(("STALE", "AMech"), [(s, m) for s, m, _ in rows]) + + def test_a_mistyped_card_fails_when_the_site_has_not_moved(self): + # #217: 3,026 typed for 3,206 used to pass as "grew". + import hashlib + self.cards["AMech"] = 3026 + self.site["AMech"] = 3206 + self.pinned["AMech"] = hashlib.sha256(self.body("AMech", 3206).encode()).hexdigest() + self.assertEqual(self.statuses()["AMech"], "WRONG") + self.assertEqual(self.failed(), ["WRONG"]) def test_a_site_behind_its_card_fails(self): self.site["AMech"] = 999 self.assertEqual(self.failed(), ["SHRANK"]) - def test_a_missing_page_fails_but_an_outage_only_warns(self): - # #176: a 404 is the cited page gone, which is how #175 began. + def test_a_missing_page_fails_but_an_outage_or_throttle_only_warns(self): + # #176: a 404 is the cited page gone, which is how #175 began. #219: a + # 429 is the host asking us to come back later. import urllib.error self.site["AMech"] = urllib.error.HTTPError("u", 404, "Not Found", {}, None) - self.site["BMech"] = urllib.error.HTTPError("u", 503, "Unavailable", {}, None) + self.site["BMech"] = urllib.error.HTTPError("u", 429, "Too Many Requests", {}, None) statuses = self.statuses() self.assertEqual((statuses["AMech"], statuses["BMech"]), ("GONE", "unread")) self.assertEqual(self.failed(), ["GONE"]) + for code in (408, 425, 503): + self.site["BMech"] = urllib.error.HTTPError("u", code, "x", {}, None) + self.assertEqual(self.statuses()["BMech"], "unread", code) + + def test_a_body_cut_short_is_unread_not_a_traceback(self): + # #220 + import http.client + self.site["AMech"] = http.client.IncompleteRead(b"
", 500) + self.assertEqual(self.statuses()["AMech"], "unread") + self.assertEqual(self.failed(), []) def test_a_page_with_no_figure_fails(self): self.site["AMech"] = "
1,000entries
" self.assertEqual(self.failed(), ["CHANGED"]) - def test_one_unreachable_site_warns_but_most_unreachable_fails(self): - # #115: a run that read nothing used to exit 0. + def test_a_json_source_that_is_not_json_fails(self): + # #222: an HTML shell served with 200 where the data file used to be. + self.check_cards.SOURCES["AMech"] = ("json", "AMech/data/ingredients.json", "ingredients") + self.site["AMech"] = "" + self.assertEqual(self.statuses()["AMech"], "CHANGED") + + def test_more_than_half_unread_fails_and_half_does_not(self): + # #115: a run that read nothing used to exit 0. #222: ten sources, so + # "more than half" can be told from "at least half". import urllib.error - self.site["AMech"] = urllib.error.URLError("no route") + names = [f"M{i}Mech" for i in range(10)] + self.check_cards.SOURCES.clear() + self.check_cards.SOURCES.update({m: ("html", f"{m}/pages/index.html", "m records") for m in names}) + self.cards = {m: 1000 for m in names} + self.site = {m: 1000 for m in names} + self.pinned = {m: "0" * 64 for m in names} + for m in names[:5]: + self.site[m] = urllib.error.URLError("no route") self.assertEqual(self.failed(), []) - self.site["BMech"] = TimeoutError("timed out") + self.site[names[5]] = TimeoutError("timed out") self.assertEqual(self.failed(), ["UNCHECKED"]) def test_a_card_and_its_source_must_both_exist(self): del self.cards["CMech"] self.cards["DMech"] = 5 - self.assertEqual(sorted(self.failed()), ["UNCARDED", "UNCARDED"]) + self.assertEqual(self.failed(), ["UNCARDED", "UNCARDED"]) + + def test_a_card_without_exactly_one_figure_is_reported_as_markup(self): + # #218: not as a missing card, and a second figure is not silently dropped. + template = self.template() + unreadable = template.replace('
1,000
', + '
1,000
', 1) + rows = self.rows(unreadable) + self.assertIn(("MARKUP", "AMech"), [(s, m) for s, m, _ in rows]) + self.assertNotIn("UNCARDED", [s for s, _, _ in rows]) + doubled = template.replace('
', + '
7
', 1) + self.assertEqual(self.failed(doubled), ["MARKUP"]) + stray = template + '
7
' + self.assertEqual(self.failed(stray), ["MARKUP"]) def test_the_culturemech_figure_is_read_only_from_its_generated_block(self): # #176: the regex takes the first match, and the README's prose could # state an older "N merged records" above the generated block. kind, path, selector = self.real_sources["CultureMech"] begin, end = self.check_cards.REGIONS["CultureMech"] - readme = ("Release 2 added 1,024 merged records.\n" + begin + - "\nThe tracked corpus currently contains **15,878 normalized records** and " - "**6,288 merged records**.\n" + end + "\n") + line = ("The tracked corpus currently contains **15,878 normalized records** and " + "**6,288 merged records**.") + readme = "Release 2 added 1,024 merged records.\n" + begin + "\n" + line + "\n" + end + "\n" read = self.check_cards.read_source - self.assertEqual(read("CultureMech", kind, path, selector, lambda url: readme), ("value", 6288)) - status, _ = read("CultureMech", kind, path, selector, - lambda url: "The corpus has 6,288 merged records.") - self.assertEqual(status, "CHANGED") + self.assertEqual(read("CultureMech", kind, path, selector, lambda url: readme)[:2], ("value", 6288)) + for broken in ("The corpus has 6,288 merged records.", begin + "\n" + line + "\n"): + self.assertEqual(read("CultureMech", kind, path, selector, lambda url: broken)[0], "CHANGED") def test_main_exits_by_the_same_rule(self): from unittest import mock - template = "".join(f'
{n:,}
' - for m, n in self.cards.items()) + import urllib.error with tempfile.TemporaryDirectory() as tmp: - path = Path(tmp) / "mechs_template.md" - path.write_text(template) - with mock.patch.object(self.check_cards, "TEMPLATE", path), \ + template, audit = Path(tmp) / "mechs_template.md", Path(tmp) / "site_audit.json" + template.write_text(self.template()) + self.pinned_at = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(hours=1) + audit.write_text(json.dumps(self.audit())) + with mock.patch.object(self.check_cards, "TEMPLATE", template), \ + mock.patch.object(self.check_cards, "AUDIT", audit), \ mock.patch.object(self.check_cards, "fetch", self.fetch), \ contextlib.redirect_stdout(io.StringIO()): self.site["AMech"] = 1050 + self.site["BMech"] = urllib.error.URLError("no route") self.assertEqual(self.check_cards.main(), 0) - self.site["AMech"] = 1500 + self.site["AMech"] = 900 self.assertEqual(self.check_cards.main(), 1) + class RefreshProvenanceTests(unittest.TestCase): """The derived numbers must all come from one set of checkouts (#85). From 70e365739c02172c4389b862b9c47681425fd70b Mon Sep 17 00:00:00 2001 From: "marcin p. joachimiak" <4625870+realmarcin@users.noreply.github.com> Date: Fri, 25 Sep 2026 13:48:04 -0700 Subject: [PATCH 3/8] Judge a mistyped card by the figure its source stated at the pin WRONG compared the live source's hash with its pinned copy, so it stopped working the moment a source changed at all, and five of ten already had (#231). site_audit.json now records figure_at_pin, read from each source's committed copy at the pin with check_cards.figure(), the nightly's own parser; a test requires it to equal the card, so a typo fails on the PR, and the nightly reports WRONG from it however far the site has moved. A pin time with no offset is UTC and an unreadable one is an AUDIT failure, not a traceback (#232). The docstring states the lead limit the code applies (#233), the tests cover the assembler's stray-figure refusal, more 4xx codes and main()'s exit for WRONG, STALE and UNCHECKED (#234), and the README names every failing verdict's remedy (#235). Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/skills/update-xmech-page/SKILL.md | 17 ++-- _fleet/README.md | 29 +++--- _fleet/data/site_audit.json | 9 ++ scripts/fleet/check_cards.py | 115 ++++++++++++---------- tests/test_fleet_page.py | 85 +++++++++++++--- 5 files changed, 174 insertions(+), 81 deletions(-) diff --git a/.claude/skills/update-xmech-page/SKILL.md b/.claude/skills/update-xmech-page/SKILL.md index fcee27e..339f511 100644 --- a/.claude/skills/update-xmech-page/SKILL.md +++ b/.claude/skills/update-xmech-page/SKILL.md @@ -232,8 +232,14 @@ Rewrite `_fleet/data/site_audit.json` for the run: per repository the pinned test compares a value with itself, #125) and its commit date, the URL each card figure is read from, the figure, the sha256 of the fetched HTML and of any data file, merged PRs, and short notes on how the site figure relates to the repo count. Set `checked_at_utc`, -`local_date`, `pinned_at_utc` and `scope`. Derive the mechanical fields rather -than typing them: the figure through `check_cards.read_source()`, which applies `REGIONS`, merged PRs from +`local_date`, `pinned_at_utc` (ISO, with its offset) and `scope`. Record +`figure_at_pin` for every source with a committed copy: read the copy at the pin +with `check_cards.figure()`, the nightly's own parser with `REGIONS` applied, +never from the template. The provenance tests require it to equal each card, so +a mistyped card fails on the PR, and the nightly reports WRONG from it (#231). +ProteinTraitsMech's data file is built in CI and has none. Derive the other +mechanical fields rather than typing them: the live figure through +`check_cards.read_source()`, merged PRs from `mech_stats.json`, SHAs and commit dates from the pins, and assert that the pins equal the stats' `source_revision` before writing. Hash the served page as committed at the pin too (`git show :pages/index.html`, or `docs/`), record @@ -267,10 +273,9 @@ snapshot at the pins, record the live figure as `site_figure_at_check` in that Mech's `site_audit.json` entry, and say in the PR which cards the check reports as grown. That stays a warning for `GRACE_DAYS` (14) after `pinned_at_utc` while the site is at most half as large again as the card; past either limit the -check reports STALE and fails, and the page is due a full refresh. The check -also reads each `*_sha256_at_pin` in the audit, so record them for every source -the check reads: a card that differs from a source still byte-identical to its -pin fails as WRONG. +check reports STALE and fails, and the page is due a full refresh. A card that +differs from its audit's `figure_at_pin` fails as WRONG however far the site has +moved (step 7). Emoji headings render with a leading hyphen in their id on GitHub Pages. Verify anchors against the deployed HTML, not a local kramdown. diff --git a/_fleet/README.md b/_fleet/README.md index baa1f4b..b5e700b 100644 --- a/_fleet/README.md +++ b/_fleet/README.md @@ -43,18 +43,23 @@ elsewhere (CultureMech's generated README block). The page is a snapshot at a refresh's pins, so a site ahead of its card only warns ("grew") for `GRACE_DAYS` (14) after the pins in `site_audit.json`, and only while the site is at most `MAX_LEAD` (50%) ahead. Past either limit it fails as STALE. It also -fails when a source is byte-identical to its copy at the pin but states a -different figure (the card was never right), when a site is behind its card, -when a source answers a 4xx other than a throttle or no longer states a figure -the parser can read, when a card lacks exactly one headline figure or has no -`SOURCES` entry, and when more than half the sources could not be fetched. One -site's outage or throttle only warns (#148, #115, #176, #217-#220). It runs on -the workflow's nightly schedule, not on pull requests, so a Mech shipping -records overnight does not block an unrelated change. A nightly red means one -of three things: the card figures in `mechs_template.md` and the `MECHS` block -in `fleet_fragment.html` need refreshing together (a full refresh, since the -page is a snapshot), a card figure is wrong, or a `SOURCES` entry needs -repointing. A new card needs a `SOURCES` entry; a test enforces that. The cards +fails when a card differs from `figure_at_pin`, the figure `site_audit.json` +records its source stating at the pin (the card was never right), when a site is +behind its card, when a source answers a 4xx other than a throttle or no longer +states a figure the parser can read, when a card lacks exactly one headline +figure or has no `SOURCES` entry, when the audit's pin time is unreadable, and +when more than half the sources could not be fetched. One site's outage or +throttle only warns (#148, #115, #176, #217-#220, #231, #232). It runs on the +workflow's nightly schedule, not on pull requests, so a Mech shipping records +overnight does not block an unrelated change. The run's closing line names the +remedy for each failing verdict (#235): + +- STALE or SHRANK: the card figures in `mechs_template.md` and the `MECHS` block + in `fleet_fragment.html` need a full refresh, since the page is a snapshot. +- WRONG: a card figure was mistyped; the unit tests also catch this on the PR. +- GONE or CHANGED: a `SOURCES` entry needs repointing. +- MARKUP, UNCARDED or AUDIT: fix the card markup, `SOURCES` or `site_audit.json`. +- UNCHECKED: most sites could not be reached; rerun before changing anything. A new card needs a `SOURCES` entry; a test enforces that. The cards are read by `scripts/fleet/card_markup.py`, the one parser the assembler, this check and the tests share, and the assembler refuses a card without exactly one headline figure (#114, #218). diff --git a/_fleet/data/site_audit.json b/_fleet/data/site_audit.json index 0ac5081..9488317 100644 --- a/_fleet/data/site_audit.json +++ b/_fleet/data/site_audit.json @@ -10,6 +10,7 @@ "commit_date": "2026-09-25T01:30:31Z", "readme_url": "https://github.com/CultureBotAI/AntibioticMech/blob/66d68c2a8d99230ab729267e6cd8ae1051b85225/README.md", "card_records": 2939, + "figure_at_pin": 2939, "merged_prs": 422, "site": "https://culturebotai.github.io/AntibioticMech/pages/index.html", "site_html_sha256": "426d439768e46f5e3c65b0241762359d06dffb3b217a3bba01e2552b65991ae9", @@ -22,6 +23,7 @@ "commit_date": "2026-09-25T01:58:24Z", "readme_url": "https://github.com/CultureBotAI/CellStructureMech/blob/42cc23b90b6c48c49efd94b7ee6efb808dded7e6/README.md", "card_records": 542, + "figure_at_pin": 542, "merged_prs": 611, "site": "https://culturebotai.github.io/CellStructureMech/pages/index.html", "site_html_sha256": "aa3a2b8d51d6cd1bbac33c472b0b40279538779c514f384437f818b3c665d679", @@ -35,6 +37,7 @@ "commit_date": "2026-09-24T14:00:15Z", "readme_url": "https://github.com/CultureBotAI/CommunityMech/blob/8505a56d644b02fe87f776be9d667db4cf3f5c5d/README.md", "card_records": 422, + "figure_at_pin": 422, "merged_prs": 578, "site": "https://culturebotai.github.io/CommunityMech/", "site_html_sha256": "5c7aba370a7e7bd50590eb16005c4e3bdd31b1f17d933b8106de7506d67d07dd", @@ -54,6 +57,7 @@ "commit_date": "2026-09-22T06:24:22Z", "readme_url": "https://github.com/CultureBotAI/CultureMech/blob/faaf033b8c5386aaf2cb6f28fc527678d00cf83b/README.md", "card_records": 6288, + "figure_at_pin": 6288, "merged_prs": 253, "site": "https://raw.githubusercontent.com/CultureBotAI/CultureMech/main/README.md", "site_html_sha256": "9bc06d232a67fa55cdef539fec2f52fa2644f7cf50e2820e223cc26106ee6f51", @@ -66,6 +70,7 @@ "commit_date": "2026-09-25T01:40:32Z", "readme_url": "https://github.com/CultureBotAI/HabitatMech/blob/b16e3099478a7ea55b82142836e9a118ae062cf6/README.md", "card_records": 3206, + "figure_at_pin": 3206, "merged_prs": 417, "site": "https://culturebotai.github.io/HabitatMech/pages/index.html", "site_html_sha256": "53564300e6d0aaf2a5b1144fceab088024586941f197113c57c42c17eefbc8e9", @@ -78,6 +83,7 @@ "commit_date": "2026-09-24T08:38:41Z", "readme_url": "https://github.com/CultureBotAI/MediaIngredientMech/blob/dfce1c9342ca7aec41b50d0f6db8adb950bb72fe/README.md", "card_records": 2953, + "figure_at_pin": 2953, "merged_prs": 379, "site": "https://culturebotai.github.io/MediaIngredientMech/", "site_html_sha256": "e98a1ac81f8faafad3aae291782b32e22e78e8f20cd822effc4cba4c22ce7b09", @@ -92,6 +98,7 @@ "commit_date": "2026-09-24T07:48:18Z", "readme_url": "https://github.com/CultureBotAI/NaturalProductMech/blob/36662aa8fcdcac9d3ddf4acba0f430e39bc2da2e/README.md", "card_records": 3115, + "figure_at_pin": 3115, "merged_prs": 84, "site": "https://culturebotai.github.io/NaturalProductMech/pages/index.html", "site_html_sha256": "28d335df6bfeb065940fad9e5ae783d6cd3ed7eacabd8c46c94ee4a9aecdbabd", @@ -117,6 +124,7 @@ "commit_date": "2026-09-21T08:37:02Z", "readme_url": "https://github.com/CultureBotAI/TaxonMech/blob/972fbd7b85a052c6b95bf1437f672168bd6d8970/README.md", "card_records": 625960, + "figure_at_pin": 625960, "merged_prs": 18, "site": "https://culturebotai.github.io/TaxonMech/pages/index.html", "site_html_sha256": "53aab681e40bafb9326b320858660a4273c7df8b9a19ad561d4730b5b32c6a89", @@ -129,6 +137,7 @@ "commit_date": "2026-09-22T15:30:43Z", "readme_url": "https://github.com/CultureBotAI/TraitMech/blob/a21c5aa34a876c43ea1e8053a58bc63d53be30a1/README.md", "card_records": 763, + "figure_at_pin": 763, "merged_prs": 656, "site": "https://culturebotai.github.io/TraitMech/pages/index.html", "site_html_sha256": "21fd928ae51ddaca7a9f1e5cafce81a37b055c97ab8a83442ba19021acc2daeb", diff --git a/scripts/fleet/check_cards.py b/scripts/fleet/check_cards.py index ff77ce3..b00e9b3 100644 --- a/scripts/fleet/check_cards.py +++ b/scripts/fleet/check_cards.py @@ -19,11 +19,12 @@ at most GRACE_DAYS old, and the site is at most MAX_LEAD ahead. A warning: the page is a snapshot at the refresh's pins, and fast Mechs publish within hours of them. - STALE the site is ahead and the refresh is older than that, or the card - understates the site by more than MAX_LEAD. Refresh the page. - WRONG the source is byte-identical to its copy at the pin, which the - audit recorded, yet states a different figure. The site has not - moved, so the card was never right (#217). + STALE the site is ahead and the refresh is older than that, or the site + leads the card by more than MAX_LEAD of the card (the card then + understates it by over a third). Refresh the page. + WRONG the card differs from figure_at_pin, the figure the audit read from + the source's own committed copy at the pin with figure() below. The + card was never right, however the site has moved since (#217, #231). SHRANK the site states fewer than the card. Records are not normally withdrawn in bulk, so either the card is wrong or the site regressed. GONE the source answered a 4xx other than a throttle. The page the card @@ -32,19 +33,21 @@ the wording moved, and the card is no longer being checked at all. MARKUP a card in the template does not carry exactly one headline figure. UNCARDED a card with no SOURCES entry, or an entry with no card. + AUDIT site_audit.json's pinned_at_utc cannot be read as a time. unread the fetch did not arrive: DNS, timeout, a dropped connection, a 5xx, or a 408, 425 or 429 throttle. A warning, because the network is not the site's fault, unless more than half the sources are unread, when the run has verified too little to call itself a pass and fails as UNCHECKED. -The pins, the pin time and the hash of each source at its pin come from +The pin time and each source's figure at the pin come from _fleet/data/site_audit.json, which the refresh writes (update-xmech-page, step 7). +ProteinTraitsMech's data file is built in CI rather than committed, so it has no +copy at the pin and no figure_at_pin; its card is checked only against the site. """ from __future__ import annotations import datetime -import hashlib import http.client import json import re @@ -166,12 +169,26 @@ def published(kind: str, body: str, selector: str) -> int | None: return int(hit.group(1).replace(",", "")) if hit else None -def read_source(mech: str, kind: str, path: str, selector: str, fetcher=None) -> tuple[str, int | str, str | None]: - """The figure a source publishes, or why there is none, and the source's hash. +def figure(mech: str, kind: str, body: str, selector: str) -> int | None: + """The figure a source's body states, read the way the nightly reads it. - Returns ("value", N, sha256 of the body), or a status, the reason and None: - "GONE" for a 4xx other than a throttle, "unread" for a fetch that did not - arrive, "CHANGED" for a body with no figure in it. + Applies the Mech's REGIONS first, so the refresh can read the committed copy + at the pin with exactly the rules used on the live site (#221). None when no + figure can be read; a body of the wrong type raises, as published() does. + """ + if mech in REGIONS: + body = region(body, REGIONS[mech]) + if body is None: + return None + return published(kind, body, selector) + + +def read_source(mech: str, kind: str, path: str, selector: str, fetcher=None) -> tuple[str, int | str]: + """The figure a source publishes, or why there is none. + + Returns ("value", N), or a status and the reason: "GONE" for a 4xx other + than a throttle, "unread" for a fetch that did not arrive, "CHANGED" for a + body with no figure in it. """ fetcher = fetcher or fetch try: @@ -180,47 +197,39 @@ def read_source(mech: str, kind: str, path: str, selector: str, fetcher=None) -> # HTTPError is a URLError, so it is caught first. A 4xx is the source # telling us it is not there, unless it is asking us to come back later. gone = 400 <= error.code < 500 and error.code not in THROTTLES - return ("GONE" if gone else "unread"), f"fetch failed: {error}", None + return ("GONE" if gone else "unread"), f"fetch failed: {error}" except (urllib.error.URLError, http.client.HTTPException, TimeoutError, OSError) as error: # HTTPException covers a body cut short (IncompleteRead) and a garbled # status line, which used to end the run with a traceback (#220). - return "unread", f"fetch failed: {error}", None - digest = hashlib.sha256(body.encode("utf-8")).hexdigest() - if mech in REGIONS: - body = region(body, REGIONS[mech]) - if body is None: - return "CHANGED", "the generated block that states the figure is gone", digest + return "unread", f"fetch failed: {error}" try: - value = published(kind, body, selector) + value = figure(mech, kind, body, selector) except (ValueError, TypeError, AttributeError) as error: # json.JSONDecodeError is a ValueError. The other two are what a # body of an unexpected type raises when it is walked. - return "CHANGED", f"unparseable: {error}", digest + return "CHANGED", f"unparseable: {error}" if value is None: - return "CHANGED", f"{kind} shape changed; no {selector!r} found", digest - return "value", value, digest + where = "the generated block that states it" if mech in REGIONS else f"no {selector!r}" + return "CHANGED", f"{kind} shape changed; {where} found" + return "value", value -def pinned_hash(entry: dict, url: str) -> str | None: - """The audit's hash of this source at the pin, when the audit has one. +def pin_time(audit: dict) -> datetime.datetime: + """The audit's pin time as an aware datetime; a time with no offset is UTC. - ProteinTraitsMech's data file is built in CI rather than committed, so there - is no copy at the pin to hash, and its card cannot be shown WRONG this way. + Raises ValueError when the field is missing or is not an ISO time (#232). """ - if entry.get("data_url") == url: - return entry.get("data_sha256_at_pin") - if entry.get("site") == url: - return entry.get("site_html_sha256_at_pin") - return None + value = audit.get("pinned_at_utc") + if not isinstance(value, str): + raise ValueError(f"pinned_at_utc is {value!r}") + moment = datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) + return moment if moment.tzinfo else moment.replace(tzinfo=datetime.timezone.utc) -def classify(card: int, site: int, digest: str | None, pinned: str | None, - age: datetime.timedelta | None) -> str: +def classify(card: int, site: int, age: datetime.timedelta | None) -> str: """How a published figure relates to the card that states it.""" if site == card: return "ok" - if digest and pinned and digest == pinned: - return "WRONG" if site < card: return "SHRANK" if age is None or age > datetime.timedelta(days=GRACE_DAYS): @@ -236,20 +245,25 @@ def span(age: datetime.timedelta) -> str: return f"{hours} hour{'s' if hours != 1 else ''}" -FAILURES = ("STALE", "WRONG", "SHRANK", "GONE", "CHANGED", "MARKUP", "UNCARDED", "UNCHECKED") +FAILURES = ("STALE", "WRONG", "SHRANK", "GONE", "CHANGED", "MARKUP", "UNCARDED", "AUDIT", "UNCHECKED") def check(template: str, fetcher=None, audit: dict | None = None, now: datetime.datetime | None = None) -> list[tuple[str, str, str]]: - """One (status, mech, detail) row per card problem and per source, plus the run-level verdict.""" + """One (status, mech, detail) row per card problem and per source, plus the run-level verdicts.""" now = now or datetime.datetime.now(datetime.timezone.utc) audit = audit or {} entries = {row["repo"].lower(): row for row in audit.get("repositories", [])} - pinned_at = audit.get("pinned_at_utc") - age = now - datetime.datetime.fromisoformat(pinned_at) if pinned_at else None + rows = [("MARKUP", mech, why) for mech, why in markup_problems(template)] + age = None + if audit: + try: + age = now - pin_time(audit) + except ValueError as error: + # Growth then counts as STALE, which fails anyway; say why. + rows.append(("AUDIT", "-", f"site_audit.json: {error}")) stated = card_figures(template) names = set(card_names(template)) - rows = [("MARKUP", mech, why) for mech, why in markup_problems(template)] for mech in sorted(names - set(SOURCES)): rows.append(("UNCARDED", mech, "card with no entry in SOURCES")) for mech, (kind, path, selector) in sorted(SOURCES.items()): @@ -258,17 +272,18 @@ def check(template: str, fetcher=None, audit: dict | None = None, continue if mech not in stated: continue # its card's markup is already reported above - status, result, digest = read_source(mech, kind, path, selector, fetcher) + card = stated[mech] + at_pin = entries.get(mech.lower(), {}).get("figure_at_pin") + if isinstance(at_pin, int) and not isinstance(at_pin, bool) and at_pin != card: + rows.append(("WRONG", mech, f"card {card:,}, but the source stated {at_pin:,} at the pin")) + continue + status, result = read_source(mech, kind, path, selector, fetcher) if status != "value": rows.append((status, mech, result)) continue - card = stated[mech] - pinned = pinned_hash(entries.get(mech.lower(), {}), source_url(path)) - verdict = classify(card, result, digest, pinned, age) + verdict = classify(card, result, age) detail = f"{card:>9,}" if verdict == "ok" else f"card {card:,}, site {result:,}" - if verdict == "WRONG": - detail += ", and the source is unchanged since the pin" - elif verdict in ("grew", "STALE") and age is not None: + if verdict in ("grew", "STALE") and age is not None: detail += f" (+{result - card:,} in the {span(age)} since the pins)" rows.append((verdict, mech, detail)) unread = sum(1 for status, _, _ in rows if status == "unread") @@ -292,8 +307,8 @@ def main() -> int: print("Failing. STALE, WRONG or SHRANK: refresh the card figures in _fleet/mechs_template.md " "and the MECHS block in _fleet/fleet_fragment.html (update-xmech-page), then rerun " "assemble_page.py. GONE or CHANGED: repoint that Mech's SOURCES entry. MARKUP or " - "UNCARDED: fix the card or its SOURCES entry. UNCHECKED: the run could not reach " - "most sites.") + "UNCARDED: fix the card or its SOURCES entry. AUDIT: fix site_audit.json. " + "UNCHECKED: the run could not reach most sites; rerun before changing anything.") return 1 if any(status in ("grew", "unread") for status, _, _ in rows): # A site a little ahead of a recently pinned card, or one that could not diff --git a/tests/test_fleet_page.py b/tests/test_fleet_page.py index b27f39b..478411a 100644 --- a/tests/test_fleet_page.py +++ b/tests/test_fleet_page.py @@ -46,6 +46,14 @@ def test_a_card_without_a_record_count_cannot_be_left_out_of_the_total(self): with self.assertRaisesRegex(ValueError, "record count"): self.render() + def test_a_figure_outside_every_card_cannot_join_the_total(self): + # #218, #234: a stray figure used to be ignored, and a second one in a + # card used to replace its headline in the total. + stray = self.template.replace("", '
7
', 1) + self.template = stray + with self.assertRaisesRegex(ValueError, "outside any card"): + self.render() + def test_card_stats_must_cover_every_fleet_member(self): self.stats["mechs"] = [m for m in self.stats["mechs"] if m["mech"] != "TaxonMech"] with self.assertRaisesRegex(ValueError, "Mech stats"): @@ -465,8 +473,8 @@ def setUp(self): self.site = {m: 1000 for m in self.NAMES} self.pinned_at = datetime.datetime(2026, 9, 25, 2, 0, tzinfo=datetime.timezone.utc) self.now = self.pinned_at + datetime.timedelta(hours=18) - # By default each source has moved since its pin, so a lead reads as growth. - self.pinned = {m: "0" * 64 for m in self.NAMES} + # The figure each source stated at the pin, as the audit records it. + self.at_pin = {m: 1000 for m in self.NAMES} def body(self, mech, value): return f"
{value:,}{mech[0].lower()} records
" @@ -484,9 +492,7 @@ def template(self): def audit(self): return {"pinned_at_utc": self.pinned_at.isoformat(), "repositories": [ - {"repo": m, "site": self.check_cards.source_url(self.check_cards.SOURCES[m][1]), - "site_html_sha256_at_pin": self.pinned[m]} - for m in self.check_cards.SOURCES]} + {"repo": m, "figure_at_pin": self.at_pin[m]} for m in self.check_cards.SOURCES if m in self.at_pin]} def rows(self, template=None): return self.check_cards.check(template or self.template(), self.fetch, self.audit(), self.now) @@ -526,14 +532,34 @@ def test_growth_without_an_audit_fails(self): rows = self.check_cards.check(self.template(), self.fetch, None, self.now) self.assertIn(("STALE", "AMech"), [(s, m) for s, m, _ in rows]) - def test_a_mistyped_card_fails_when_the_site_has_not_moved(self): - # #217: 3,026 typed for 3,206 used to pass as "grew". - import hashlib + def test_a_mistyped_card_fails_however_the_site_has_moved(self): + # #217, #231: 3,026 typed for 3,206 used to pass as "grew", and a hash + # comparison caught it only while the source was unchanged since the pin. self.cards["AMech"] = 3026 - self.site["AMech"] = 3206 - self.pinned["AMech"] = hashlib.sha256(self.body("AMech", 3206).encode()).hexdigest() - self.assertEqual(self.statuses()["AMech"], "WRONG") - self.assertEqual(self.failed(), ["WRONG"]) + self.at_pin["AMech"] = 3206 + for live in (3206, 3300): + self.site["AMech"] = live + self.assertEqual(self.statuses()["AMech"], "WRONG", live) + self.assertEqual(self.failed(), ["WRONG"]) + + def test_a_source_with_no_figure_at_the_pin_is_checked_against_the_site_only(self): + # ProteinTraitsMech's data file is built in CI, so the audit has no copy. + del self.at_pin["AMech"] + self.site["AMech"] = 1100 + self.assertEqual(self.statuses()["AMech"], "grew") + + def test_a_pin_time_without_an_offset_is_utc_and_an_unreadable_one_fails(self): + # #232: both used to end the run with a traceback before any row printed. + audit = self.audit() + audit["pinned_at_utc"] = self.pinned_at.replace(tzinfo=None).isoformat() + self.site["AMech"] = 1100 + rows = self.check_cards.check(self.template(), self.fetch, audit, self.now) + self.assertIn(("grew", "AMech"), [(s, m) for s, m, _ in rows]) + audit["pinned_at_utc"] = "25 September 2026" + rows = self.check_cards.check(self.template(), self.fetch, audit, self.now) + statuses = [s for s, _, _ in rows] + self.assertIn("AUDIT", statuses) + self.assertIn(("STALE", "AMech"), [(s, m) for s, m, _ in rows]) def test_a_site_behind_its_card_fails(self): self.site["AMech"] = 999 @@ -551,6 +577,9 @@ def test_a_missing_page_fails_but_an_outage_or_throttle_only_warns(self): for code in (408, 425, 503): self.site["BMech"] = urllib.error.HTTPError("u", code, "x", {}, None) self.assertEqual(self.statuses()["BMech"], "unread", code) + for code in (403, 410): # #234: any other 4xx + self.site["AMech"] = urllib.error.HTTPError("u", code, "x", {}, None) + self.assertEqual(self.statuses()["AMech"], "GONE", code) def test_a_body_cut_short_is_unread_not_a_traceback(self): # #220 @@ -578,7 +607,7 @@ def test_more_than_half_unread_fails_and_half_does_not(self): self.check_cards.SOURCES.update({m: ("html", f"{m}/pages/index.html", "m records") for m in names}) self.cards = {m: 1000 for m in names} self.site = {m: 1000 for m in names} - self.pinned = {m: "0" * 64 for m in names} + self.at_pin = {m: 1000 for m in names} for m in names[:5]: self.site[m] = urllib.error.URLError("no route") self.assertEqual(self.failed(), []) @@ -634,6 +663,17 @@ def test_main_exits_by_the_same_rule(self): self.assertEqual(self.check_cards.main(), 0) self.site["AMech"] = 900 self.assertEqual(self.check_cards.main(), 1) + # #234: every failing verdict fails the run, not just SHRANK. + self.site["AMech"] = 1600 # STALE + self.assertEqual(self.check_cards.main(), 1) + self.site["AMech"] = 1000 + self.at_pin["AMech"] = 1001 # WRONG + audit.write_text(json.dumps(self.audit())) + self.assertEqual(self.check_cards.main(), 1) + self.at_pin["AMech"] = 1000 + audit.write_text(json.dumps(self.audit())) + self.site["AMech"] = self.site["CMech"] = TimeoutError("t") # UNCHECKED + self.assertEqual(self.check_cards.main(), 1) class RefreshProvenanceTests(unittest.TestCase): @@ -679,6 +719,25 @@ def test_the_audit_pins_the_revisions_the_stats_counted(self): self.assertIn(entry["repo"], self.audit, "Mech missing from site_audit.json") self.assertEqual(self.audit[entry["repo"]]["sha"], entry["source_revision"]) + def test_every_card_equals_the_figure_its_source_stated_at_the_pin(self): + # #231: the audit builder read card_records from the template, so the + # audit could only repeat a mistyped card. figure_at_pin is read from the + # source's committed copy at the pin with check_cards.figure(), and the + # nightly's WRONG verdict depends on it being there. + import check_cards + audit = json.loads((ROOT / "_fleet/data/site_audit.json").read_text()) + check_cards.pin_time(audit) # #232: must parse + entries = {r["repo"].lower(): r for r in audit["repositories"]} + cards = card_figures((ROOT / "_fleet/mechs_template.md").read_text()) + for mech in check_cards.SOURCES: + with self.subTest(mech=mech): + entry = entries[mech.lower()] + if mech == "ProteinTraitsMech": + # Built in CI, so there is no committed copy at the pin. + self.assertNotIn("figure_at_pin", entry) + continue + self.assertEqual(entry.get("figure_at_pin"), cards[mech]) + def test_the_audit_records_this_refresh_and_nothing_else(self): # The audit's other fields had no gate at all (#128): its merged PRs, # the card figure it read, the CLAW pin, and which repositories it lists. From df4878a55c005e0b89954daa32db604a755ce4a1 Mon Sep 17 00:00:00 2001 From: "marcin p. joachimiak" <4625870+realmarcin@users.noreply.github.com> Date: Fri, 25 Sep 2026 14:05:34 -0700 Subject: [PATCH 4/8] Run the nightly card check after a failed test, and name WRONG's own remedy A mistyped card fails the provenance tests first, and the card check used to be skipped with them, so its WRONG line and every other card's verdict never printed; the step now runs unless the job was cancelled (#239). WRONG's remedy is to correct the card or the audit, not a full refresh (#240). The README's remedy list no longer swallows the next paragraph (#241). A pin time in the future is an AUDIT failure, and the provenance test bounds it by the newest pinned commit and the check (#242). The tests pin the 14-day grace, a missing pin time, AUDIT on its own, "no offset means UTC" under a non-UTC zone, and the 400/499/500 edges (#243). Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/fleet-page.yml | 5 +++- _fleet/README.md | 12 ++++++-- scripts/fleet/check_cards.py | 14 ++++++++-- tests/test_fleet_page.py | 47 ++++++++++++++++++++++++++++++-- 4 files changed, 68 insertions(+), 10 deletions(-) diff --git a/.github/workflows/fleet-page.yml b/.github/workflows/fleet-page.yml index ee98a95..972170c 100644 --- a/.github/workflows/fleet-page.yml +++ b/.github/workflows/fleet-page.yml @@ -33,5 +33,8 @@ jobs: # are hand-curated from ten sites that publish on their own cadence, so a # blocking check would make an unrelated docs fix unmergeable whenever a # Mech shipped records overnight. Nightly red is the right signal. - - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + # !cancelled() runs it even when an earlier step failed: a mistyped card + # fails the provenance tests first, and the card report, with its WRONG + # or AUDIT line and every other card's verdict, must still print (#239). + - if: ${{ !cancelled() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} run: python scripts/fleet/check_cards.py diff --git a/_fleet/README.md b/_fleet/README.md index b5e700b..8ebedf2 100644 --- a/_fleet/README.md +++ b/_fleet/README.md @@ -47,7 +47,8 @@ fails when a card differs from `figure_at_pin`, the figure `site_audit.json` records its source stating at the pin (the card was never right), when a site is behind its card, when a source answers a 4xx other than a throttle or no longer states a figure the parser can read, when a card lacks exactly one headline -figure or has no `SOURCES` entry, when the audit's pin time is unreadable, and +figure or has no `SOURCES` entry, when the audit's pin time is missing, +unreadable or in the future, and when more than half the sources could not be fetched. One site's outage or throttle only warns (#148, #115, #176, #217-#220, #231, #232). It runs on the workflow's nightly schedule, not on pull requests, so a Mech shipping records @@ -56,10 +57,15 @@ remedy for each failing verdict (#235): - STALE or SHRANK: the card figures in `mechs_template.md` and the `MECHS` block in `fleet_fragment.html` need a full refresh, since the page is a snapshot. -- WRONG: a card figure was mistyped; the unit tests also catch this on the PR. +- WRONG: a card, or the audit's `figure_at_pin`, was mistyped; correct it so both + agree with the pinned source, with no refresh. The unit tests catch this on the + PR and in the nightly, which still runs the card check after a failed test + step so its report prints (#239, #240). - GONE or CHANGED: a `SOURCES` entry needs repointing. - MARKUP, UNCARDED or AUDIT: fix the card markup, `SOURCES` or `site_audit.json`. -- UNCHECKED: most sites could not be reached; rerun before changing anything. A new card needs a `SOURCES` entry; a test enforces that. The cards +- UNCHECKED: most sites could not be reached; rerun before changing anything. + +A new card needs a `SOURCES` entry; a test enforces that. The cards are read by `scripts/fleet/card_markup.py`, the one parser the assembler, this check and the tests share, and the assembler refuses a card without exactly one headline figure (#114, #218). diff --git a/scripts/fleet/check_cards.py b/scripts/fleet/check_cards.py index b00e9b3..d8993df 100644 --- a/scripts/fleet/check_cards.py +++ b/scripts/fleet/check_cards.py @@ -33,7 +33,8 @@ the wording moved, and the card is no longer being checked at all. MARKUP a card in the template does not carry exactly one headline figure. UNCARDED a card with no SOURCES entry, or an entry with no card. - AUDIT site_audit.json's pinned_at_utc cannot be read as a time. + AUDIT site_audit.json's pinned_at_utc is missing, cannot be read as a time, + or is in the future. unread the fetch did not arrive: DNS, timeout, a dropped connection, a 5xx, or a 408, 425 or 429 throttle. A warning, because the network is not the site's fault, unless more than half the sources are unread, @@ -262,6 +263,12 @@ def check(template: str, fetcher=None, audit: dict | None = None, except ValueError as error: # Growth then counts as STALE, which fails anyway; say why. rows.append(("AUDIT", "-", f"site_audit.json: {error}")) + else: + if age < datetime.timedelta(0): + # A pin in the future would hold off the grace limit until the + # clock caught up with it (#242). + rows.append(("AUDIT", "-", f"site_audit.json: pinned_at_utc {audit['pinned_at_utc']} is in the future")) + age = None stated = card_figures(template) names = set(card_names(template)) for mech in sorted(names - set(SOURCES)): @@ -304,9 +311,10 @@ def main() -> int: tally[status] = tally.get(status, 0) + 1 print("\n" + ", ".join(f"{count} {status.lower()}" for status, count in sorted(tally.items())) + ".") if any(status in FAILURES for status, _, _ in rows): - print("Failing. STALE, WRONG or SHRANK: refresh the card figures in _fleet/mechs_template.md " + print("Failing. STALE or SHRANK: refresh the card figures in _fleet/mechs_template.md " "and the MECHS block in _fleet/fleet_fragment.html (update-xmech-page), then rerun " - "assemble_page.py. GONE or CHANGED: repoint that Mech's SOURCES entry. MARKUP or " + "assemble_page.py. WRONG: correct the card, or the audit's figure_at_pin if that is " + "what was mistyped, so both agree with the pinned source; no refresh. GONE or CHANGED: repoint that Mech's SOURCES entry. MARKUP or " "UNCARDED: fix the card or its SOURCES entry. AUDIT: fix site_audit.json. " "UNCHECKED: the run could not reach most sites; rerun before changing anything.") return 1 diff --git a/tests/test_fleet_page.py b/tests/test_fleet_page.py index 478411a..638939e 100644 --- a/tests/test_fleet_page.py +++ b/tests/test_fleet_page.py @@ -515,6 +515,7 @@ def test_a_site_ahead_of_a_recent_pin_only_warns(self): self.assertEqual(self.failed(), []) def test_growth_past_the_grace_period_fails(self): + self.assertEqual(self.check_cards.GRACE_DAYS, 14) # #243: the documented figure self.site["AMech"] = 1001 self.now = self.pinned_at + datetime.timedelta(days=self.check_cards.GRACE_DAYS, hours=1) self.assertEqual(self.failed(), ["STALE"]) @@ -561,6 +562,40 @@ def test_a_pin_time_without_an_offset_is_utc_and_an_unreadable_one_fails(self): self.assertIn("AUDIT", statuses) self.assertIn(("STALE", "AMech"), [(s, m) for s, m, _ in rows]) + def audit_failures(self, audit): + return sorted(s for s, _, _ in self.check_cards.check(self.template(), self.fetch, audit, self.now) + if s in self.check_cards.FAILURES) + + def test_a_missing_future_or_unreadable_pin_time_fails_on_its_own(self): + # #242, #243: with every card equal to its site, the audit alone fails. + audit = self.audit() + del audit["pinned_at_utc"] + self.assertEqual(self.audit_failures(audit), ["AUDIT"]) + audit["pinned_at_utc"] = "not a time" + self.assertEqual(self.audit_failures(audit), ["AUDIT"]) + audit["pinned_at_utc"] = (self.now + datetime.timedelta(days=365)).isoformat() + self.assertEqual(self.audit_failures(audit), ["AUDIT"]) + # And a future pin must not hold off the grace limit for a grown card. + self.site["AMech"] = 1001 + self.assertIn("STALE", self.audit_failures(audit)) + + def test_a_pin_time_without_an_offset_is_read_as_utc_not_local_time(self): + # #243: in a UTC runner, local time and UTC agree, so pin a zone that differs. + import time + saved = os.environ.get("TZ") + os.environ["TZ"] = "America/Los_Angeles" + time.tzset() + try: + moment = self.check_cards.pin_time({"pinned_at_utc": "2026-09-25T02:06:03"}) + finally: + if saved is None: + os.environ.pop("TZ", None) + else: + os.environ["TZ"] = saved + time.tzset() + self.assertEqual(moment, datetime.datetime(2026, 9, 25, 2, 6, 3, tzinfo=datetime.timezone.utc)) + self.assertEqual(moment.utcoffset(), datetime.timedelta(0)) + def test_a_site_behind_its_card_fails(self): self.site["AMech"] = 999 self.assertEqual(self.failed(), ["SHRANK"]) @@ -574,10 +609,10 @@ def test_a_missing_page_fails_but_an_outage_or_throttle_only_warns(self): statuses = self.statuses() self.assertEqual((statuses["AMech"], statuses["BMech"]), ("GONE", "unread")) self.assertEqual(self.failed(), ["GONE"]) - for code in (408, 425, 503): + for code in (408, 425, 500, 503): self.site["BMech"] = urllib.error.HTTPError("u", code, "x", {}, None) self.assertEqual(self.statuses()["BMech"], "unread", code) - for code in (403, 410): # #234: any other 4xx + for code in (400, 403, 410, 499): # #234, #243: any other 4xx self.site["AMech"] = urllib.error.HTTPError("u", code, "x", {}, None) self.assertEqual(self.statuses()["AMech"], "GONE", code) @@ -726,7 +761,13 @@ def test_every_card_equals_the_figure_its_source_stated_at_the_pin(self): # nightly's WRONG verdict depends on it being there. import check_cards audit = json.loads((ROOT / "_fleet/data/site_audit.json").read_text()) - check_cards.pin_time(audit) # #232: must parse + pinned = check_cards.pin_time(audit) # #232: must parse + # #242: and must fall between the newest pinned commit and the check. + newest = max(datetime.datetime.fromisoformat(r["commit_date"].replace("Z", "+00:00")) + for r in audit["repositories"]) + checked = datetime.datetime.fromisoformat(audit["checked_at_utc"]) + self.assertLessEqual(newest, pinned) + self.assertLessEqual(pinned, checked) entries = {r["repo"].lower(): r for r in audit["repositories"]} cards = card_figures((ROOT / "_fleet/mechs_template.md").read_text()) for mech in check_cards.SOURCES: From bf03ec826fc751fbebfbd06bd9c66203c4012c19 Mon Sep 17 00:00:00 2001 From: "marcin p. joachimiak" <4625870+realmarcin@users.noreply.github.com> Date: Fri, 25 Sep 2026 14:24:14 -0700 Subject: [PATCH 5/8] Say a missing block is missing, name every copy WRONG must fix, survive a bad audit CultureMech's CHANGED detail said its missing generated block was "found" (#247). WRONG's remedy names every copy of a card figure and says to rerun the assembler, and a new test holds each MECHS records: to its card (#248). A missing or malformed site_audit.json is an AUDIT row instead of a traceback, so the rest of the report prints (#250). The tests assert the remedy text, AUDIT on its own, a pin a minute ahead and an overstated WRONG card (#249). Co-Authored-By: Claude Opus 5.5 (1M context) --- _fleet/README.md | 11 +++++--- scripts/fleet/check_cards.py | 55 +++++++++++++++++++++++++++--------- tests/test_fleet_page.py | 53 +++++++++++++++++++++++++++++++--- 3 files changed, 98 insertions(+), 21 deletions(-) diff --git a/_fleet/README.md b/_fleet/README.md index 8ebedf2..297d3dd 100644 --- a/_fleet/README.md +++ b/_fleet/README.md @@ -47,8 +47,8 @@ fails when a card differs from `figure_at_pin`, the figure `site_audit.json` records its source stating at the pin (the card was never right), when a site is behind its card, when a source answers a 4xx other than a throttle or no longer states a figure the parser can read, when a card lacks exactly one headline -figure or has no `SOURCES` entry, when the audit's pin time is missing, -unreadable or in the future, and +figure or has no `SOURCES` entry, when the audit is missing or malformed or +its pin time is missing, unreadable or in the future, and when more than half the sources could not be fetched. One site's outage or throttle only warns (#148, #115, #176, #217-#220, #231, #232). It runs on the workflow's nightly schedule, not on pull requests, so a Mech shipping records @@ -57,8 +57,11 @@ remedy for each failing verdict (#235): - STALE or SHRANK: the card figures in `mechs_template.md` and the `MECHS` block in `fleet_fragment.html` need a full refresh, since the page is a snapshot. -- WRONG: a card, or the audit's `figure_at_pin`, was mistyped; correct it so both - agree with the pinned source, with no refresh. The unit tests catch this on the +- WRONG: a card, or the audit's `figure_at_pin`, was mistyped. Correct the card + and every copy of its figure (the MECHS `records:` in `fleet_fragment.html`, + `card_records` in `site_audit.json`, and the pages step 6 of the update skill + lists), or `figure_at_pin` if that is what was wrong, then rerun + `assemble_page.py`; no re-pin (#248). The unit tests catch this on the PR and in the nightly, which still runs the card check after a failed test step so its report prints (#239, #240). - GONE or CHANGED: a `SOURCES` entry needs repointing. diff --git a/scripts/fleet/check_cards.py b/scripts/fleet/check_cards.py index d8993df..7d2a5f6 100644 --- a/scripts/fleet/check_cards.py +++ b/scripts/fleet/check_cards.py @@ -33,8 +33,8 @@ the wording moved, and the card is no longer being checked at all. MARKUP a card in the template does not carry exactly one headline figure. UNCARDED a card with no SOURCES entry, or an entry with no card. - AUDIT site_audit.json's pinned_at_utc is missing, cannot be read as a time, - or is in the future. + AUDIT site_audit.json is missing or malformed, or its pinned_at_utc is + missing, cannot be read as a time, or is in the future. unread the fetch did not arrive: DNS, timeout, a dropped connection, a 5xx, or a 408, 425 or 429 throttle. A warning, because the network is not the site's fault, unless more than half the sources are unread, @@ -210,8 +210,10 @@ def read_source(mech: str, kind: str, path: str, selector: str, fetcher=None) -> # body of an unexpected type raises when it is walked. return "CHANGED", f"unparseable: {error}" if value is None: - where = "the generated block that states it" if mech in REGIONS else f"no {selector!r}" - return "CHANGED", f"{kind} shape changed; {where} found" + # #247: this used to say the missing block was "found". + where = ("the generated block is missing or states no figure" if mech in REGIONS + else f"no {selector!r} found") + return "CHANGED", f"{kind} shape changed; {where}" return "value", value @@ -249,13 +251,32 @@ def span(age: datetime.timedelta) -> str: FAILURES = ("STALE", "WRONG", "SHRANK", "GONE", "CHANGED", "MARKUP", "UNCARDED", "AUDIT", "UNCHECKED") -def check(template: str, fetcher=None, audit: dict | None = None, - now: datetime.datetime | None = None) -> list[tuple[str, str, str]]: - """One (status, mech, detail) row per card problem and per source, plus the run-level verdicts.""" +def audit_entries(audit) -> tuple[dict, str | None]: + """The audit's repositories keyed by lower-cased name, or why they cannot be read.""" + if not isinstance(audit, dict): + return {}, f"site_audit.json holds a {type(audit).__name__}, not an object" + repositories = audit.get("repositories", []) + if not isinstance(repositories, list) or not all( + isinstance(row, dict) and isinstance(row.get("repo"), str) for row in repositories): + return {}, "site_audit.json: repositories must be a list of objects, each with a repo name" + return {row["repo"].lower(): row for row in repositories}, None + + +def check(template: str, fetcher=None, audit=None, now: datetime.datetime | None = None, + audit_error: str | None = None) -> list[tuple[str, str, str]]: + """One (status, mech, detail) row per card problem and per source, plus the run-level verdicts. + + A malformed audit is an AUDIT row, not an exception, so the other rows still + print (#250); audit_error carries a problem found while reading the file. + """ now = now or datetime.datetime.now(datetime.timezone.utc) - audit = audit or {} - entries = {row["repo"].lower(): row for row in audit.get("repositories", [])} rows = [("MARKUP", mech, why) for mech, why in markup_problems(template)] + if audit_error: + rows.append(("AUDIT", "-", audit_error)) + entries, problem = audit_entries(audit) if audit is not None else ({}, None) + if problem: + rows.append(("AUDIT", "-", problem)) + audit = None age = None if audit: try: @@ -302,8 +323,14 @@ def check(template: str, fetcher=None, audit: dict | None = None, def main() -> int: - audit = json.loads(AUDIT.read_text()) if AUDIT.exists() else None - rows = check(TEMPLATE.read_text(), audit=audit) + audit, audit_error = None, None + try: + audit = json.loads(AUDIT.read_text()) + except FileNotFoundError: + audit_error = "site_audit.json is missing" + except (OSError, ValueError) as error: + audit_error = f"site_audit.json cannot be read: {error}" + rows = check(TEMPLATE.read_text(), audit=audit, audit_error=audit_error) for status, mech, detail in rows: print(f" {status:<9} {mech:<20} {detail}") tally: dict[str, int] = {} @@ -313,8 +340,10 @@ def main() -> int: if any(status in FAILURES for status, _, _ in rows): print("Failing. STALE or SHRANK: refresh the card figures in _fleet/mechs_template.md " "and the MECHS block in _fleet/fleet_fragment.html (update-xmech-page), then rerun " - "assemble_page.py. WRONG: correct the card, or the audit's figure_at_pin if that is " - "what was mistyped, so both agree with the pinned source; no refresh. GONE or CHANGED: repoint that Mech's SOURCES entry. MARKUP or " + "assemble_page.py. WRONG: correct the card and every copy of its figure (the MECHS " + "records in _fleet/fleet_fragment.html, card_records in site_audit.json and the pages " + "update-xmech-page step 6 lists), or the audit's figure_at_pin if that is what was " + "mistyped, then rerun assemble_page.py; no re-pin. GONE or CHANGED: repoint that Mech's SOURCES entry. MARKUP or " "UNCARDED: fix the card or its SOURCES entry. AUDIT: fix site_audit.json. " "UNCHECKED: the run could not reach most sites; rerun before changing anything.") return 1 diff --git a/tests/test_fleet_page.py b/tests/test_fleet_page.py index 638939e..e7e29b4 100644 --- a/tests/test_fleet_page.py +++ b/tests/test_fleet_page.py @@ -542,6 +542,10 @@ def test_a_mistyped_card_fails_however_the_site_has_moved(self): self.site["AMech"] = live self.assertEqual(self.statuses()["AMech"], "WRONG", live) self.assertEqual(self.failed(), ["WRONG"]) + # #249: overstated too, with the site ahead of both. + self.cards["AMech"] = 3260 + self.site["AMech"] = 3300 + self.assertEqual(self.statuses()["AMech"], "WRONG") def test_a_source_with_no_figure_at_the_pin_is_checked_against_the_site_only(self): # ProteinTraitsMech's data file is built in CI, so the audit has no copy. @@ -573,12 +577,24 @@ def test_a_missing_future_or_unreadable_pin_time_fails_on_its_own(self): self.assertEqual(self.audit_failures(audit), ["AUDIT"]) audit["pinned_at_utc"] = "not a time" self.assertEqual(self.audit_failures(audit), ["AUDIT"]) - audit["pinned_at_utc"] = (self.now + datetime.timedelta(days=365)).isoformat() - self.assertEqual(self.audit_failures(audit), ["AUDIT"]) + for ahead in (datetime.timedelta(minutes=1), datetime.timedelta(days=365)): # #249 + audit["pinned_at_utc"] = (self.now + ahead).isoformat() + self.assertEqual(self.audit_failures(audit), ["AUDIT"], ahead) # And a future pin must not hold off the grace limit for a grown card. self.site["AMech"] = 1001 self.assertIn("STALE", self.audit_failures(audit)) + def test_a_malformed_audit_is_an_audit_row_not_a_traceback(self): + # #250: the other rows must still print. + for audit in ([], {"pinned_at_utc": self.pinned_at.isoformat(), "repositories": [{"figure_at_pin": 1}]}, + {"pinned_at_utc": self.pinned_at.isoformat(), "repositories": "x"}): + rows = self.check_cards.check(self.template(), self.fetch, audit, self.now) + self.assertIn("AUDIT", [s for s, _, _ in rows], audit) + self.assertEqual(sorted(m for s, m, _ in rows if s == "ok"), list(self.NAMES), audit) + rows = self.check_cards.check(self.template(), self.fetch, None, self.now, + audit_error="site_audit.json is missing") + self.assertEqual(sorted(s for s, _, _ in rows if s in self.check_cards.FAILURES), ["AUDIT"]) + def test_a_pin_time_without_an_offset_is_read_as_utc_not_local_time(self): # #243: in a UTC runner, local time and UTC agree, so pin a zone that differs. import time @@ -678,8 +694,12 @@ def test_the_culturemech_figure_is_read_only_from_its_generated_block(self): readme = "Release 2 added 1,024 merged records.\n" + begin + "\n" + line + "\n" + end + "\n" read = self.check_cards.read_source self.assertEqual(read("CultureMech", kind, path, selector, lambda url: readme)[:2], ("value", 6288)) - for broken in ("The corpus has 6,288 merged records.", begin + "\n" + line + "\n"): - self.assertEqual(read("CultureMech", kind, path, selector, lambda url: broken)[0], "CHANGED") + for broken in ("The corpus has 6,288 merged records.", begin + "\n" + line + "\n", + begin + "\nno figure here\n" + end): + status, detail = read("CultureMech", kind, path, selector, lambda url: broken) + self.assertEqual(status, "CHANGED") + # #247: it used to say the missing block was "found". + self.assertIn("missing or states no figure", detail) def test_main_exits_by_the_same_rule(self): from unittest import mock @@ -709,6 +729,23 @@ def test_main_exits_by_the_same_rule(self): audit.write_text(json.dumps(self.audit())) self.site["AMech"] = self.site["CMech"] = TimeoutError("t") # UNCHECKED self.assertEqual(self.check_cards.main(), 1) + self.site = {m: 1000 for m in self.NAMES} + # #249: the remedy WRONG prints is a correction, not a refresh. + self.at_pin["AMech"] = 1001 + audit.write_text(json.dumps(self.audit())) + out = io.StringIO() + with contextlib.redirect_stdout(out): + self.assertEqual(self.check_cards.main(), 1) + remedy = out.getvalue().split("WRONG:", 1)[1].split(" GONE or CHANGED:", 1)[0] + self.assertIn("every copy", remedy) + self.assertIn("no re-pin", remedy) + self.assertNotIn("update-xmech-page)", remedy) + self.at_pin["AMech"] = 1000 + # AUDIT alone fails the run, and an unreadable file is AUDIT, not a traceback (#250). + audit.write_text("{not json") + self.assertEqual(self.check_cards.main(), 1) + audit.unlink() + self.assertEqual(self.check_cards.main(), 1) class RefreshProvenanceTests(unittest.TestCase): @@ -754,6 +791,14 @@ def test_the_audit_pins_the_revisions_the_stats_counted(self): self.assertIn(entry["repo"], self.audit, "Mech missing from site_audit.json") self.assertEqual(self.audit[entry["repo"]]["sha"], entry["source_revision"]) + def test_every_graph_panel_states_its_card_figure(self): + # #248: the MECHS block repeats each card's figure as records:, and a + # correction to one used to leave the other behind unnoticed. + fragment = (ROOT / "_fleet/fleet_fragment.html").read_text() + metadata = fragment.split("var MECHS = {", 1)[1].split("\n };", 1)[0] + panels = {m: int(n) for m, n in re.findall(r"^\s+([A-Za-z]+Mech):\s*\{.*?\brecords:\s*(\d+)", metadata, re.M)} + self.assertEqual(panels, card_figures((ROOT / "_fleet/mechs_template.md").read_text())) + def test_every_card_equals_the_figure_its_source_stated_at_the_pin(self): # #231: the audit builder read card_records from the template, so the # audit could only repeat a mistyped card. figure_at_pin is read from the From de375b4c0f0744c9c0d1367d383ac32c70ce3e6f Mon Sep 17 00:00:00 2001 From: "marcin p. joachimiak" <4625870+realmarcin@users.noreply.github.com> Date: Fri, 25 Sep 2026 14:37:27 -0700 Subject: [PATCH 6/8] Report every malformed audit, and send WRONG to every occurrence of the figure An audit of null, {} or one without repositories got no AUDIT row: grown sites read STALE and a mistyped card could pass (#257). WRONG's remedy now says to grep the tree for the old figure, as the update skill's step 6 does, since it also appears in the MECHS extra: prose and cross-references (#258). The tests cover those audit shapes and the remedy's content (#259). Co-Authored-By: Claude Opus 5.5 (1M context) --- _fleet/README.md | 9 +++++---- scripts/fleet/check_cards.py | 20 ++++++++++++++------ tests/test_fleet_page.py | 18 ++++++++++++++---- 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/_fleet/README.md b/_fleet/README.md index 297d3dd..eb18779 100644 --- a/_fleet/README.md +++ b/_fleet/README.md @@ -58,10 +58,11 @@ remedy for each failing verdict (#235): - STALE or SHRANK: the card figures in `mechs_template.md` and the `MECHS` block in `fleet_fragment.html` need a full refresh, since the page is a snapshot. - WRONG: a card, or the audit's `figure_at_pin`, was mistyped. Correct the card - and every copy of its figure (the MECHS `records:` in `fleet_fragment.html`, - `card_records` in `site_audit.json`, and the pages step 6 of the update skill - lists), or `figure_at_pin` if that is what was wrong, then rerun - `assemble_page.py`; no re-pin (#248). The unit tests catch this on the + and every other occurrence of its figure, found by grepping the tree for it as + step 6 of the update skill does: the MECHS `records:` and `extra:` text in + `fleet_fragment.html`, cross-references, `card_records` in `site_audit.json` + and the pages that repeat it. Or correct `figure_at_pin` if that is what was + wrong. Then rerun `assemble_page.py`; no re-pin (#248, #258). The unit tests catch this on the PR and in the nightly, which still runs the card check after a failed test step so its report prints (#239, #240). - GONE or CHANGED: a `SOURCES` entry needs repointing. diff --git a/scripts/fleet/check_cards.py b/scripts/fleet/check_cards.py index 7d2a5f6..118b87c 100644 --- a/scripts/fleet/check_cards.py +++ b/scripts/fleet/check_cards.py @@ -255,7 +255,9 @@ def audit_entries(audit) -> tuple[dict, str | None]: """The audit's repositories keyed by lower-cased name, or why they cannot be read.""" if not isinstance(audit, dict): return {}, f"site_audit.json holds a {type(audit).__name__}, not an object" - repositories = audit.get("repositories", []) + if "repositories" not in audit: + return {}, "site_audit.json has no repositories list" # #257 + repositories = audit["repositories"] if not isinstance(repositories, list) or not all( isinstance(row, dict) and isinstance(row.get("repo"), str) for row in repositories): return {}, "site_audit.json: repositories must be a list of objects, each with a repo name" @@ -278,7 +280,7 @@ def check(template: str, fetcher=None, audit=None, now: datetime.datetime | None rows.append(("AUDIT", "-", problem)) audit = None age = None - if audit: + if audit is not None: # {} too, so a missing pin time is reported (#257) try: age = now - pin_time(audit) except ValueError as error: @@ -326,6 +328,11 @@ def main() -> int: audit, audit_error = None, None try: audit = json.loads(AUDIT.read_text()) + if not isinstance(audit, dict): + # check() reads None as "no audit supplied", so a null file would + # otherwise pass silently (#257). + audit_error = f"site_audit.json holds {type(audit).__name__}, not an object" + audit = None except FileNotFoundError: audit_error = "site_audit.json is missing" except (OSError, ValueError) as error: @@ -340,10 +347,11 @@ def main() -> int: if any(status in FAILURES for status, _, _ in rows): print("Failing. STALE or SHRANK: refresh the card figures in _fleet/mechs_template.md " "and the MECHS block in _fleet/fleet_fragment.html (update-xmech-page), then rerun " - "assemble_page.py. WRONG: correct the card and every copy of its figure (the MECHS " - "records in _fleet/fleet_fragment.html, card_records in site_audit.json and the pages " - "update-xmech-page step 6 lists), or the audit's figure_at_pin if that is what was " - "mistyped, then rerun assemble_page.py; no re-pin. GONE or CHANGED: repoint that Mech's SOURCES entry. MARKUP or " + "assemble_page.py. WRONG: correct the card and every other occurrence of its " + "figure, found by grepping the tree for it as update-xmech-page step 6 does (the MECHS " + "records: and extra: text in _fleet/fleet_fragment.html, cross-references, card_records " + "in site_audit.json, the pages that repeat it), or the audit's figure_at_pin if that is " + "what was mistyped, then rerun assemble_page.py; no re-pin. GONE or CHANGED: repoint that Mech's SOURCES entry. MARKUP or " "UNCARDED: fix the card or its SOURCES entry. AUDIT: fix site_audit.json. " "UNCHECKED: the run could not reach most sites; rerun before changing anything.") return 1 diff --git a/tests/test_fleet_page.py b/tests/test_fleet_page.py index e7e29b4..5a3293c 100644 --- a/tests/test_fleet_page.py +++ b/tests/test_fleet_page.py @@ -586,8 +586,11 @@ def test_a_missing_future_or_unreadable_pin_time_fails_on_its_own(self): def test_a_malformed_audit_is_an_audit_row_not_a_traceback(self): # #250: the other rows must still print. - for audit in ([], {"pinned_at_utc": self.pinned_at.isoformat(), "repositories": [{"figure_at_pin": 1}]}, - {"pinned_at_utc": self.pinned_at.isoformat(), "repositories": "x"}): + pinned = self.pinned_at.isoformat() + for audit in ([], [1], {}, {"pinned_at_utc": pinned}, # #257, #259 + {"pinned_at_utc": pinned, "repositories": [{"figure_at_pin": 1}]}, + {"pinned_at_utc": pinned, "repositories": "x"}, + {"pinned_at_utc": pinned, "repositories": 5}): rows = self.check_cards.check(self.template(), self.fetch, audit, self.now) self.assertIn("AUDIT", [s for s, _, _ in rows], audit) self.assertEqual(sorted(m for s, m, _ in rows if s == "ok"), list(self.NAMES), audit) @@ -737,13 +740,20 @@ def test_main_exits_by_the_same_rule(self): with contextlib.redirect_stdout(out): self.assertEqual(self.check_cards.main(), 1) remedy = out.getvalue().split("WRONG:", 1)[1].split(" GONE or CHANGED:", 1)[0] - self.assertIn("every copy", remedy) - self.assertIn("no re-pin", remedy) + for named in ("every other occurrence", "grepping the tree", "fleet_fragment.html", + "extra:", "card_records", "no re-pin"): # #248, #258, #259 + self.assertIn(named, remedy) self.assertNotIn("update-xmech-page)", remedy) self.at_pin["AMech"] = 1000 # AUDIT alone fails the run, and an unreadable file is AUDIT, not a traceback (#250). audit.write_text("{not json") self.assertEqual(self.check_cards.main(), 1) + for body in ("null", "{}", "[1]"): # #257: none may pass as "no audit" + audit.write_text(body) + out = io.StringIO() + with contextlib.redirect_stdout(out): + self.assertEqual(self.check_cards.main(), 1, body) + self.assertIn("AUDIT", out.getvalue(), body) audit.unlink() self.assertEqual(self.check_cards.main(), 1) From e6437e33ca07fcd747fbca58710fb6d99cdf171a Mon Sep 17 00:00:00 2001 From: "marcin p. joachimiak" <4625870+realmarcin@users.noreply.github.com> Date: Fri, 25 Sep 2026 14:45:45 -0700 Subject: [PATCH 7/8] Count headline tiles, not parsed figures, and require a whole figure_at_pin A second tile with a stray space escaped the one-figure rule, and "," parsed and then crashed int(""); the parser now counts every headline tile and reads a figure only when it starts with a digit (#261). A second card for one Mech is MARKUP, not a card the nightly silently skips (#262). A figure_at_pin that is a string, float or missing is an AUDIT row, and the provenance test requires an int (#260). The review skill names both growth limits (#263). Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/skills/review-open-issues/SKILL.md | 6 ++- _fleet/README.md | 6 ++- scripts/fleet/card_markup.py | 34 ++++++++++++----- scripts/fleet/check_cards.py | 24 ++++++++++-- tests/test_fleet_page.py | 44 +++++++++++++++++++--- 5 files changed, 90 insertions(+), 24 deletions(-) diff --git a/.claude/skills/review-open-issues/SKILL.md b/.claude/skills/review-open-issues/SKILL.md index 6e7d26f..b7fb442 100644 --- a/.claude/skills/review-open-issues/SKILL.md +++ b/.claude/skills/review-open-issues/SKILL.md @@ -218,8 +218,10 @@ python scripts/fleet/check_cards.py # card headline figures vs each Mech's sit A stale card therefore shows up as a failed scheduled run, not a failed PR check. A card behind a fast Mech only warns ("grew") for 14 days after the -refresh's pins, so a green run does not mean every card equals its site: read -the run's log, not just its colour, before crediting the cards as current. +refresh's pins, and only while its site is at most half as large again as the +card; past either limit it fails as STALE (#263). So a green run does not mean +every card equals its site: read the run's log, not just its colour, before +crediting the cards as current. An issue asserting a defect that one of these already blocks is P2 unless it shows the gate is porous — and they have been porous: a test can pass because diff --git a/_fleet/README.md b/_fleet/README.md index eb18779..5a70da4 100644 --- a/_fleet/README.md +++ b/_fleet/README.md @@ -47,8 +47,10 @@ fails when a card differs from `figure_at_pin`, the figure `site_audit.json` records its source stating at the pin (the card was never right), when a site is behind its card, when a source answers a 4xx other than a throttle or no longer states a figure the parser can read, when a card lacks exactly one headline -figure or has no `SOURCES` entry, when the audit is missing or malformed or -its pin time is missing, unreadable or in the future, and +figure or has no `SOURCES` entry, when the audit is missing or malformed, its +pin time is missing, unreadable or in the future, or a source lacks a +whole-number `figure_at_pin` (all but ProteinTraitsMech, whose file is built in +CI), and when more than half the sources could not be fetched. One site's outage or throttle only warns (#148, #115, #176, #217-#220, #231, #232). It runs on the workflow's nightly schedule, not on pull requests, so a Mech shipping records diff --git a/scripts/fleet/card_markup.py b/scripts/fleet/card_markup.py index f27c348..222136e 100644 --- a/scripts/fleet/card_markup.py +++ b/scripts/fleet/card_markup.py @@ -7,10 +7,16 @@ """ from __future__ import annotations +import collections import re ARTICLE = re.compile(r']*\bdata-mech="([^"]+)"[^>]*>(.*?)
', re.S) -FIGURE = re.compile(r'
([\d,]+)') +# A card's headline tile, counted whatever it holds, so a malformed second tile +# cannot slip past the one-per-card rule (#261). +TILE = re.compile(r'
') +# The figure in a tile: digits with thousands commas, starting with a digit, so +# "," is unreadable rather than int("") (#261). +FIGURE = re.compile(r'
(\d[\d,]*)') def card_names(template: str) -> list[str]: @@ -27,12 +33,18 @@ def markup_problems(template: str) -> list[tuple[str, str]]: """ problems = [] for mech, body in ARTICLE.findall(template): - count = len(FIGURE.findall(body)) - if count == 0: - problems.append((mech, "card has no readable headline figure")) - elif count > 1: - problems.append((mech, f"card has {count} headline figures; it must have exactly one")) - outside = len(FIGURE.findall(ARTICLE.sub("", template))) + tiles = len(TILE.findall(body)) + if tiles == 0: + problems.append((mech, "card has no headline figure")) + elif tiles > 1: + problems.append((mech, f"card has {tiles} headline figures; it must have exactly one")) + elif not FIGURE.search(body): + problems.append((mech, "card's headline figure is unreadable")) + for mech, count in collections.Counter(card_names(template)).items(): + if count > 1: + # The nightly would check only one of them (#262). + problems.append((mech, f"{count} cards for one Mech")) + outside = len(TILE.findall(ARTICLE.sub("", template))) if outside: problems.append(("-", f"{outside} headline figure(s) outside any card")) return problems @@ -42,12 +54,14 @@ def card_figures(template: str) -> dict[str, int]: """Each card's headline figure, keyed by Mech. Read inside the card's own
, so a card without a figure is absent - rather than borrowing the next card's. A card with more than one figure is - absent too; markup_problems() says which cards those are. + rather than borrowing the next card's. A card with more than one headline + tile, an unreadable one, or a second card for the same Mech is absent too; + markup_problems() says which cards those are. """ + names = collections.Counter(card_names(template)) figures: dict[str, int] = {} for mech, body in ARTICLE.findall(template): hits = FIGURE.findall(body) - if len(hits) == 1: + if names[mech] == 1 and len(TILE.findall(body)) == 1 and len(hits) == 1: figures[mech] = int(hits[0].replace(",", "")) return figures diff --git a/scripts/fleet/check_cards.py b/scripts/fleet/check_cards.py index 118b87c..b4439f3 100644 --- a/scripts/fleet/check_cards.py +++ b/scripts/fleet/check_cards.py @@ -33,8 +33,9 @@ the wording moved, and the card is no longer being checked at all. MARKUP a card in the template does not carry exactly one headline figure. UNCARDED a card with no SOURCES entry, or an entry with no card. - AUDIT site_audit.json is missing or malformed, or its pinned_at_utc is - missing, cannot be read as a time, or is in the future. + AUDIT site_audit.json is missing or malformed, its pinned_at_utc is + missing, cannot be read as a time, or is in the future, or a + source's figure_at_pin is missing or not a whole number. unread the fetch did not arrive: DNS, timeout, a dropped connection, a 5xx, or a 408, 425 or 429 throttle. A warning, because the network is not the site's fault, unless more than half the sources are unread, @@ -113,6 +114,10 @@ GRACE_DAYS = 14 MAX_LEAD = 0.5 +# Sources with no committed copy at the pin, so no figure_at_pin: the served +# file is built in CI. Every other source must have one, or WRONG is off (#260). +NO_PIN_COPY = ("ProteinTraitsMech",) + # 4xx answers that mean "not now" rather than "not here" (#219). THROTTLES = (408, 425, 429) @@ -303,8 +308,19 @@ def check(template: str, fetcher=None, audit=None, now: datetime.datetime | None if mech not in stated: continue # its card's markup is already reported above card = stated[mech] - at_pin = entries.get(mech.lower(), {}).get("figure_at_pin") - if isinstance(at_pin, int) and not isinstance(at_pin, bool) and at_pin != card: + entry = entries.get(mech.lower()) + at_pin = entry.get("figure_at_pin") if entry else None + whole = isinstance(at_pin, int) and not isinstance(at_pin, bool) + if audit is not None: + # A missing or mistyped figure_at_pin would switch WRONG off without + # a word (#260). + if entry is None: + rows.append(("AUDIT", mech, "no entry in site_audit.json")) + elif at_pin is None and mech not in NO_PIN_COPY: + rows.append(("AUDIT", mech, "site_audit.json has no figure_at_pin for it")) + elif at_pin is not None and not whole: + rows.append(("AUDIT", mech, f"figure_at_pin is {at_pin!r}, not a whole number")) + if whole and at_pin != card: rows.append(("WRONG", mech, f"card {card:,}, but the source stated {at_pin:,} at the pin")) continue status, result = read_source(mech, kind, path, selector, fetcher) diff --git a/tests/test_fleet_page.py b/tests/test_fleet_page.py index 5a3293c..a3310d5 100644 --- a/tests/test_fleet_page.py +++ b/tests/test_fleet_page.py @@ -491,8 +491,14 @@ def template(self): for m, n in self.cards.items()) def audit(self): - return {"pinned_at_utc": self.pinned_at.isoformat(), "repositories": [ - {"repo": m, "figure_at_pin": self.at_pin[m]} for m in self.check_cards.SOURCES if m in self.at_pin]} + entries = [] + for m in self.check_cards.SOURCES: + if m in self.at_pin: + entry = {"repo": m} + if self.at_pin[m] is not None: + entry["figure_at_pin"] = self.at_pin[m] + entries.append(entry) + return {"pinned_at_utc": self.pinned_at.isoformat(), "repositories": entries} def rows(self, template=None): return self.check_cards.check(template or self.template(), self.fetch, self.audit(), self.now) @@ -549,9 +555,23 @@ def test_a_mistyped_card_fails_however_the_site_has_moved(self): def test_a_source_with_no_figure_at_the_pin_is_checked_against_the_site_only(self): # ProteinTraitsMech's data file is built in CI, so the audit has no copy. - del self.at_pin["AMech"] + from unittest import mock + self.at_pin["AMech"] = None self.site["AMech"] = 1100 - self.assertEqual(self.statuses()["AMech"], "grew") + with mock.patch.object(self.check_cards, "NO_PIN_COPY", ("AMech",)): + self.assertEqual(self.statuses()["AMech"], "grew") + self.assertEqual(self.failed(), []) + + def test_a_missing_or_mistyped_figure_at_the_pin_is_an_audit_failure(self): + # #260: each of these used to switch WRONG off without a word, so a + # mistyped card passed as "grew". + self.cards["AMech"] = 3026 + self.site["AMech"] = 3206 + for at_pin in ("3206", 3206.0, True, None): + self.at_pin["AMech"] = at_pin + rows = self.rows() + self.assertIn(("AUDIT", "AMech"), [(s, m) for s, m, _ in rows], at_pin) + self.assertIn("AUDIT", self.failed(), at_pin) def test_a_pin_time_without_an_offset_is_utc_and_an_unreadable_one_fails(self): # #232: both used to end the run with a traceback before any row printed. @@ -686,6 +706,17 @@ def test_a_card_without_exactly_one_figure_is_reported_as_markup(self): self.assertEqual(self.failed(doubled), ["MARKUP"]) stray = template + '
7
' self.assertEqual(self.failed(stray), ["MARKUP"]) + # #261: an unreadable figure is MARKUP, not int("") ending the run, and + # a malformed second tile, in a card or outside one, still counts. + comma = template.replace("1,000", ",", 1) + self.assertEqual(self.failed(comma), ["MARKUP"]) + spaced = template.replace('
', + '
7
', 1) + self.assertEqual(self.failed(spaced), ["MARKUP"]) + self.assertEqual(self.failed(template + '
7
'), ["MARKUP"]) + # #262: a second card for one Mech would have been checked instead of the first. + twice = '
3,026
' + template + self.assertEqual(self.failed(twice), ["MARKUP"]) def test_the_culturemech_figure_is_read_only_from_its_generated_block(self): # #176: the regex takes the first match, and the README's prose could @@ -828,11 +859,12 @@ def test_every_card_equals_the_figure_its_source_stated_at_the_pin(self): for mech in check_cards.SOURCES: with self.subTest(mech=mech): entry = entries[mech.lower()] - if mech == "ProteinTraitsMech": + if mech in check_cards.NO_PIN_COPY: # Built in CI, so there is no committed copy at the pin. self.assertNotIn("figure_at_pin", entry) continue - self.assertEqual(entry.get("figure_at_pin"), cards[mech]) + self.assertIs(type(entry.get("figure_at_pin")), int) # #260: not 3206.0 + self.assertEqual(entry["figure_at_pin"], cards[mech]) def test_the_audit_records_this_refresh_and_nothing_else(self): # The audit's other fields had no gate at all (#128): its merged PRs, From 2af4418a22d8f2e69eb85603f818984ada70fb94 Mon Sep 17 00:00:00 2001 From: "marcin p. joachimiak" <4625870+realmarcin@users.noreply.github.com> Date: Fri, 25 Sep 2026 14:55:45 -0700 Subject: [PATCH 8/8] Read a card figure only when its commas group thousands "32,06" was read as 3206 and passed every gate while the page showed it malformed; a figure is now plain digits or digits grouped in threes (#264). Co-Authored-By: Claude Opus 5.5 (1M context) --- scripts/fleet/card_markup.py | 7 ++++--- tests/test_fleet_page.py | 7 +++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/fleet/card_markup.py b/scripts/fleet/card_markup.py index 222136e..10a3e15 100644 --- a/scripts/fleet/card_markup.py +++ b/scripts/fleet/card_markup.py @@ -14,9 +14,10 @@ # A card's headline tile, counted whatever it holds, so a malformed second tile # cannot slip past the one-per-card rule (#261). TILE = re.compile(r'
') -# The figure in a tile: digits with thousands commas, starting with a digit, so -# "," is unreadable rather than int("") (#261). -FIGURE = re.compile(r'
(\d[\d,]*)') +# The figure in a tile: plain digits, or digits grouped in threes by commas, so +# "," is unreadable rather than int("") (#261) and "32,06" is unreadable +# rather than 3206 (#264). +FIGURE = re.compile(r'
(\d{1,3}(?:,\d{3})+|\d+)') def card_names(template: str) -> list[str]: diff --git a/tests/test_fleet_page.py b/tests/test_fleet_page.py index a3310d5..d64b102 100644 --- a/tests/test_fleet_page.py +++ b/tests/test_fleet_page.py @@ -708,8 +708,11 @@ def test_a_card_without_exactly_one_figure_is_reported_as_markup(self): self.assertEqual(self.failed(stray), ["MARKUP"]) # #261: an unreadable figure is MARKUP, not int("") ending the run, and # a malformed second tile, in a card or outside one, still counts. - comma = template.replace("1,000", ",", 1) - self.assertEqual(self.failed(comma), ["MARKUP"]) + for bad in (",", "10,00", "1000,", "1,0000"): # #261, #264 + self.assertEqual(self.failed(template.replace("1,000", bad, 1)), ["MARKUP"], bad) + from card_markup import card_figures as figures + for good, value in (("1000", 1000), ("625,960", 625960), ("7", 7)): + self.assertEqual(figures(template.replace("1,000", good, 1))["AMech"], value, good) spaced = template.replace('
', '
7
', 1) self.assertEqual(self.failed(spaced), ["MARKUP"])