LABEL
+# text the figure is in a sentence, as "N LABEL" — CultureMech and
+# NaturalProductMech state theirs in prose rather than a tile
+# json the page computes it at runtime, so the markup carries a placeholder
+# and only the data file has the number (#86)
+#
+# Several repo roots are client-side meta-refresh shells that return 200, so
+# these are the pages/ or app/ URLs, never the root.
+SOURCES: dict[str, tuple[str, str, str]] = {
+ "HabitatMech": ("html", "HabitatMech/pages/index.html", "habitat records"),
+ "CommunityMech": ("html", "CommunityMech/", "communities"),
+ "TaxonMech": ("html", "TaxonMech/pages/index.html", "taxon records"),
+ "TraitMech": ("html", "TraitMech/pages/index.html", "trait records"),
+ "CellStructureMech": ("html", "CellStructureMech/pages/index.html", "structure records"),
+ "AntibioticMech": ("html", "AntibioticMech/pages/index.html", "compound records"),
+ "NaturalProductMech": ("text", "NaturalProductMech/pages/index.html", "natural product structures"),
+ # Not the app/ landing tile, which is a legacy hand-typed figure matching no
+ # data layer; pages/ is the merged canonical count the card states (#86).
+ "CultureMech": ("text", "CultureMech/pages/", "media records"),
+ "ProteinTraitsMech": ("json", "proteintraitsmech/data/facets.json", "total"),
+ "MediaIngredientMech": ("json", "MediaIngredientMech/data/ingredients.json", "ingredients"),
+}
+
+CARD = re.compile(r'data-mech="([A-Za-z]+)".*?([\d,]+)', re.S)
+
+
+def cards(template: str) -> dict[str, int]:
+ """The headline figure each card states, keyed by Mech."""
+ return {m.group(1): int(m.group(2).replace(",", "")) for m in CARD.finditer(template)}
+
+
+def fetch(url: str, timeout: int = 30) -> str:
+ request = urllib.request.Request(url, headers={"User-Agent": "culturebotai-card-check"})
+ return urllib.request.urlopen(request, timeout=timeout).read().decode("utf-8", "replace")
+
+
+def published(kind: str, body: str, selector: str) -> int | None:
+ """The figure the site publishes, or None when the shape has changed."""
+ if kind == "json":
+ document = json.loads(body)
+ # ingredients.json has shipped as a bare list in some releases. Test the
+ # shape before reaching into it: a list has no .get, and the error that
+ # raises is not a parse error, so it used to escape as a traceback
+ # instead of an "unread" line (#110).
+ if isinstance(document, list):
+ return len(document)
+ if not isinstance(document, dict):
+ return None
+ value = document.get(selector)
+ if isinstance(value, list):
+ return len(value)
+ if isinstance(value, int) and not isinstance(value, bool):
+ return value
+ return None
+ if kind == "text":
+ # The figure sits in a sentence. Strip tags first so markup between the
+ # number and the words it belongs to cannot hide the pairing.
+ prose = re.sub(r"<[^>]+>", " ", body)
+ hit = re.search(r"([\d,]+)\s+" + re.escape(selector), prose)
+ return int(hit.group(1).replace(",", "")) if hit else None
+ hit = re.search(r"([\d,]+)\s*\s*" + re.escape(selector), body)
+ return int(hit.group(1).replace(",", "")) if hit else None
+
+
+def main() -> int:
+ stated = cards(TEMPLATE.read_text())
+ missing = sorted(set(stated) - set(SOURCES))
+ drifted, unreadable, matched = [], [], []
+
+ for mech, (kind, path, selector) in sorted(SOURCES.items()):
+ if mech not in stated:
+ unreadable.append((mech, "no card in the template"))
+ continue
+ try:
+ body = fetch(SITE + path)
+ except (urllib.error.URLError, TimeoutError, OSError) as error:
+ unreadable.append((mech, f"fetch failed: {error}"))
+ continue
+ try:
+ value = published(kind, body, selector)
+ except (ValueError, TypeError, AttributeError) as error:
+ # json.JSONDecodeError is a ValueError. The other two are what a
+ # body of an unexpected type raises when it is walked.
+ unreadable.append((mech, f"unparseable: {error}"))
+ continue
+ if value is None:
+ unreadable.append((mech, f"{kind} shape changed; no {selector!r} found"))
+ elif value != stated[mech]:
+ drifted.append((mech, stated[mech], value))
+ else:
+ matched.append(mech)
+
+ for mech in matched:
+ print(f" ok {mech:<20} {stated[mech]:>9,}")
+ for mech, card, site in drifted:
+ print(f" DRIFTED {mech:<20} card {card:,}, site {site:,}")
+ for mech, why in unreadable:
+ print(f" unread {mech:<20} {why}")
+ if missing:
+ print(f" MISSING cards with no entry in SOURCES: {', '.join(missing)}")
+
+ print(f"\n{len(matched)} match, {len(drifted)} drifted, {len(unreadable)} unreadable.")
+ if drifted or missing:
+ print("Refresh the card figures in _fleet/mechs_template.md and the MECHS block "
+ "in _fleet/fleet_fragment.html, then rerun assemble_page.py.")
+ return 1
+ if unreadable:
+ # A site that cannot be reached is not the same as a wrong number, and a
+ # nightly red for someone else's outage teaches people to ignore it.
+ print("Nothing drifted, but some sites could not be read; see above.")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tests/test_fleet_page.py b/tests/test_fleet_page.py
index 7ebe034..32535e8 100644
--- a/tests/test_fleet_page.py
+++ b/tests/test_fleet_page.py
@@ -379,5 +379,26 @@ def test_importing_the_census_module_does_not_scan_or_write(self):
self.assertEqual(written, [], f"importing prefix_census wrote {written}")
self.assertEqual(tracked.read_bytes(), before)
+
+class CardSourceTests(unittest.TestCase):
+ """Every card must have somewhere to be checked against (#104)."""
+
+ def test_every_card_has_a_published_source(self):
+ # The drift check is only as complete as this table. An eleventh Mech
+ # would otherwise get a card and be silently exempt from checking,
+ # which is the failure this whole issue is about.
+ import check_cards
+ stated = check_cards.cards((ROOT / "_fleet/mechs_template.md").read_text())
+ self.assertEqual(sorted(set(stated) - set(check_cards.SOURCES)), [],
+ "card with no entry in check_cards.SOURCES")
+ self.assertEqual(sorted(set(check_cards.SOURCES) - set(stated)), [],
+ "SOURCES entry with no card")
+
+ def test_the_card_parser_reads_every_member(self):
+ snapshot = json.loads((ROOT / "_fleet/data/manifest.json").read_text())
+ import check_cards
+ stated = check_cards.cards((ROOT / "_fleet/mechs_template.md").read_text())
+ self.assertEqual(sorted(stated), sorted(snapshot["mechs"]))
+
if __name__ == '__main__':
unittest.main()