From 441159ca982c5a0fed3410722fcca6b35cf83ebb Mon Sep 17 00:00:00 2001 From: Lukas Wallrich Date: Wed, 2 Sep 2026 17:51:05 +0100 Subject: [PATCH 1/2] Make sitemap lastmod accurate and stable Hugo reads page dates from `git log --name-only`, and Git quotes non-ASCII paths unless core.quotepath is off; the quoted paths never matched, so every glossary term or curated resource with a non-ASCII filename (737 pages) had no lastmod. The build workflows now set core.quotepath=false. Generated collections carry their own `lastmod`: the glossary and curated resource generators keep an entry's previous date while its content is unchanged and set today's date only when it changes. Existing entries bootstrap from the file's last commit (glossary) or the sheet submission timestamp (curated resources). The data-processing workflow compares curated resources against the build-resources branch so dates do not drift between seed merges. Hugo now prefers an explicit front-matter lastmod over Git. scripts/check_sitemap_lastmod.py reports undated URLs, timestamps stamped on most of the site, and pages whose date disagrees with their source; the deploy workflow runs it as a warning-only step. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01CAq4MsBnm8dNGe1zDqZwgn --- .github/workflows/data-processing.yml | 21 +++ .github/workflows/deploy.yaml | 17 ++ .github/workflows/staging-aggregate.yaml | 6 + config/_default/config.toml | 8 + content/glossary/_create_glossaries.py | 18 +- content/resources/resource.py | 50 ++++- scripts/check_sitemap_lastmod.py | 223 +++++++++++++++++++++++ scripts/generated_lastmod.py | 73 ++++++++ 8 files changed, 412 insertions(+), 4 deletions(-) create mode 100644 scripts/check_sitemap_lastmod.py create mode 100644 scripts/generated_lastmod.py diff --git a/.github/workflows/data-processing.yml b/.github/workflows/data-processing.yml index 6500c7666d6..bc3feb90b64 100644 --- a/.github/workflows/data-processing.yml +++ b/.github/workflows/data-processing.yml @@ -60,6 +60,10 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 # Checkout the repository code to the runner environment + with: + # Full history: the generators read per-file commit dates to bootstrap + # `lastmod`, which a shallow clone would report as HEAD for every file. + fetch-depth: 0 #====================== # Workflow Configuration @@ -224,12 +228,29 @@ jobs: continue-on-error: true run: python3 scripts/build_get_involved.py + #======================================== + # Fetch the previous generated curated resources from the build-resources + # branch, so unchanged entries keep their lastmod date. + # If the branch or the extraction is unavailable, the directory stays + # absent and resource.py falls back to the seed copy in the checkout. + #======================================== + - name: Fetch previous curated resources state + id: previous-curated-resources + continue-on-error: true + run: | + mkdir -p /tmp/previous-generated + git fetch origin build-resources || true + git archive origin/build-resources content/curated_resources \ + | tar -x -C /tmp/previous-generated || true + #======================================== # Process and organize curated resources data #======================================== - name: Run Curated Resources script id: curated-resources continue-on-error: true # Continue even if this step fails + env: + CURATED_PREVIOUS_DIR: /tmp/previous-generated/content/curated_resources run: python3 content/resources/resource.py # Execute the curated resources script that processes and organizes resource data diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 2b5a3cec30f..efe3d603560 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -41,6 +41,12 @@ jobs: with: fetch-depth: 0 + # Hugo derives page dates from `git log --name-only`, and Git quotes + # non-ASCII paths (glossary terms, accented resource titles) unless this + # is off. Quoted paths never match, leaving those pages without a date. + - name: Let Git report non-ASCII paths unquoted + run: git config core.quotepath false + # ======================= # Data Artifact Retrieval # ======================= @@ -208,6 +214,17 @@ jobs: env: HUGO_ENV: production + # ======================= + # Sitemap Quality Check + # ======================= + # Reports pages missing a , a timestamp stamped on most of the + # site, and a sample of pages whose date disagrees with their source + # (front-matter lastmod or Git). Warns only: a stale sitemap date must + # not block a deployment. + - name: Check sitemap lastmod + continue-on-error: true + run: python3 scripts/check_sitemap_lastmod.py public/sitemap.xml --compare-source --sample 200 + # ======================= # Deployment Artifact #======================================== diff --git a/.github/workflows/staging-aggregate.yaml b/.github/workflows/staging-aggregate.yaml index f4a817d08b5..f225775f6ee 100644 --- a/.github/workflows/staging-aggregate.yaml +++ b/.github/workflows/staging-aggregate.yaml @@ -263,6 +263,12 @@ jobs: uses: actions/checkout@v6 with: ref: ${{ needs.aggregate-prs.outputs.aggregated-branch || 'main' }} + fetch-depth: 0 # full history so Hugo's Git-derived dates match production + + # See deploy.yaml: Git quotes non-ASCII paths unless this is off, and + # Hugo then finds no date for them. + - name: Let Git report non-ASCII paths unquoted + run: git config core.quotepath false - name: Configure Git run: | diff --git a/config/_default/config.toml b/config/_default/config.toml index 0a4c8fb3a18..1fcca618438 100644 --- a/config/_default/config.toml +++ b/config/_default/config.toml @@ -26,6 +26,14 @@ theme = "academic" # Get last modified date for content from Git? enableGitInfo = true +# An explicit `lastmod` in front matter wins over the Git commit date. Generated +# collections (glossary, curated resources) carry their own `lastmod`, set by +# their generators only when an entry's content changes; hand-maintained pages +# without one fall back to Git. Requires `core.quotepath=false` in the build +# checkout so Git reports non-ASCII filenames unquoted (see deploy.yaml). +[frontmatter] +lastmod = ["lastmod", ":git", "date", "publishDate"] + # Enable generation of robots.txt file enableRobotsTXT = true diff --git a/content/glossary/_create_glossaries.py b/content/glossary/_create_glossaries.py index 479f682685f..260871ee7af 100755 --- a/content/glossary/_create_glossaries.py +++ b/content/glossary/_create_glossaries.py @@ -3,10 +3,15 @@ import json import pandas as pd import os +import sys from io import StringIO +from pathlib import Path from pypinyin import lazy_pinyin, Style script_dir = os.path.dirname(os.path.abspath(__file__)) + +sys.path.insert(0, str(Path(script_dir).resolve().parents[1] / 'scripts')) +from generated_lastmod import load_previous, resolve_lastmod, git_commit_date language_map = { 'EN': 'english', 'AR': 'arabic', @@ -289,6 +294,11 @@ def clean_filename(title, max_length=200): # Create markdown files for language_name, entries in formatted_data.items(): language_dir = os.path.join(script_dir, language_name) + + # Read the previous entries before deleting them, so each term can keep its + # `lastmod` for as long as its content is unchanged. + previous_entries = load_previous(language_dir) + # Remove existing glossary entry files to ensure deleted entries don't persist # Preserve _index.md files as they are not regenerated if os.path.exists(language_dir): @@ -304,7 +314,13 @@ def clean_filename(title, max_length=200): for entry in entries: file_name = clean_filename(entry['title']) file_path = os.path.join(language_dir, file_name + ".md") - + + entry["lastmod"] = resolve_lastmod( + entry, + previous_entries.get(file_name + ".md"), + lambda: git_commit_date(file_path), + ) + with open(file_path, 'w', encoding='utf-8') as f: json.dump(entry, f, ensure_ascii=False, indent=4) diff --git a/content/resources/resource.py b/content/resources/resource.py index 6066c7b2efd..fd61791b1ca 100644 --- a/content/resources/resource.py +++ b/content/resources/resource.py @@ -17,10 +17,44 @@ """ import json -import pandas as pd +import os import re +import sys from pathlib import Path +import pandas as pd +from dateutil import parser as date_parser + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / 'scripts')) +from generated_lastmod import load_previous, resolve_lastmod + + +def previous_directory(fpath: Path) -> Path: + """Where the previous run's files are. + + The daily workflow points CURATED_PREVIOUS_DIR at a checkout of the + `build-resources` branch, which holds the last generated state. Without it + (local runs, or a missing branch) the checkout's own seed copy is used. + """ + env_dir = os.environ.get('CURATED_PREVIOUS_DIR') + if env_dir and Path(env_dir).is_dir(): + return Path(env_dir) + return fpath + + +def submission_date(timestamp) -> str: + """Sheet submission date as YYYY-MM-DD, or None if it cannot be parsed. + + Used as the `lastmod` for entries carried over from before this key + existed. The sheet mixes ISO and US month-first formats. + """ + if not timestamp or not isinstance(timestamp, str): + return None + try: + return date_parser.parse(timestamp).strftime('%Y-%m-%d') + except (ValueError, OverflowError): + return None + ## Function definition @@ -87,6 +121,10 @@ def convert_row_to_file(df, fpath): If there are duplicates, an index is appended to the filename. """ + # Read the previous run's files before deleting them, so each resource can + # keep its `lastmod` for as long as its content is unchanged. + previous = load_previous(previous_directory(fpath)) + # Replace the generated collection rather than leaving files for rows that # have been removed from the source sheet. for path in fpath.iterdir(): @@ -108,8 +146,14 @@ def convert_row_to_file(df, fpath): else: filename_counts[filename] = 1 filename_md = fpath / f"{filename}.md" - - filename_md.write_text(json.dumps(row.to_dict(), indent=4)) + + entry = row.to_dict() + entry["lastmod"] = resolve_lastmod( + entry, + previous.get(filename_md.name), + lambda: submission_date(entry.get("timestamp")), + ) + filename_md.write_text(json.dumps(entry, indent=4)) # Import data and prettify it: diff --git a/scripts/check_sitemap_lastmod.py b/scripts/check_sitemap_lastmod.py new file mode 100644 index 00000000000..28159521efb --- /dev/null +++ b/scripts/check_sitemap_lastmod.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +"""Check that sitemap.xml carries a plausible for every URL. + +Two levels of checking: + +* Structural (always): counts URLs without (grouped by the first path + segment) and lists the most widely shared timestamps. A timestamp on more + than --max-bulk-share of all URLs fails the check: that is the signature of + every page being stamped with the build or checkout time. Smaller blocks are + reported but tolerated, since one commit can legitimately touch a whole + generated collection. +* Source comparison (--compare-source): rebuilds the expected lastmod for each + page from the repository -- explicit front-matter `lastmod`, else the Git + commit date of the source file -- and reports disagreements. Needs `hugo` + and the full Git history. + +Usage: + python3 scripts/check_sitemap_lastmod.py public/sitemap.xml + python3 scripts/check_sitemap_lastmod.py https://forrt.org/sitemap.xml + python3 scripts/check_sitemap_lastmod.py public/sitemap.xml --compare-source + +Exits non-zero when a check fails, so it can gate a build. +""" + +import argparse +import csv +import io +import json +import random +import re +import subprocess +import sys +import urllib.request +import xml.etree.ElementTree as ET +from collections import Counter, defaultdict +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import urlparse + +REPO = Path(__file__).resolve().parents[1] +SM_NS = "{http://www.sitemaps.org/schemas/sitemap/0.9}" +YAML_FIELD = r"^{key}:\s*[\"']?([^\"'\n#]+)" + + +def load_sitemap(source): + """Return [(loc, lastmod_or_None)] from a local path or an http(s) URL.""" + if source.startswith(("http://", "https://")): + with urllib.request.urlopen(source, timeout=60) as response: + data = response.read() + else: + data = Path(source).read_bytes() + root = ET.fromstring(data) + entries = [] + for url in root.iter(SM_NS + "url"): + loc = url.findtext(SM_NS + "loc") or "" + lastmod = url.findtext(SM_NS + "lastmod") + entries.append((loc.strip(), (lastmod or "").strip() or None)) + return entries + + +def section_of(loc): + path = urlparse(loc).path.strip("/") + return path.split("/")[0] if path else "(home)" + + +def to_instant(value): + """Parse an ISO date or datetime into an aware datetime, or None.""" + if not value: + return None + text = value.strip().replace("Z", "+00:00") + try: + parsed = datetime.fromisoformat(text) + except ValueError: + return None + return parsed.replace(tzinfo=timezone.utc) if parsed.tzinfo is None else parsed + + +def check_structure(entries, max_missing, max_bulk_share): + total = len(entries) + print(f"URLs in sitemap: {total}") + if not total: + return False + + missing = [loc for loc, lastmod in entries if lastmod is None] + print(f"\nURLs without : {len(missing)} (limit {max_missing})") + by_section = defaultdict(list) + for loc in missing: + by_section[section_of(loc)].append(loc) + for name, locs in sorted(by_section.items(), key=lambda kv: -len(kv[1])): + print(f" {name}: {len(locs)} e.g. {', '.join(locs[:3])}") + + counts = Counter(lastmod for _, lastmod in entries if lastmod) + limit = max_bulk_share * total + bulk = [(stamp, n) for stamp, n in counts.most_common() if n > limit] + print(f"\nMost widely shared timestamps (fail above {max_bulk_share:.0%} of URLs):") + for stamp, n in counts.most_common(3): + sections = Counter( + section_of(loc) for loc, lastmod in entries if lastmod == stamp + ) + where = ", ".join(f"{name} ({c})" for name, c in sections.most_common(3)) + print(f" {stamp}: {n} URLs -- {where}") + + return len(missing) <= max_missing and not bulk + + +def hugo_pages(): + """Map URL path -> source path for every regular page, via `hugo list all`.""" + result = subprocess.run( + ["hugo", "list", "all"], + cwd=REPO, + capture_output=True, + text=True, + check=True, + ) + pages = {} + for row in csv.DictReader(io.StringIO(result.stdout)): + if row.get("kind") != "page": + continue + pages[urlparse(row["permalink"]).path] = row["path"] + return pages + + +def front_matter_field(path, key): + """Value of a front-matter `key` (JSON or YAML front matter), or None.""" + try: + text = path.read_text(encoding="utf-8", errors="replace") + except OSError: + return None + stripped = text.lstrip() + if stripped.startswith("{"): + try: + data, _ = json.JSONDecoder().raw_decode(stripped) + except ValueError: + return None + value = data.get(key) if isinstance(data, dict) else None + return str(value) if value else None + if stripped.startswith("---"): + end = stripped.find("\n---", 3) + head = stripped[:end] if end > 0 else stripped + match = re.search(YAML_FIELD.format(key=key), head, re.MULTILINE) + return match.group(1).strip() if match else None + return None + + +def git_dates(rel_path): + """(author date, committer date) of the last commit touching the file.""" + result = subprocess.run( + ["git", "-c", "core.quotepath=false", "log", "-1", "--format=%aI\t%cI", + "--", rel_path], + cwd=REPO, + capture_output=True, + text=True, + ) + parts = result.stdout.strip().split("\t") + return tuple(to_instant(p) for p in parts) if len(parts) == 2 else (None, None) + + +def check_against_source(entries, sample_size): + pages = hugo_pages() + candidates = sorted( + (loc, lastmod) for loc, lastmod in entries + if urlparse(loc).path in pages + ) + print(f"\nSitemap URLs matching a regular page: {len(candidates)}") + if not candidates: + print("No sitemap URL maps to a page: `hugo list all` output not usable") + return False + if sample_size and sample_size < len(candidates): + candidates = random.Random(0).sample(candidates, sample_size) + print(f"Checking a seeded random sample of {len(candidates)}") + + mismatches = [] + for loc, lastmod in candidates: + # Mirrors the Hugo `[frontmatter] lastmod` order: lastmod, then Git, + # then date (files overlaid from a build artifact have no Git history). + rel = pages[urlparse(loc).path] + explicit = front_matter_field(REPO / rel, "lastmod") + if explicit: + expected_raw = explicit + alternatives = [to_instant(explicit)] + else: + author, committer = git_dates(rel) + if author: + expected_raw = author.isoformat() + alternatives = [author, committer] + else: + expected_raw = front_matter_field(REPO / rel, "date") + alternatives = [to_instant(expected_raw)] + alternatives = [d for d in alternatives if d] + actual = to_instant(lastmod) + if actual and any(abs((actual - d).total_seconds()) <= 1 for d in alternatives): + continue + mismatches.append((loc, lastmod, expected_raw, rel)) + + print(f"Mismatches: {len(mismatches)} of {len(candidates)} checked") + for loc, lastmod, expected_raw, rel in mismatches: + print(f" {loc}\n sitemap={lastmod} expected={expected_raw} source={rel}") + return not mismatches + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("sitemap", nargs="?", default="public/sitemap.xml") + parser.add_argument("--max-missing", type=int, default=5, + help="tolerated URLs without lastmod (default: 5)") + parser.add_argument("--max-bulk-share", type=float, default=0.5, + help="max share of URLs sharing one timestamp (default: 0.5)") + parser.add_argument("--compare-source", action="store_true", + help="also rebuild expected dates from the repository") + parser.add_argument("--sample", type=int, default=300, + help="pages to compare, 0 for all (default: 300)") + args = parser.parse_args() + + entries = load_sitemap(args.sitemap) + ok = check_structure(entries, args.max_missing, args.max_bulk_share) + if args.compare_source: + ok = check_against_source(entries, args.sample) and ok + print("\nOK" if ok else "\nFAILED") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/generated_lastmod.py b/scripts/generated_lastmod.py new file mode 100644 index 00000000000..54027969e34 --- /dev/null +++ b/scripts/generated_lastmod.py @@ -0,0 +1,73 @@ +"""Stable `lastmod` dates for generated JSON-front-matter content. + +Hugo prefers an explicit `lastmod` front-matter key over the Git commit date +(see `[frontmatter]` in config/_default/config.toml). Generators that rewrite +their whole content directory on every run therefore have to carry the date +forward themselves: an entry keeps its previous `lastmod` while its content is +unchanged, and gets today's date when the content differs. +""" + +import json +import subprocess +from datetime import datetime, timezone +from pathlib import Path + + +def today_utc() -> str: + return datetime.now(timezone.utc).strftime('%Y-%m-%d') + + +def load_previous(directory) -> dict: + """Read every generated `*.md` JSON file in `directory`, keyed by filename. + + `_index.md` is hand-written, not generated, and files that do not parse are + skipped so a stray file cannot abort a run. + """ + previous = {} + directory = Path(directory) + if not directory.is_dir(): + return previous + for path in sorted(directory.glob('*.md')): + if path.name == '_index.md': + continue + try: + previous[path.name] = json.loads(path.read_text(encoding='utf-8')) + except (OSError, ValueError): + continue + return previous + + +def _content_text(entry: dict) -> str: + """Serialized form of an entry, ignoring its `lastmod`. + + Compared as text rather than as dicts because curated resource rows carry + pandas NaN values, and `nan != nan` would mark every row as changed. + """ + without_lastmod = {k: v for k, v in entry.items() if k != 'lastmod'} + return json.dumps(without_lastmod, sort_keys=True, ensure_ascii=False) + + +def resolve_lastmod(new_entry: dict, previous_entry, bootstrap) -> str: + """Return the `YYYY-MM-DD` (UTC) date to record for `new_entry`. + + Unchanged content keeps the previous date; if the previous file predates + `lastmod`, `bootstrap()` supplies a date (today when it returns None). + """ + if previous_entry is None: + return today_utc() + if _content_text(previous_entry) != _content_text(new_entry): + return today_utc() + return previous_entry.get('lastmod') or bootstrap() or today_utc() + + +def git_commit_date(path) -> str: + """Date of the last commit touching `path`, or None if unavailable.""" + path = Path(path) + try: + result = subprocess.run( + ['git', 'log', '-1', '--format=%cs', '--', path.name], + cwd=str(path.parent), capture_output=True, text=True, check=True, + ) + except (OSError, subprocess.CalledProcessError): + return None + return result.stdout.strip() or None From 77c7b2be12ede6d2b4bcf200bfa4914a5e5aebc1 Mon Sep 17 00:00:00 2001 From: richarddushime Date: Thu, 3 Sep 2026 00:04:18 +0200 Subject: [PATCH 2/2] fix:silent fallback and batched gitlookup --- .github/workflows/data-processing.yml | 19 ++++++++++-- config/_default/config.toml | 24 ++++++++++----- content/glossary/_create_glossaries.py | 9 ++++-- scripts/check_sitemap_lastmod.py | 22 +++++++++++--- scripts/generated_lastmod.py | 41 +++++++++++++++++++++----- 5 files changed, 91 insertions(+), 24 deletions(-) diff --git a/.github/workflows/data-processing.yml b/.github/workflows/data-processing.yml index bc3feb90b64..8b45c49706f 100644 --- a/.github/workflows/data-processing.yml +++ b/.github/workflows/data-processing.yml @@ -232,17 +232,32 @@ jobs: # Fetch the previous generated curated resources from the build-resources # branch, so unchanged entries keep their lastmod date. # If the branch or the extraction is unavailable, the directory stays - # absent and resource.py falls back to the seed copy in the checkout. + # absent and resource.py falls back to the seed copy in the checkout -- + # which silently re-dates every drifted resource on every run, so the + # step logs the file count to make that fallback visible in the log. #======================================== - name: Fetch previous curated resources state id: previous-curated-resources continue-on-error: true run: | mkdir -p /tmp/previous-generated - git fetch origin build-resources || true + # Explicit refspec: actions/checkout may configure a narrow + # remote.origin.fetch, in which case a bare `git fetch origin + # build-resources` updates FETCH_HEAD without creating the + # origin/build-resources tracking ref that git archive needs. + git fetch origin +refs/heads/build-resources:refs/remotes/origin/build-resources || true git archive origin/build-resources content/curated_resources \ | tar -x -C /tmp/previous-generated || true + PREV_DIR=/tmp/previous-generated/content/curated_resources + if [ -d "$PREV_DIR" ]; then + echo "✅ Previous curated resources: $(find "$PREV_DIR" -name '*.md' | wc -l) files" + else + echo "⚠️ build-resources state unavailable - falling back to the" + echo " seed copy in this checkout. Resources that have drifted" + echo " from the seed will be stamped with today's date." + fi + #======================================== # Process and organize curated resources data #======================================== diff --git a/config/_default/config.toml b/config/_default/config.toml index 1fcca618438..b74ce656e62 100644 --- a/config/_default/config.toml +++ b/config/_default/config.toml @@ -26,14 +26,6 @@ theme = "academic" # Get last modified date for content from Git? enableGitInfo = true -# An explicit `lastmod` in front matter wins over the Git commit date. Generated -# collections (glossary, curated resources) carry their own `lastmod`, set by -# their generators only when an entry's content changes; hand-maintained pages -# without one fall back to Git. Requires `core.quotepath=false` in the build -# checkout so Git reports non-ASCII filenames unquoted (see deploy.yaml). -[frontmatter] -lastmod = ["lastmod", ":git", "date", "publishDate"] - # Enable generation of robots.txt file enableRobotsTXT = true @@ -121,3 +113,19 @@ ignoreFiles = ["\\.ipynb$", ".ipynb_checkpoints$", "\\.Rmd$", "\\.Rmarkdown$", " [build] no404check = true timeout = 60 + +# Date resolution order for front matter. +# +# NOTE: keep this table at the end of the file. A TOML table header captures +# every bare key that follows it, so placing this higher up would silently +# demote the top-level settings below it (enableRobotsTXT, removePathAccents, +# summaryLength, ...) into `frontmatter.*`, where Hugo ignores them. +# +# An explicit `lastmod` in front matter wins over the Git commit date (Hugo's +# default puts `:git` first). Generated collections (glossary, curated +# resources) carry their own `lastmod`, set by their generators only when an +# entry's content changes; hand-maintained pages without one fall back to Git. +# Requires `core.quotepath=false` in the build checkout so Git reports +# non-ASCII filenames unquoted (see deploy.yaml). +[frontmatter] + lastmod = ["lastmod", ":git", "date", "publishDate"] diff --git a/content/glossary/_create_glossaries.py b/content/glossary/_create_glossaries.py index 260871ee7af..c0e17714262 100755 --- a/content/glossary/_create_glossaries.py +++ b/content/glossary/_create_glossaries.py @@ -11,7 +11,7 @@ script_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, str(Path(script_dir).resolve().parents[1] / 'scripts')) -from generated_lastmod import load_previous, resolve_lastmod, git_commit_date +from generated_lastmod import load_previous, resolve_lastmod, git_commit_dates language_map = { 'EN': 'english', 'AR': 'arabic', @@ -296,8 +296,11 @@ def clean_filename(title, max_length=200): language_dir = os.path.join(script_dir, language_name) # Read the previous entries before deleting them, so each term can keep its - # `lastmod` for as long as its content is unchanged. + # `lastmod` for as long as its content is unchanged. Commit dates are + # collected in one pass up front; they only bootstrap entries written + # before `lastmod` existed. previous_entries = load_previous(language_dir) + commit_dates = git_commit_dates(language_dir) # Remove existing glossary entry files to ensure deleted entries don't persist # Preserve _index.md files as they are not regenerated @@ -318,7 +321,7 @@ def clean_filename(title, max_length=200): entry["lastmod"] = resolve_lastmod( entry, previous_entries.get(file_name + ".md"), - lambda: git_commit_date(file_path), + lambda: commit_dates.get(file_name + ".md"), ) with open(file_path, 'w', encoding='utf-8') as f: diff --git a/scripts/check_sitemap_lastmod.py b/scripts/check_sitemap_lastmod.py index 28159521efb..7bbaa6d776e 100644 --- a/scripts/check_sitemap_lastmod.py +++ b/scripts/check_sitemap_lastmod.py @@ -9,6 +9,13 @@ every page being stamped with the build or checkout time. Smaller blocks are reported but tolerated, since one commit can legitimately touch a whole generated collection. + + Do not lower --max-bulk-share below 0.5 hoping to catch a stale block: the + two generated collections are 47% (curated_resources) and 41% (glossary) of + the site, and regenerating either in one commit produces exactly the same + shape as the bug. This check only catches a *site-wide* stamp; a single + collection frozen on a stale date is caught by the generators writing their + own `lastmod` and by --compare-source verifying it reaches the sitemap. * Source comparison (--compare-source): rebuilds the expected lastmod for each page from the repository -- explicit front-matter `lastmod`, else the Git commit date of the source file -- and reports disagreements. Needs `hugo` @@ -93,12 +100,15 @@ def check_structure(entries, max_missing, max_bulk_share): limit = max_bulk_share * total bulk = [(stamp, n) for stamp, n in counts.most_common() if n > limit] print(f"\nMost widely shared timestamps (fail above {max_bulk_share:.0%} of URLs):") - for stamp, n in counts.most_common(3): + # Every offender is printed, not just the leaders: a block ranked below the + # top few can still fail the check, and a silent failure is unactionable. + for stamp, n in dict.fromkeys(counts.most_common(3) + bulk): sections = Counter( section_of(loc) for loc, lastmod in entries if lastmod == stamp ) where = ", ".join(f"{name} ({c})" for name, c in sections.most_common(3)) - print(f" {stamp}: {n} URLs -- {where}") + flag = " <-- FAILS" if n > limit else "" + print(f" {stamp}: {n} URLs -- {where}{flag}") return len(missing) <= max_missing and not bulk @@ -166,8 +176,12 @@ def check_against_source(entries, sample_size): print("No sitemap URL maps to a page: `hugo list all` output not usable") return False if sample_size and sample_size < len(candidates): - candidates = random.Random(0).sample(candidates, sample_size) - print(f"Checking a seeded random sample of {len(candidates)}") + # Seeded on the date so a run is reproducible while the sample rotates + # between deploys: a fixed seed would check the same few hundred pages + # forever and never look at the rest of the site. + seed = datetime.now(timezone.utc).strftime("%Y-%m-%d") + candidates = random.Random(seed).sample(candidates, sample_size) + print(f"Checking a random sample of {len(candidates)} (seed {seed})") mismatches = [] for loc, lastmod in candidates: diff --git a/scripts/generated_lastmod.py b/scripts/generated_lastmod.py index 54027969e34..e226c78717c 100644 --- a/scripts/generated_lastmod.py +++ b/scripts/generated_lastmod.py @@ -60,14 +60,41 @@ def resolve_lastmod(new_entry: dict, previous_entry, bootstrap) -> str: return previous_entry.get('lastmod') or bootstrap() or today_utc() -def git_commit_date(path) -> str: - """Date of the last commit touching `path`, or None if unavailable.""" - path = Path(path) +# Prefix that cannot collide with a path line, so date and path lines in +# `git log --name-only` output can be told apart without guessing. +_DATE_PREFIX = 'commit-date:' + + +def git_commit_dates(directory) -> dict: + """Map filename -> `YYYY-MM-DD` of the last commit touching it. + + A single `git log` walk over the directory rather than one subprocess per + file: bootstrapping a fresh collection means ~1,300 lookups, measured at + ~10s of pure process spawning locally and worse on a CI runner, against + ~0.1s for the batched walk. `core.quotepath=false` is required here -- + without it Git escapes the non-ASCII glossary filenames and none of them + match, which is the same defect this whole mechanism exists to fix. + + Returns an empty map (callers fall back to today) if Git is unavailable. + """ + directory = Path(directory) + cwd = directory if directory.is_dir() else directory.parent try: result = subprocess.run( - ['git', 'log', '-1', '--format=%cs', '--', path.name], - cwd=str(path.parent), capture_output=True, text=True, check=True, + ['git', '-c', 'core.quotepath=false', 'log', + f'--format={_DATE_PREFIX}%cs', '--name-only', '--', '.'], + cwd=str(cwd), capture_output=True, text=True, check=True, ) except (OSError, subprocess.CalledProcessError): - return None - return result.stdout.strip() or None + return {} + + dates = {} + current = None + for line in result.stdout.splitlines(): + if line.startswith(_DATE_PREFIX): + current = line[len(_DATE_PREFIX):].strip() or None + elif line.strip() and current: + # git log walks newest-first, so the first sighting of a path is + # its most recent commit; later sightings are older history. + dates.setdefault(Path(line.strip()).name, current) + return dates