Make sitemap lastmod accurate and stable

.github/workflows/data-processing.yml

@@ -60,6 +60,10 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v6
         # Checkout the repository code to the runner environment
+        with:
+          # Full history: the generators read per-file commit dates to bootstrap
+          # `lastmod`, which a shallow clone would report as HEAD for every file.
+          fetch-depth: 0
 
       #======================
       # Workflow Configuration
@@ -224,12 +228,44 @@ jobs:
         continue-on-error: true
         run: python3 scripts/build_get_involved.py
 
+      #========================================
+      # Fetch the previous generated curated resources from the build-resources
+      # branch, so unchanged entries keep their lastmod date.
+      # If the branch or the extraction is unavailable, the directory stays
+      # absent and resource.py falls back to the seed copy in the checkout --
+      # which silently re-dates every drifted resource on every run, so the
+      # step logs the file count to make that fallback visible in the log.
+      #========================================
+      - name: Fetch previous curated resources state
+        id: previous-curated-resources
+        continue-on-error: true
+        run: |
+          mkdir -p /tmp/previous-generated
+          # Explicit refspec: actions/checkout may configure a narrow
+          # remote.origin.fetch, in which case a bare `git fetch origin
+          # build-resources` updates FETCH_HEAD without creating the
+          # origin/build-resources tracking ref that git archive needs.
+          git fetch origin +refs/heads/build-resources:refs/remotes/origin/build-resources || true
+          git archive origin/build-resources content/curated_resources \
+            | tar -x -C /tmp/previous-generated || true
+
+          PREV_DIR=/tmp/previous-generated/content/curated_resources
+          if [ -d "$PREV_DIR" ]; then
+            echo "✅ Previous curated resources: $(find "$PREV_DIR" -name '*.md' | wc -l) files"
+          else
+            echo "⚠️ build-resources state unavailable - falling back to the"
+            echo "   seed copy in this checkout. Resources that have drifted"
+            echo "   from the seed will be stamped with today's date."
+          fi
+
       #========================================
       # Process and organize curated resources data
       #========================================
       - name: Run Curated Resources script
         id: curated-resources
         continue-on-error: true  # Continue even if this step fails
+        env:
+          CURATED_PREVIOUS_DIR: /tmp/previous-generated/content/curated_resources
         run: python3 content/resources/resource.py
         # Execute the curated resources script that processes and organizes resource data
 

.github/workflows/deploy.yaml

@@ -41,6 +41,12 @@ jobs:
         with:
           fetch-depth: 0
 
+      # Hugo derives page dates from `git log --name-only`, and Git quotes
+      # non-ASCII paths (glossary terms, accented resource titles) unless this
+      # is off. Quoted paths never match, leaving those pages without a date.
+      - name: Let Git report non-ASCII paths unquoted
+        run: git config core.quotepath false
+
       # =======================
       # Data Artifact Retrieval
       # =======================
@@ -208,6 +214,17 @@ jobs:
         env:
           HUGO_ENV: production
 
+      # =======================
+      # Sitemap Quality Check
+      # =======================
+      # Reports pages missing a <lastmod>, a timestamp stamped on most of the
+      # site, and a sample of pages whose date disagrees with their source
+      # (front-matter lastmod or Git). Warns only: a stale sitemap date must
+      # not block a deployment.
+      - name: Check sitemap lastmod
+        continue-on-error: true
+        run: python3 scripts/check_sitemap_lastmod.py public/sitemap.xml --compare-source --sample 200
+
       # =======================
       # Deployment Artifact
       #========================================

.github/workflows/staging-aggregate.yaml

@@ -263,6 +263,12 @@ jobs:
         uses: actions/checkout@v6
         with:
           ref: ${{ needs.aggregate-prs.outputs.aggregated-branch || 'main' }}
+          fetch-depth: 0  # full history so Hugo's Git-derived dates match production
+
+      # See deploy.yaml: Git quotes non-ASCII paths unless this is off, and
+      # Hugo then finds no date for them.
+      - name: Let Git report non-ASCII paths unquoted
+        run: git config core.quotepath false
 
       - name: Configure Git
         run: |

config/_default/config.toml

@@ -113,3 +113,19 @@ ignoreFiles = ["\\.ipynb$", ".ipynb_checkpoints$", "\\.Rmd$", "\\.Rmarkdown$", "
 [build]
   no404check = true
   timeout = 60
+
+# Date resolution order for front matter.
+#
+# NOTE: keep this table at the end of the file. A TOML table header captures
+# every bare key that follows it, so placing this higher up would silently
+# demote the top-level settings below it (enableRobotsTXT, removePathAccents,
+# summaryLength, ...) into `frontmatter.*`, where Hugo ignores them.
+#
+# An explicit `lastmod` in front matter wins over the Git commit date (Hugo's
+# default puts `:git` first). Generated collections (glossary, curated
+# resources) carry their own `lastmod`, set by their generators only when an
+# entry's content changes; hand-maintained pages without one fall back to Git.
+# Requires `core.quotepath=false` in the build checkout so Git reports
+# non-ASCII filenames unquoted (see deploy.yaml).
+[frontmatter]
+  lastmod = ["lastmod", ":git", "date", "publishDate"]

content/glossary/_create_glossaries.py

@@ -3,10 +3,15 @@
 import json
 import pandas as pd
 import os
+import sys
 from io import StringIO
+from pathlib import Path
 from pypinyin import lazy_pinyin, Style
 
 script_dir = os.path.dirname(os.path.abspath(__file__))
+
+sys.path.insert(0, str(Path(script_dir).resolve().parents[1] / 'scripts'))
+from generated_lastmod import load_previous, resolve_lastmod, git_commit_dates
 language_map = {
     'EN': 'english',
     'AR': 'arabic',
@@ -289,6 +294,14 @@ def clean_filename(title, max_length=200):
 # Create markdown files
 for language_name, entries in formatted_data.items():
     language_dir = os.path.join(script_dir, language_name)
+
+    # Read the previous entries before deleting them, so each term can keep its
+    # `lastmod` for as long as its content is unchanged. Commit dates are
+    # collected in one pass up front; they only bootstrap entries written
+    # before `lastmod` existed.
+    previous_entries = load_previous(language_dir)
+    commit_dates = git_commit_dates(language_dir)
+
     # Remove existing glossary entry files to ensure deleted entries don't persist
     # Preserve _index.md files as they are not regenerated
     if os.path.exists(language_dir):
@@ -304,7 +317,13 @@ def clean_filename(title, max_length=200):
     for entry in entries:
         file_name = clean_filename(entry['title'])
         file_path = os.path.join(language_dir, file_name + ".md")
-
+
+        entry["lastmod"] = resolve_lastmod(
+            entry,
+            previous_entries.get(file_name + ".md"),
+            lambda: commit_dates.get(file_name + ".md"),
+        )
+
         with open(file_path, 'w', encoding='utf-8') as f:
             json.dump(entry, f, ensure_ascii=False, indent=4)
 

content/resources/resource.py

@@ -17,10 +17,44 @@
 """
 
 import json
-import pandas as pd
+import os
 import re
+import sys
 from pathlib import Path
 
+import pandas as pd
+from dateutil import parser as date_parser
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2] / 'scripts'))
+from generated_lastmod import load_previous, resolve_lastmod
+
+
+def previous_directory(fpath: Path) -> Path:
+    """Where the previous run's files are.
+
+    The daily workflow points CURATED_PREVIOUS_DIR at a checkout of the
+    `build-resources` branch, which holds the last generated state. Without it
+    (local runs, or a missing branch) the checkout's own seed copy is used.
+    """
+    env_dir = os.environ.get('CURATED_PREVIOUS_DIR')
+    if env_dir and Path(env_dir).is_dir():
+        return Path(env_dir)
+    return fpath
+
+
+def submission_date(timestamp) -> str:
+    """Sheet submission date as YYYY-MM-DD, or None if it cannot be parsed.
+
+    Used as the `lastmod` for entries carried over from before this key
+    existed. The sheet mixes ISO and US month-first formats.
+    """
+    if not timestamp or not isinstance(timestamp, str):
+        return None
+    try:
+        return date_parser.parse(timestamp).strftime('%Y-%m-%d')
+    except (ValueError, OverflowError):
+        return None
+
 
 ## Function definition
 
@@ -87,6 +121,10 @@ def convert_row_to_file(df, fpath):
     If there are duplicates, an index is appended to the filename.
     """
 
+    # Read the previous run's files before deleting them, so each resource can
+    # keep its `lastmod` for as long as its content is unchanged.
+    previous = load_previous(previous_directory(fpath))
+
     # Replace the generated collection rather than leaving files for rows that
     # have been removed from the source sheet.
     for path in fpath.iterdir():
@@ -108,8 +146,14 @@ def convert_row_to_file(df, fpath):
         else:
             filename_counts[filename] = 1
             filename_md = fpath / f"{filename}.md"
-
-        filename_md.write_text(json.dumps(row.to_dict(), indent=4))
+
+        entry = row.to_dict()
+        entry["lastmod"] = resolve_lastmod(
+            entry,
+            previous.get(filename_md.name),
+            lambda: submission_date(entry.get("timestamp")),
+        )
+        filename_md.write_text(json.dumps(entry, indent=4))
 
 
 # Import data and prettify it: