diff --git a/.github/scripts/check_skill_staleness.py b/.github/scripts/check_skill_staleness.py new file mode 100644 index 0000000..b3df2a1 --- /dev/null +++ b/.github/scripts/check_skill_staleness.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +"""Detect stale skills and emit a freshness-reminder report. + +A skill is "stale" when its directory has had no git commit for at least +``--max-age-days`` (default 60 days ~= 2 months). Freshness is measured from the +last commit that touched ``skills//`` — a real, always-present signal, +unlike CHANGELOG dates which are inconsistent across skills. + +The script reads each ``skills//SKILL.md`` frontmatter for the ``author`` +field (an internal alias, surfaced in the issue as plain text for a maintainer +to route to — it is NOT a GitHub login and is never mentioned or assigned) and +prints: + + * a human-readable summary to stderr, and + * a JSON array of stale skills to stdout (or to ``--output`` / the file named + by the ``GITHUB_OUTPUT``-style ``--output`` flag) for the workflow to consume. + +Each stale entry has: ``id``, ``authors`` (list), ``last_commit_iso``, +``last_commit_sha``, ``age_days``. + +Exit code is always 0 (a stale skill is not a failure); ``--fail-on-stale`` makes +it exit 1 when any skill is stale, for callers that want a hard gate. +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from dataclasses import dataclass, asdict +from datetime import datetime, timezone +from pathlib import Path + + +FRONTMATTER_AUTHOR_RE = re.compile(r"^\s*author\s*:\s*(.+?)\s*$", re.MULTILINE) + + +@dataclass +class StaleSkill: + id: str + authors: list[str] + last_commit_iso: str + last_commit_sha: str + age_days: int + + +ISSUE_TITLE_PREFIX = "[skill-freshness] Verify" +WORKFLOW_PATH = ".github/workflows/skill-staleness-reminder.yml" + + +def issue_title(skill: StaleSkill) -> str: + return f"{ISSUE_TITLE_PREFIX} `{skill.id}` is still current" + + +def issue_body(skill: StaleSkill, max_age_days: int) -> str: + # The frontmatter `author` is the skill's declared owner, but in this repo it + # is an internal alias, NOT a GitHub login (e.g. "hokang" -> GitHub user + # "howard-m-k"). So it is surfaced as plain text for a maintainer to route to, + # never @-mentioned or used as an assignee. + listed_author = ", ".join(f"`{a}`" for a in skill.authors) or ( + "_none listed in SKILL.md frontmatter_" + ) + return "\n".join( + [ + f"The **`{skill.id}`** skill has not been updated in " + f"**{skill.age_days} days** (last commit `{skill.last_commit_iso}`), " + f"which exceeds the {max_age_days}-day freshness window.", + "", + f"**Listed author (SKILL.md frontmatter):** {listed_author}", + "", + "A maintainer has been assigned to triage this. Please confirm the skill " + "is still current, or reassign to the author above (or whoever now owns the " + "skill) to verify:", + "", + "- [ ] AWS APIs, CLI commands, and console paths referenced are still valid", + "- [ ] Thresholds, defaults, and version numbers still reflect current AWS behavior", + "- [ ] Documentation links in the skill and its `references/` still resolve", + "- [ ] Evals still pass (`.skilleval.yaml` / `evals/`)", + "", + "**If it's still accurate:** bump the patch version in `SKILL.md` frontmatter " + "and add a `CHANGELOG.md` line noting the freshness review (a commit touching " + f"`skills/{skill.id}/` resets the clock and closes this reminder next cycle).", + "", + "**If it needs changes:** open a PR with the updates.", + "", + f"_Generated automatically by the Skill Staleness Reminder workflow " + f"(`{WORKFLOW_PATH}`). It reopens on the next scheduled run if the skill " + "is still untouched._", + ] + ) + + +def _repo_root() -> Path: + """Resolve the repository root from this script's location (.github/scripts/).""" + return Path(__file__).resolve().parents[2] + + +def _parse_authors(skill_md: Path) -> list[str]: + """Extract author handle(s) from a SKILL.md frontmatter block. + + Only the frontmatter (between the first pair of ``---`` fences) is scanned so + a stray ``author:`` in prose can't be picked up. Handles a single value or a + comma-separated list, strips surrounding quotes, and drops a leading ``@``. + """ + text = skill_md.read_text(encoding="utf-8") + if not text.startswith("---"): + return [] + end = text.find("---", 3) + if end == -1: + return [] + frontmatter = text[3:end] + + m = FRONTMATTER_AUTHOR_RE.search(frontmatter) + if not m: + return [] + raw = m.group(1).strip().strip('"').strip("'") + authors = [] + for part in raw.split(","): + handle = part.strip().lstrip("@").strip() + if handle: + authors.append(handle) + return authors + + +def _last_commit(repo_root: Path, rel_dir: str) -> tuple[datetime, str] | None: + """Return (commit datetime UTC, short sha) of the last commit touching rel_dir. + + Returns None when the path has no commit history (e.g. a brand-new, + uncommitted skill) — such skills are treated as fresh and skipped. + """ + result = subprocess.run( + ["git", "log", "-1", "--format=%cI%x09%h", "--", rel_dir], + cwd=repo_root, + capture_output=True, + text=True, + check=True, + ) + line = result.stdout.strip() + if not line: + return None + iso, _, sha = line.partition("\t") + # %cI is strict ISO 8601 with offset; normalize to aware UTC. + dt = datetime.fromisoformat(iso).astimezone(timezone.utc) + return dt, sha + + +def find_stale_skills(repo_root: Path, max_age_days: int, now: datetime) -> list[StaleSkill]: + skills_dir = repo_root / "skills" + stale: list[StaleSkill] = [] + if not skills_dir.is_dir(): + return stale + + for skill_path in sorted(p for p in skills_dir.iterdir() if p.is_dir()): + skill_md = skill_path / "SKILL.md" + if not skill_md.is_file(): + continue # not a skill directory + + rel_dir = f"skills/{skill_path.name}" + last = _last_commit(repo_root, rel_dir) + if last is None: + continue # uncommitted / no history -> treat as fresh + + last_dt, sha = last + age_days = (now - last_dt).days + if age_days >= max_age_days: + stale.append( + StaleSkill( + id=skill_path.name, + authors=_parse_authors(skill_md), + last_commit_iso=last_dt.date().isoformat(), + last_commit_sha=sha, + age_days=age_days, + ) + ) + return stale + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--max-age-days", + type=int, + default=60, + help="Age threshold in days; skills untouched this long are stale (default: 60 ~= 2 months).", + ) + parser.add_argument( + "--output", + type=Path, + default=None, + help="Write the JSON array to this file (in addition to stdout).", + ) + parser.add_argument( + "--fail-on-stale", + action="store_true", + help="Exit 1 if any skill is stale (default: always exit 0).", + ) + parser.add_argument( + "--render-dir", + type=Path, + default=None, + help=( + "Write ready-to-post issue files into this directory: one " + ".title and .body per stale skill, plus an " + "index.txt listing the stale skill ids. Lets the workflow post " + "issues without fragile shell heredocs." + ), + ) + args = parser.parse_args(argv) + + repo_root = _repo_root() + now = datetime.now(timezone.utc) + stale = find_stale_skills(repo_root, args.max_age_days, now) + + payload = json.dumps([asdict(s) for s in stale], indent=2) + print(payload) + if args.output: + args.output.write_text(payload + "\n", encoding="utf-8") + + if args.render_dir is not None: + args.render_dir.mkdir(parents=True, exist_ok=True) + for s in stale: + (args.render_dir / f"{s.id}.title").write_text( + issue_title(s), encoding="utf-8" + ) + (args.render_dir / f"{s.id}.body").write_text( + issue_body(s, args.max_age_days) + "\n", encoding="utf-8" + ) + (args.render_dir / "index.txt").write_text( + "".join(f"{s.id}\n" for s in stale), encoding="utf-8" + ) + + if stale: + print( + f"\n{len(stale)} skill(s) not updated in >= {args.max_age_days} days:", + file=sys.stderr, + ) + for s in stale: + who = ", ".join(f"@{a}" for a in s.authors) or "(no author in frontmatter)" + print( + f" - {s.id}: last commit {s.last_commit_iso} " + f"({s.age_days}d ago, {s.last_commit_sha}) -> {who}", + file=sys.stderr, + ) + else: + print( + f"\nAll skills updated within the last {args.max_age_days} days.", + file=sys.stderr, + ) + + if args.fail_on_stale and stale: + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/skill-staleness-reminder.yml b/.github/workflows/skill-staleness-reminder.yml new file mode 100644 index 0000000..2deca24 --- /dev/null +++ b/.github/workflows/skill-staleness-reminder.yml @@ -0,0 +1,145 @@ +name: Skill Staleness Reminder + +# Reminds skill authors to verify their skill is still fresh when its directory +# has had no commit for >= 60 days (~2 months). +# +# Runs on the 1st of every other month via a plain cron expression. `*/2` on the +# month field steps from January, so scheduled runs land on Jan, Mar, May, Jul, +# Sep, and Nov. workflow_dispatch allows manual runs on demand. + +on: + schedule: + # 15:00 UTC on the 1st of every other (odd) month. + - cron: "0 15 1 */2 *" + workflow_dispatch: + inputs: + max_age_days: + description: "Staleness threshold in days" + required: false + default: "60" + dry_run: + description: "Log what would happen without creating/commenting on issues" + type: boolean + required: false + default: true + +permissions: + contents: read + issues: write + +concurrency: + group: skill-staleness-reminder + cancel-in-progress: false + +jobs: + remind: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + # Full history so `git log` per skill directory can find the last commit. + fetch-depth: 0 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Detect stale skills and render issue content + id: detect + run: | + MAX_AGE="${{ github.event.inputs.max_age_days }}" + MAX_AGE="${MAX_AGE:-60}" + python3 .github/scripts/check_skill_staleness.py \ + --max-age-days "$MAX_AGE" \ + --output stale.json \ + --render-dir reminders + echo "count=$(python3 -c 'import json;print(len(json.load(open("stale.json"))))')" >> "$GITHUB_OUTPUT" + + - name: Open or update reminder issues + if: steps.detect.outputs.count != '0' + env: + GH_TOKEN: ${{ github.token }} + # Scheduled runs always act. Manual runs honor the dry_run input + # (default true), so a workflow_dispatch on a branch is safe to trigger. + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run || 'false' }} + run: | + if [ "$DRY_RUN" = "true" ]; then + echo "::notice::DRY RUN — no issues will be created or commented on." + fi + + # Ensure the reminder label exists (idempotent). Skipped in dry run. + if [ "$DRY_RUN" != "true" ]; then + gh label create skill-freshness \ + --description "Periodic reminder to verify a skill is still current" \ + --color BFD4F2 2>/dev/null || true + fi + + # Build the triage pool: collaborators with admin or maintain permission. + # A random one is assigned per issue to spread triage load. They can then + # reassign to the actual skill author (see body). The frontmatter author + # is NOT used as an assignee — it is an internal alias, not a GitHub login. + gh api "repos/${GITHUB_REPOSITORY}/collaborators?per_page=100" \ + --jq '.[] | select(.permissions.admin or .permissions.maintain) | .login' \ + > maintainers.txt + pool_size=$(wc -l < maintainers.txt | tr -d ' ') + echo "Triage pool: $pool_size maintainer(s)." + + # The script rendered one .title / .body per stale skill and an + # index.txt of ids — no shell heredocs, no leading-whitespace issues. + while IFS= read -r id; do + [ -n "$id" ] || continue + title="$(cat "reminders/$id.title")" + body_file="reminders/$id.body" + echo "Processing stale skill: $id" + + # Dedup: reuse an open issue with the same title if one exists. + # Title is matched exactly via a standalone jq using env.TITLE, which + # avoids depending on `gh` forwarding a --arg to its embedded jq. + existing=$(gh issue list --state open --label skill-freshness \ + --json number,title \ + | TITLE="$title" jq -r '.[] | select(.title == env.TITLE) | .number' \ + | head -n1) + + if [ -n "$existing" ]; then + # Already tracked — just add a fresh comment; leave the existing + # assignee in place so an in-progress triage isn't disrupted. + if [ "$DRY_RUN" = "true" ]; then + echo " [dry-run] would comment on existing issue #$existing" + else + echo " updating existing issue #$existing" + gh issue comment "$existing" --body-file "$body_file" + fi + continue + fi + + # New issue: pick a random maintainer from the pool (if any). + # Python's random.choice is used instead of `shuf` for portability. + assignee="" + if [ "$pool_size" -gt 0 ]; then + assignee=$(python3 -c "import random,sys; lines=[l.strip() for l in open('maintainers.txt') if l.strip()]; print(random.choice(lines))") + fi + + if [ "$DRY_RUN" = "true" ]; then + echo " [dry-run] would open new issue, assignee=${assignee:-}" + continue + fi + + if [ -n "$assignee" ]; then + echo " opening new issue, assigning to $assignee" + gh issue create --title "$title" --body-file "$body_file" \ + --label skill-freshness --assignee "$assignee" + else + echo " opening new issue (no assignable maintainer found)" + gh issue create --title "$title" --body-file "$body_file" \ + --label skill-freshness + fi + done < reminders/index.txt + + - name: Summary + run: | + echo "Stale skills this run: ${{ steps.detect.outputs.count }}" >> "$GITHUB_STEP_SUMMARY" + if [ -f stale.json ]; then + echo '```json' >> "$GITHUB_STEP_SUMMARY" + cat stale.json >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + fi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9f6e16b..d6c047c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,6 +63,12 @@ Start with the SKILL.md, as well as `references/` and and `assets/` if needed. See [Contributing via Pull Requests](CONTRIBUTING.md#contributing-via-pull-requests) +### Keeping a Skill Fresh + +Skills reference AWS APIs, thresholds, and documentation that drift over time. A scheduled workflow ([`.github/workflows/skill-staleness-reminder.yml`](.github/workflows/skill-staleness-reminder.yml)) runs on the 1st of every other month and, for any skill whose `skills//` directory has had no commit in the last 60 days, opens a `skill-freshness` issue. Each new issue is assigned to a random repository maintainer to triage, and the issue body lists the skill's `SKILL.md` frontmatter `author` so the maintainer can reassign to the skill owner to verify. + +If an issue lands with you, verify the skill against the checklist in it. When it's still accurate, bump the patch version in `SKILL.md` and add a `CHANGELOG.md` line noting the review — any commit touching the skill directory resets the clock and stops the reminder next cycle. + ## Reporting Bugs/Feature Requests