ci: agent-record measures which worker suits which work

docs/AGENT_WORK.md

@@ -101,6 +101,17 @@ PR creation. Fix the reported problem and retry finish. Neither helper merges or
 - A different worker than the author reviews the PR.
 - No auto-merge. Merging is the maintainer's (`CONTRIBUTING.md`).
 
+## Which worker suits which work
+
+`scripts/agent-record <pr>` appends one line per finished agent PR to
+`${FIZZY_AGENT_LOG:-~/.cache/fizzy-agent-record.txt}` (skipping a PR already there) and prints it:
+`PR | worker | rounds-to-green | review findings | CI reds`. It reads GitHub through `gh` only. The worker is the
+claimed issue's `running:*` label (found by the PR's `Part of fizzyedit/work#N` line); rounds-to-green is the
+commits pushed after the PR left draft or first went red, whichever was first; review findings are
+`<review comments>c/<review threads>t`; CI reds are failed or timed-out check runs across every commit of the
+head's history. `agent-record --all <YYYY-MM-DD>` backfills merged PRs since then. `python3
+scripts/test-agent-record.py` checks it against saved `gh` JSON.
+
 ## Self-hosted runner
 
 On a dedicated Ubuntu 24.04 or newer machine with systemd (x86_64 or aarch64), run the

scripts/agent-record

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+"""Append one line per finished agent PR: "PR | worker | rounds-to-green | review findings | CI reds".
+
+    agent-record <pr>            record one PR (printed, appended unless already recorded)
+    agent-record --all <since>   backfill merged PRs from <since> (YYYY-MM-DD) that reference fizzyedit/work
+
+Reads GitHub through `gh` only and parses nothing but labels and the "Part of fizzyedit/work#N"
+line. The log is ${FIZZY_AGENT_LOG:-~/.cache/fizzy-agent-record.txt}; the repo ${FIZZY_REPO:-fizzyedit/fizzy}.
+"""
+
+import json
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+REPO = os.environ.get("FIZZY_REPO", "fizzyedit/fizzy")
+WORK = "fizzyedit/work"
+FAILED = {"failure", "timed_out"}
+THREADS = """query($o:String!,$r:String!,$n:Int!,$c:String){repository(owner:$o,name:$r){
+pullRequest(number:$n){reviewThreads(first:100,after:$c){nodes{id}
+pageInfo{hasNextPage endCursor}}}}}"""
+
+
+def log_path():
+    return Path(os.environ.get("FIZZY_AGENT_LOG") or "~/.cache/fizzy-agent-record.txt").expanduser()
+
+
+class Skip(Exception):
+    """A PR that cannot be recorded now; the message says why."""
+
+
+def gh(*args):
+    out = subprocess.run(["gh", *args], capture_output=True, text=True)
+    if out.returncode != 0:
+        raise Skip(f"gh {' '.join(args)} failed: {out.stderr.strip()}")
+    if not out.stdout.strip():
+        raise Skip(f"gh {' '.join(args)} printed nothing")
+    return json.loads(out.stdout)
+
+
+def pages(path, *extra):
+    """A paginated list endpoint as one flat list of items."""
+    data = gh("api", "--paginate", "--slurp", *extra, path)
+    items = []
+    for page in data or []:
+        if isinstance(page, dict):
+            page = next((v for v in page.values() if isinstance(v, list)), [])
+        items.extend(page if isinstance(page, list) else [page])
+    return items
+
+
+def threads(pr):
+    owner, name = REPO.split("/")
+    count, cursor = 0, None
+    while True:
+        args = ["api", "graphql", "-f", f"query={THREADS}", "-f", f"o={owner}", "-f", f"r={name}", "-F", f"n={pr}"]
+        if cursor:
+            args += ["-f", f"c={cursor}"]
+        conn = gh(*args)["data"]["repository"]["pullRequest"]["reviewThreads"]
+        count += len(conn["nodes"])
+        if not conn["pageInfo"]["hasNextPage"]:
+            return count
+        cursor = conn["pageInfo"]["endCursor"]
+
+
+def measure(pr):
+    """The recorded line for a PR, or None when it does not reference fizzyedit/work."""
+    info = gh("api", f"repos/{REPO}/pulls/{pr}")
+    if not info.get("merged_at") and info.get("state") != "closed":
+        raise Skip("not merged or closed yet, so its numbers are not final")
+    ref = re.search(r"Part of fizzyedit/work#(\d+)", info.get("body") or "")
+    if not ref:
+        return None
+    issue = gh("api", f"repos/{WORK}/issues/{ref[1]}")
+    workers = sorted(l["name"][len("running:"):] for l in issue["labels"] if l["name"].startswith("running:"))
+    if not workers:
+        print(f"agent-record: PR {pr}: {WORK}#{ref[1]} has no running:* label, worker recorded as unknown", file=sys.stderr)
+
+    commits = pages(f"repos/{REPO}/pulls/{pr}/commits")
+    reds = {}
+    for commit in commits:
+        for run in pages(f"repos/{REPO}/commits/{commit['sha']}/check-runs"):
+            if run.get("conclusion") in FAILED:
+                reds[run["id"]] = run.get("completed_at") or ""
+
+    # A push is a commit; rounds count those after the PR first left draft or first went red.
+    events = pages(f"repos/{REPO}/issues/{pr}/timeline")
+    marks = [e["created_at"] for e in events if e.get("event") == "ready_for_review"]
+    marks += [t for t in reds.values() if t]
+    rounds = 0
+    if marks:
+        since = min(marks)
+        rounds = sum(1 for c in commits if c["commit"]["committer"]["date"] > since)
+
+    comments = len(pages(f"repos/{REPO}/pulls/{pr}/comments"))
+    findings = f"{comments}c/{threads(pr)}t"
+    return f"#{pr} | {'+'.join(workers) or 'unknown'} | {rounds} | {findings} | {len(reds)}"
+
+
+def recorded(path):
+    if not path.exists():
+        return {}
+    return {l.split(" | ", 1)[0]: l for l in path.read_text().splitlines()}
+
+
+def record(pr, path):
+    """The PR's line, measured and appended unless the log already holds it."""
+    held = recorded(path).get(f"#{pr}")
+    if held:
+        return held
+    line = measure(pr)
+    if line:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with path.open("a") as f:
+            f.write(line + "\n")
+    return line
+
+
+def main(argv):
+    path = log_path()
+    if len(argv) == 1 and argv[0].isdigit():
+        try:
+            line = record(int(argv[0]), path)
+        except Skip as why:
+            sys.exit(f"agent-record: PR {argv[0]} skipped: {why}")
+        if line is None:
+            sys.exit(f"agent-record: PR {argv[0]} does not say 'Part of {WORK}#N'")
+        print(line)
+    elif len(argv) == 2 and argv[0] == "--all":
+        query = f'repo:{REPO} is:pr is:merged merged:>={argv[1]} "{WORK}"'
+        try:
+            found = pages("search/issues", "--method", "GET", "-f", f"q={query}")
+        except Skip as why:
+            sys.exit(f"agent-record: search failed: {why}")
+        for item in found:
+            try:
+                line = record(item["number"], path)
+            except Skip as why:
+                print(f"agent-record: PR {item['number']} skipped: {why}", file=sys.stderr)
+                continue
+            if line:
+                print(line)
+    else:
+        sys.exit(__doc__)
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

scripts/test-agent-record.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+"""Checks scripts/agent-record against saved gh JSON: a fake `gh` answers each endpoint from a fixture."""
+
+import json
+import os
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+RECORD = Path(__file__).resolve().parent / "agent-record"
+
+FAKE_GH = """#!/usr/bin/env python3
+import os, sys
+args = [a for a in sys.argv[1:] if not a.startswith("-")]
+key = [a for a in args if a.startswith(("repos/", "search/")) or a == "graphql"][0].replace("/", "_")
+path = os.path.join(os.environ["FIXTURES"], key + ".json")
+if not os.path.exists(path):
+    sys.stderr.write("HTTP 404: " + key)
+    sys.exit(1)
+print(open(path).read())
+"""
+
+
+def sha(n):
+    return f"sha{n}"
+
+
+def fixtures():
+    c = lambda n, date: {"sha": sha(n), "commit": {"committer": {"date": date}}}
+    run = lambda i, conclusion, at: {"id": i, "conclusion": conclusion, "completed_at": at}
+    return {
+        "repos_fizzyedit_fizzy_pulls_7": {"body": "ci: thing\n\nPart of fizzyedit/work#14\n", "state": "closed", "merged_at": "2026-10-03T00:00:00Z"},
+        "repos_fizzyedit_fizzy_pulls_8": {"body": "no reference here", "state": "closed", "merged_at": "x"},
+        "repos_fizzyedit_fizzy_pulls_9": {"body": "Part of fizzyedit/work#14", "state": "open", "merged_at": None},
+        "repos_fizzyedit_fizzy_pulls_10": {"body": "Part of fizzyedit/work#99", "state": "closed", "merged_at": "x"},
+        "repos_fizzyedit_fizzy_pulls_11": {"body": "Part of fizzyedit/work#15", "state": "closed", "merged_at": "x"},
+        "repos_fizzyedit_fizzy_pulls_11_commits": [[]],
+        "repos_fizzyedit_fizzy_issues_11_timeline": [[]],
+        "repos_fizzyedit_fizzy_pulls_11_comments": [[]],
+        "repos_fizzyedit_work_issues_15": {"labels": [{"name": "agent"}]},
+        "repos_fizzyedit_work_issues_14": {"labels": [{"name": "agent"}, {"name": "running:codex"}]},
+        "repos_fizzyedit_fizzy_pulls_7_commits": [[
+            c(1, "2026-10-01T10:00:00Z"), c(2, "2026-10-01T11:00:00Z"),
+            c(3, "2026-10-02T09:00:00Z"), c(4, "2026-10-02T10:00:00Z")]],
+        "repos_fizzyedit_fizzy_commits_sha1_check-runs": [{"check_runs": [run(1, "success", "2026-10-01T10:05:00Z")]}],
+        "repos_fizzyedit_fizzy_commits_sha2_check-runs": [{"check_runs": [run(2, "failure", "2026-10-01T11:30:00Z")]}],
+        "repos_fizzyedit_fizzy_commits_sha3_check-runs": [{"check_runs": [
+            run(3, "failure", "2026-10-02T09:30:00Z"), run(4, "cancelled", "2026-10-02T09:31:00Z")]}],
+        "repos_fizzyedit_fizzy_commits_sha4_check-runs": [{"check_runs": [run(5, "success", "2026-10-02T10:05:00Z")]}],
+        "repos_fizzyedit_fizzy_issues_7_timeline": [[
+            {"event": "ready_for_review", "created_at": "2026-10-01T10:30:00Z"}, {"event": "labeled"}]],
+        "repos_fizzyedit_fizzy_pulls_7_comments": [[{"id": 1}, {"id": 2}, {"id": 3}]],
+        "graphql": {"data": {"repository": {"pullRequest": {"reviewThreads": {
+            "nodes": [{"id": "a"}, {"id": "b"}], "pageInfo": {"hasNextPage": False, "endCursor": None}}}}}},
+        "search_issues": [{"items": [{"number": 9}, {"number": 10}, {"number": 11}, {"number": 12}, {"number": 7}, {"number": 8}]}],
+    }
+
+
+def run(args, env):
+    return subprocess.run([sys.executable, str(RECORD), *args], capture_output=True, text=True, env=env)
+
+
+def main():
+    with tempfile.TemporaryDirectory() as tmp:
+        tmp = Path(tmp)
+        (tmp / "fx").mkdir()
+        (tmp / "bin").mkdir()
+        for name, body in fixtures().items():
+            (tmp / "fx" / f"{name}.json").write_text(json.dumps(body))
+        (tmp / "fx" / "repos_fizzyedit_fizzy_pulls_12.json").write_text("")  # gh printed nothing
+        gh = tmp / "bin" / "gh"
+        gh.write_text(FAKE_GH)
+        gh.chmod(0o755)
+        log = tmp / "cache" / "log.txt"
+        env = {**os.environ, "PATH": f"{tmp / 'bin'}:{os.environ['PATH']}", "FIXTURES": str(tmp / "fx"),
+               "FIZZY_AGENT_LOG": str(log)}
+
+        # Ready at 10:30: commits 2, 3, 4 come after; reds are runs 2 and 3 (cancelled is not a failure).
+        want = "#7 | codex | 3 | 3c/2t | 2"
+        out = run(["7"], env)
+        assert out.returncode == 0 and out.stdout.strip() == want, out
+        assert log.read_text() == want + "\n"
+
+        out = run(["7"], env)
+        assert out.stdout.strip() == want and log.read_text() == want + "\n", "recorded twice"
+
+        out = run(["8"], env)
+        assert out.returncode != 0 and "Part of" in out.stderr, out
+
+        log.unlink()
+        # 9 is still open, 10's work issue is a 404, 12 gets empty output: each is skipped.
+        # 11 has no running label: recorded as unknown, with a warning. 7 still lands.
+        unknown = "#11 | unknown | 0 | 0c/2t | 0"
+        out = run(["--all", "2026-10-01"], env)
+        assert out.returncode == 0 and out.stdout.split("\n")[:-1] == [unknown, want], out
+        assert log.read_text() == unknown + "\n" + want + "\n", out
+        for n in ("PR 9 skipped", "PR 10 skipped", "PR 12 skipped", "PR 11: "):
+            assert n in out.stderr, out.stderr
+
+        for n in ("9", "10", "12"):
+            out = run([n], env)
+            assert out.returncode != 0 and "skipped" in out.stderr, out
+        assert log.read_text() == unknown + "\n" + want + "\n", "a skipped PR was recorded"
+
+        assert run([], env).returncode != 0
+    print("ok")
+
+
+main()