diff --git a/docs/AGENT_WORK.md b/docs/AGENT_WORK.md index d590ced9..df0b246e 100644 --- a/docs/AGENT_WORK.md +++ b/docs/AGENT_WORK.md @@ -101,6 +101,17 @@ PR creation. Fix the reported problem and retry finish. Neither helper merges or - A different worker than the author reviews the PR. - No auto-merge. Merging is the maintainer's (`CONTRIBUTING.md`). +## Which worker suits which work + +`scripts/agent-record ` appends one line per finished agent PR to +`${FIZZY_AGENT_LOG:-~/.cache/fizzy-agent-record.txt}` (skipping a PR already there) and prints it: +`PR | worker | rounds-to-green | review findings | CI reds`. It reads GitHub through `gh` only. The worker is the +claimed issue's `running:*` label (found by the PR's `Part of fizzyedit/work#N` line); rounds-to-green is the +commits pushed after the PR left draft or first went red, whichever was first; review findings are +`c/t`; CI reds are failed or timed-out check runs across every commit of the +head's history. `agent-record --all ` backfills merged PRs since then. `python3 +scripts/test-agent-record.py` checks it against saved `gh` JSON. + ## Self-hosted runner On a dedicated Ubuntu 24.04 or newer machine with systemd (x86_64 or aarch64), run the diff --git a/scripts/agent-record b/scripts/agent-record new file mode 100755 index 00000000..76c32f4b --- /dev/null +++ b/scripts/agent-record @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +"""Append one line per finished agent PR: "PR | worker | rounds-to-green | review findings | CI reds". + + agent-record record one PR (printed, appended unless already recorded) + agent-record --all backfill merged PRs from (YYYY-MM-DD) that reference fizzyedit/work + +Reads GitHub through `gh` only and parses nothing but labels and the "Part of fizzyedit/work#N" +line. The log is ${FIZZY_AGENT_LOG:-~/.cache/fizzy-agent-record.txt}; the repo ${FIZZY_REPO:-fizzyedit/fizzy}. +""" + +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +REPO = os.environ.get("FIZZY_REPO", "fizzyedit/fizzy") +WORK = "fizzyedit/work" +FAILED = {"failure", "timed_out"} +THREADS = """query($o:String!,$r:String!,$n:Int!,$c:String){repository(owner:$o,name:$r){ +pullRequest(number:$n){reviewThreads(first:100,after:$c){nodes{id} +pageInfo{hasNextPage endCursor}}}}}""" + + +def log_path(): + return Path(os.environ.get("FIZZY_AGENT_LOG") or "~/.cache/fizzy-agent-record.txt").expanduser() + + +class Skip(Exception): + """A PR that cannot be recorded now; the message says why.""" + + +def gh(*args): + out = subprocess.run(["gh", *args], capture_output=True, text=True) + if out.returncode != 0: + raise Skip(f"gh {' '.join(args)} failed: {out.stderr.strip()}") + if not out.stdout.strip(): + raise Skip(f"gh {' '.join(args)} printed nothing") + return json.loads(out.stdout) + + +def pages(path, *extra): + """A paginated list endpoint as one flat list of items.""" + data = gh("api", "--paginate", "--slurp", *extra, path) + items = [] + for page in data or []: + if isinstance(page, dict): + page = next((v for v in page.values() if isinstance(v, list)), []) + items.extend(page if isinstance(page, list) else [page]) + return items + + +def threads(pr): + owner, name = REPO.split("/") + count, cursor = 0, None + while True: + args = ["api", "graphql", "-f", f"query={THREADS}", "-f", f"o={owner}", "-f", f"r={name}", "-F", f"n={pr}"] + if cursor: + args += ["-f", f"c={cursor}"] + conn = gh(*args)["data"]["repository"]["pullRequest"]["reviewThreads"] + count += len(conn["nodes"]) + if not conn["pageInfo"]["hasNextPage"]: + return count + cursor = conn["pageInfo"]["endCursor"] + + +def measure(pr): + """The recorded line for a PR, or None when it does not reference fizzyedit/work.""" + info = gh("api", f"repos/{REPO}/pulls/{pr}") + if not info.get("merged_at") and info.get("state") != "closed": + raise Skip("not merged or closed yet, so its numbers are not final") + ref = re.search(r"Part of fizzyedit/work#(\d+)", info.get("body") or "") + if not ref: + return None + issue = gh("api", f"repos/{WORK}/issues/{ref[1]}") + workers = sorted(l["name"][len("running:"):] for l in issue["labels"] if l["name"].startswith("running:")) + if not workers: + print(f"agent-record: PR {pr}: {WORK}#{ref[1]} has no running:* label, worker recorded as unknown", file=sys.stderr) + + commits = pages(f"repos/{REPO}/pulls/{pr}/commits") + reds = {} + for commit in commits: + for run in pages(f"repos/{REPO}/commits/{commit['sha']}/check-runs"): + if run.get("conclusion") in FAILED: + reds[run["id"]] = run.get("completed_at") or "" + + # A push is a commit; rounds count those after the PR first left draft or first went red. + events = pages(f"repos/{REPO}/issues/{pr}/timeline") + marks = [e["created_at"] for e in events if e.get("event") == "ready_for_review"] + marks += [t for t in reds.values() if t] + rounds = 0 + if marks: + since = min(marks) + rounds = sum(1 for c in commits if c["commit"]["committer"]["date"] > since) + + comments = len(pages(f"repos/{REPO}/pulls/{pr}/comments")) + findings = f"{comments}c/{threads(pr)}t" + return f"#{pr} | {'+'.join(workers) or 'unknown'} | {rounds} | {findings} | {len(reds)}" + + +def recorded(path): + if not path.exists(): + return {} + return {l.split(" | ", 1)[0]: l for l in path.read_text().splitlines()} + + +def record(pr, path): + """The PR's line, measured and appended unless the log already holds it.""" + held = recorded(path).get(f"#{pr}") + if held: + return held + line = measure(pr) + if line: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a") as f: + f.write(line + "\n") + return line + + +def main(argv): + path = log_path() + if len(argv) == 1 and argv[0].isdigit(): + try: + line = record(int(argv[0]), path) + except Skip as why: + sys.exit(f"agent-record: PR {argv[0]} skipped: {why}") + if line is None: + sys.exit(f"agent-record: PR {argv[0]} does not say 'Part of {WORK}#N'") + print(line) + elif len(argv) == 2 and argv[0] == "--all": + query = f'repo:{REPO} is:pr is:merged merged:>={argv[1]} "{WORK}"' + try: + found = pages("search/issues", "--method", "GET", "-f", f"q={query}") + except Skip as why: + sys.exit(f"agent-record: search failed: {why}") + for item in found: + try: + line = record(item["number"], path) + except Skip as why: + print(f"agent-record: PR {item['number']} skipped: {why}", file=sys.stderr) + continue + if line: + print(line) + else: + sys.exit(__doc__) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/scripts/test-agent-record.py b/scripts/test-agent-record.py new file mode 100755 index 00000000..9781f5b9 --- /dev/null +++ b/scripts/test-agent-record.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Checks scripts/agent-record against saved gh JSON: a fake `gh` answers each endpoint from a fixture.""" + +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +RECORD = Path(__file__).resolve().parent / "agent-record" + +FAKE_GH = """#!/usr/bin/env python3 +import os, sys +args = [a for a in sys.argv[1:] if not a.startswith("-")] +key = [a for a in args if a.startswith(("repos/", "search/")) or a == "graphql"][0].replace("/", "_") +path = os.path.join(os.environ["FIXTURES"], key + ".json") +if not os.path.exists(path): + sys.stderr.write("HTTP 404: " + key) + sys.exit(1) +print(open(path).read()) +""" + + +def sha(n): + return f"sha{n}" + + +def fixtures(): + c = lambda n, date: {"sha": sha(n), "commit": {"committer": {"date": date}}} + run = lambda i, conclusion, at: {"id": i, "conclusion": conclusion, "completed_at": at} + return { + "repos_fizzyedit_fizzy_pulls_7": {"body": "ci: thing\n\nPart of fizzyedit/work#14\n", "state": "closed", "merged_at": "2026-10-03T00:00:00Z"}, + "repos_fizzyedit_fizzy_pulls_8": {"body": "no reference here", "state": "closed", "merged_at": "x"}, + "repos_fizzyedit_fizzy_pulls_9": {"body": "Part of fizzyedit/work#14", "state": "open", "merged_at": None}, + "repos_fizzyedit_fizzy_pulls_10": {"body": "Part of fizzyedit/work#99", "state": "closed", "merged_at": "x"}, + "repos_fizzyedit_fizzy_pulls_11": {"body": "Part of fizzyedit/work#15", "state": "closed", "merged_at": "x"}, + "repos_fizzyedit_fizzy_pulls_11_commits": [[]], + "repos_fizzyedit_fizzy_issues_11_timeline": [[]], + "repos_fizzyedit_fizzy_pulls_11_comments": [[]], + "repos_fizzyedit_work_issues_15": {"labels": [{"name": "agent"}]}, + "repos_fizzyedit_work_issues_14": {"labels": [{"name": "agent"}, {"name": "running:codex"}]}, + "repos_fizzyedit_fizzy_pulls_7_commits": [[ + c(1, "2026-10-01T10:00:00Z"), c(2, "2026-10-01T11:00:00Z"), + c(3, "2026-10-02T09:00:00Z"), c(4, "2026-10-02T10:00:00Z")]], + "repos_fizzyedit_fizzy_commits_sha1_check-runs": [{"check_runs": [run(1, "success", "2026-10-01T10:05:00Z")]}], + "repos_fizzyedit_fizzy_commits_sha2_check-runs": [{"check_runs": [run(2, "failure", "2026-10-01T11:30:00Z")]}], + "repos_fizzyedit_fizzy_commits_sha3_check-runs": [{"check_runs": [ + run(3, "failure", "2026-10-02T09:30:00Z"), run(4, "cancelled", "2026-10-02T09:31:00Z")]}], + "repos_fizzyedit_fizzy_commits_sha4_check-runs": [{"check_runs": [run(5, "success", "2026-10-02T10:05:00Z")]}], + "repos_fizzyedit_fizzy_issues_7_timeline": [[ + {"event": "ready_for_review", "created_at": "2026-10-01T10:30:00Z"}, {"event": "labeled"}]], + "repos_fizzyedit_fizzy_pulls_7_comments": [[{"id": 1}, {"id": 2}, {"id": 3}]], + "graphql": {"data": {"repository": {"pullRequest": {"reviewThreads": { + "nodes": [{"id": "a"}, {"id": "b"}], "pageInfo": {"hasNextPage": False, "endCursor": None}}}}}}, + "search_issues": [{"items": [{"number": 9}, {"number": 10}, {"number": 11}, {"number": 12}, {"number": 7}, {"number": 8}]}], + } + + +def run(args, env): + return subprocess.run([sys.executable, str(RECORD), *args], capture_output=True, text=True, env=env) + + +def main(): + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + (tmp / "fx").mkdir() + (tmp / "bin").mkdir() + for name, body in fixtures().items(): + (tmp / "fx" / f"{name}.json").write_text(json.dumps(body)) + (tmp / "fx" / "repos_fizzyedit_fizzy_pulls_12.json").write_text("") # gh printed nothing + gh = tmp / "bin" / "gh" + gh.write_text(FAKE_GH) + gh.chmod(0o755) + log = tmp / "cache" / "log.txt" + env = {**os.environ, "PATH": f"{tmp / 'bin'}:{os.environ['PATH']}", "FIXTURES": str(tmp / "fx"), + "FIZZY_AGENT_LOG": str(log)} + + # Ready at 10:30: commits 2, 3, 4 come after; reds are runs 2 and 3 (cancelled is not a failure). + want = "#7 | codex | 3 | 3c/2t | 2" + out = run(["7"], env) + assert out.returncode == 0 and out.stdout.strip() == want, out + assert log.read_text() == want + "\n" + + out = run(["7"], env) + assert out.stdout.strip() == want and log.read_text() == want + "\n", "recorded twice" + + out = run(["8"], env) + assert out.returncode != 0 and "Part of" in out.stderr, out + + log.unlink() + # 9 is still open, 10's work issue is a 404, 12 gets empty output: each is skipped. + # 11 has no running label: recorded as unknown, with a warning. 7 still lands. + unknown = "#11 | unknown | 0 | 0c/2t | 0" + out = run(["--all", "2026-10-01"], env) + assert out.returncode == 0 and out.stdout.split("\n")[:-1] == [unknown, want], out + assert log.read_text() == unknown + "\n" + want + "\n", out + for n in ("PR 9 skipped", "PR 10 skipped", "PR 12 skipped", "PR 11: "): + assert n in out.stderr, out.stderr + + for n in ("9", "10", "12"): + out = run([n], env) + assert out.returncode != 0 and "skipped" in out.stderr, out + assert log.read_text() == unknown + "\n" + want + "\n", "a skipped PR was recorded" + + assert run([], env).returncode != 0 + print("ok") + + +main()