From 23977f1612927bdd28dd61dfabd48b1ed1f19448 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 20 Nov 2026 13:56:52 +0000 Subject: [PATCH 1/2] fix(server): watch_pull_request says a merge or close wakes the agent Since #26271 a watched thread is woken once when its pull request merges or closes, but the tool description still listed only checks, comments and conflicts, so agents polled or asked the user instead. Add the merge wake to the description and an mcp-tool-use eval case for it. Fixes pingdotgg/t3code#39726 Co-Authored-By: Claude Opus 5.5 --- apps/server/src/mcp/toolkits/pullRequests/tools.ts | 2 +- apps/server/src/provider/agentContextBudget.json | 2 +- evals/mcp-tool-use/cases/pull-requests.ndjson | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/server/src/mcp/toolkits/pullRequests/tools.ts b/apps/server/src/mcp/toolkits/pullRequests/tools.ts index 5a4b671a98dea..293ec8c486735 100644 --- a/apps/server/src/mcp/toolkits/pullRequests/tools.ts +++ b/apps/server/src/mcp/toolkits/pullRequests/tools.ts @@ -390,7 +390,7 @@ const WatchPullRequestInput = Schema.Struct({ const WatchPullRequestTool = Tool.make("watch_pull_request", { description: - "Watch an open PR for this thread, linking it first if needed. T3 wakes you on failed checks, passed required checks, new comments or reviews, or conflicts. Handle old comments first. Only later ones wake you. After this call, end your turn. The watch stays on after each wake, so do not call this again on a wake. A wake is not a merge decision. Check readiness yourself.", + "Watch an open PR for this thread, linking it first if needed. T3 wakes you on failed checks, passed required checks, new comments or reviews, or conflicts, and once when it merges or closes unless you did that with the host's CLI. Handle old comments first. Only later ones wake you. After this call, end your turn. The watch stays on after each wake, so do not call this again on a wake. A wake is not a merge decision. Check readiness yourself.", parameters: WatchPullRequestInput, success: WatchPullRequestResult, failure: PullRequestToolError, diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index 267fff59a6021..d3741441e4bc4 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { "claudeSystemAppendBytes": 6245, "codexAdditionalContextBytes": 7639, - "mcpToolListJsonBytes": 57299 + "mcpToolListJsonBytes": 57374 } diff --git a/evals/mcp-tool-use/cases/pull-requests.ndjson b/evals/mcp-tool-use/cases/pull-requests.ndjson index 950680da0988b..730977823672e 100644 --- a/evals/mcp-tool-use/cases/pull-requests.ndjson +++ b/evals/mcp-tool-use/cases/pull-requests.ndjson @@ -57,3 +57,4 @@ {"id": "ci-flake-rerun-through-gh", "area": "pull-requests", "kind": "bad", "pattern": "CI re-run with gh run rerun", "experience": "Asked to re-run a flaky check, agents looked up run ids with gh run list and ran gh run rerun --failed.", "rule": "ci_rerun re-runs a PR's failing checks", "privacy": "commit-safe", "setup": {"role": "top", "world": {"branch": "fix/retry-queue", "commits": [{"sha": "9f3c2a1", "subject": "fix(queue): retry a failed job once"}], "diff": "diff --git a/src/queue.ts b/src/queue.ts\n--- a/src/queue.ts\n+++ b/src/queue.ts\n@@ -4,3 +4,4 @@ export async function run(job: Job) {\n try {\n return await job.execute();\n- } catch {\n+ } catch (error) {\n+ if (job.attempts > 1) throw error;\n return run({ ...job, attempts: job.attempts + 1 });\n", "files": {"src/queue.ts": "export async function run(job: Job) {\n try {\n return await job.execute();\n } catch (error) {\n if (job.attempts > 1) throw error;\n return run({ ...job, attempts: job.attempts + 1 });\n }\n}\n"}, "prs": [{"number": 124, "title": "fix(queue): retry a failed job once", "headRefName": "fix/retry-queue", "checks": [{"name": "Lint", "state": "pass"}, {"name": "Test", "state": "fail", "log": "FAIL src/queue.test.ts > gives up after one retry\nAssertionError: expected 2 calls, got 3\n at src/queue.test.ts:18:5"}]}]}}, "prompt": "The Test failure on #124 is a known flake. Re-run it.", "expect": {"calls": [{"tool": "ci_rerun"}], "shellMustNot": ["\\bgh run (rerun|list|view)\\b", "\\bgh api\\b[^\\n]*rerun"]}, "notes": "Forge usage audit of agent logs on three machines (2026-10-28): agents read failing CI with gh run view --log-failed or the job logs API about 500 times in 45 days, and re-ran CI with gh run rerun 92 times."} {"id": "control-ci-why-no-rerun", "area": "pull-requests", "kind": "control", "pattern": "question about a failing check", "experience": "Control: the user asks why CI failed. Reading the log is right; re-running it was not asked.", "rule": "Re-run only when asked", "privacy": "commit-safe", "setup": {"role": "top", "world": {"branch": "fix/retry-queue", "commits": [{"sha": "9f3c2a1", "subject": "fix(queue): retry a failed job once"}], "diff": "diff --git a/src/queue.ts b/src/queue.ts\n--- a/src/queue.ts\n+++ b/src/queue.ts\n@@ -4,3 +4,4 @@ export async function run(job: Job) {\n try {\n return await job.execute();\n- } catch {\n+ } catch (error) {\n+ if (job.attempts > 1) throw error;\n return run({ ...job, attempts: job.attempts + 1 });\n", "files": {"src/queue.ts": "export async function run(job: Job) {\n try {\n return await job.execute();\n } catch (error) {\n if (job.attempts > 1) throw error;\n return run({ ...job, attempts: job.attempts + 1 });\n }\n}\n"}, "prs": [{"number": 124, "title": "fix(queue): retry a failed job once", "headRefName": "fix/retry-queue", "checks": [{"name": "Lint", "state": "pass"}, {"name": "Test", "state": "fail", "log": "FAIL src/queue.test.ts > gives up after one retry\nAssertionError: expected 2 calls, got 3\n at src/queue.test.ts:18:5"}]}]}}, "prompt": "Why is CI failing on #124? Don't change anything yet.", "expect": {"notCalls": ["ci_rerun", "pull_request_merge"], "shellMustNot": ["\\bgh run rerun\\b"]}, "notes": "Control for ci-flake-rerun-through-gh."} {"id": "land-stack-in-one-merge", "area": "pull-requests", "kind": "bad", "pattern": "a green, approved stack merged layer by layer or through gh stack merge", "experience": "Asked to land a two-PR stack, the agent had only a one-PR merge tool, so it merged the bottom layer alone, which restarts CI on the layer above, or fell back to gh stack merge in the shell.", "rule": "pull_request_merge stack:true on the top PR merges every layer below it at once.", "privacy": "commit-safe", "setup": {"role": "top", "context": "PR #124 (feature/error-detail, on main) and PR #125 (feature/frozen-host, on top of #124) form a native GitHub stack and are linked to this thread.", "world": {"repo": "acme/app", "branch": "feature/frozen-host", "stack": 300, "linkedPullRequests": [{"number": 124, "state": "open", "title": "feat(server): include the failing step in connection errors"}, {"number": 125, "state": "open", "title": "feat(server): detect a frozen host and say so"}], "prs": [{"number": 124, "title": "feat(server): include the failing step in connection errors", "headRefName": "feature/error-detail", "baseRefName": "main", "headRefOid": "e1d2c3b", "reviewDecision": "APPROVED", "mergeable": "MERGEABLE", "checks": [{"name": "CI", "state": "pass"}]}, {"number": 125, "title": "feat(server): detect a frozen host and say so", "headRefName": "feature/frozen-host", "baseRefName": "feature/error-detail", "headRefOid": "f7e6d5c", "reviewDecision": "APPROVED", "mergeable": "MERGEABLE", "checks": [{"name": "CI", "state": "pass"}]}]}}, "prompt": "Both PRs in the stack are approved and green. Land the stack.", "expect": {"calls": [{"tool": "pull_request_merge", "args": {"stack": true, "expectedHeadSha": "f7e6d5c"}, "argsAnyOf": [{"number": 125}, {"url": "~/pull/125"}, {"number": null, "url": null}]}], "maxCalls": {"pull_request_merge": 1}, "shellMustNot": ["gh stack merge", "gh pr merge", "gh api[^\\n]*/merge"], "callsMustNot": [{"tool": "pull_request_merge", "args": {"auto": true}}]}, "maxSteps": 8, "source": {"request": "juliusmarminge (maintainer) request, 2026-11-18: the PR tools could read stacks but not merge one; layer-by-layer merges restart CI above and can reorder a GitHub stack"}} +{"id": "watch-for-teammate-merge-no-own-poll", "area": "pull-requests", "kind": "bad", "pattern": "assumes a merge will not wake it, so polls or asks the user", "experience": "Told a teammate would merge the PR and asked to report when it merged, agents read watch_pull_request's wake list (checks, comments, conflicts), concluded a merge would not wake them, and ran their own polling or asked the user to check back.", "rule": "watch_pull_request: T3 wakes you ... and once when it merges or closes unless you did that with the host's CLI", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "The HTML preview frame has no accessible name. Fix it and open a PR."}, {"role": "assistant", "text": "Opened [PR #124](https://github.com/acme/app/pull/124). The preview frame now has the title \"HTML preview\".", "toolCalls": [{"tool": "Bash", "input": {"command": "git push -u origin fix/html-preview-title && gh pr create --fill"}, "result": "https://github.com/acme/app/pull/124"}, {"tool": "link_pull_request", "input": {"url": "https://github.com/acme/app/pull/124"}, "result": {"host": "github.com", "repository": "acme/app", "number": 124, "url": "https://github.com/acme/app/pull/124", "alreadyLinked": false}}]}], "mocks": {"watch_pull_request": "default", "link_pull_request": "default"}, "world": {"branch": "fix/html-preview-title", "commits": [{"sha": "abc1234", "subject": "fix(web): HTML preview frame gets an accessible title"}], "diff": "diff --git a/apps/web/src/HtmlPreview.tsx b/apps/web/src/HtmlPreview.tsx\n--- a/apps/web/src/HtmlPreview.tsx\n+++ b/apps/web/src/HtmlPreview.tsx\n@@ -1,3 +1,3 @@\n export function HtmlPreview({ html }: { html: string }) {\n- return