diff --git a/apps/server/src/mcp/toolkits/pullRequests/tools.ts b/apps/server/src/mcp/toolkits/pullRequests/tools.ts index 5a4b671a98dea..0b5dad39da07a 100644 --- a/apps/server/src/mcp/toolkits/pullRequests/tools.ts +++ b/apps/server/src/mcp/toolkits/pullRequests/tools.ts @@ -383,14 +383,14 @@ const WatchPullRequestInput = Schema.Struct({ until: Schema.optional( PullRequestWatchUntil.annotate({ description: - 'How long this thread stays working. Omit it to keep the current value; a new watch starts at "checks". "checks": until checks finish, for a PR you hand to the user. "merged": until it merges or closes or you unwatch it, when you will act on it yourself, such as merging after someone else approves.', + 'How long this thread stays working. Omit to keep the current value; a new watch starts at "checks". "checks": until checks finish, for a PR you hand to the user. "merged": until it merges, closes or is unwatched, when you act on it yourself, such as merging after approval.', }), ), }); const WatchPullRequestTool = Tool.make("watch_pull_request", { description: - "Watch an open PR for this thread, linking it first if needed. T3 wakes you on failed checks, passed required checks, new comments or reviews, or conflicts. Handle old comments first. Only later ones wake you. After this call, end your turn. The watch stays on after each wake, so do not call this again on a wake. A wake is not a merge decision. Check readiness yourself.", + "Watch an open PR for this thread, linking it if needed. T3 wakes you on failed checks, passed required checks, new comments or reviews, conflicts, and once on merge or close unless you did it via CLI. Handle old comments first; only new ones wake you. After this call, end your turn. The watch stays on: do not call this again on a wake. A wake is not a merge decision. Check readiness yourself.", parameters: WatchPullRequestInput, success: WatchPullRequestResult, failure: PullRequestToolError, diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index 267fff59a6021..e3e66105b01cc 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { "claudeSystemAppendBytes": 6245, "codexAdditionalContextBytes": 7639, - "mcpToolListJsonBytes": 57299 + "mcpToolListJsonBytes": 57298 } diff --git a/evals/mcp-tool-use/cases/pull-requests.ndjson b/evals/mcp-tool-use/cases/pull-requests.ndjson index 950680da0988b..5d572617d6df0 100644 --- a/evals/mcp-tool-use/cases/pull-requests.ndjson +++ b/evals/mcp-tool-use/cases/pull-requests.ndjson @@ -57,3 +57,4 @@ {"id": "ci-flake-rerun-through-gh", "area": "pull-requests", "kind": "bad", "pattern": "CI re-run with gh run rerun", "experience": "Asked to re-run a flaky check, agents looked up run ids with gh run list and ran gh run rerun --failed.", "rule": "ci_rerun re-runs a PR's failing checks", "privacy": "commit-safe", "setup": {"role": "top", "world": {"branch": "fix/retry-queue", "commits": [{"sha": "9f3c2a1", "subject": "fix(queue): retry a failed job once"}], "diff": "diff --git a/src/queue.ts b/src/queue.ts\n--- a/src/queue.ts\n+++ b/src/queue.ts\n@@ -4,3 +4,4 @@ export async function run(job: Job) {\n try {\n return await job.execute();\n- } catch {\n+ } catch (error) {\n+ if (job.attempts > 1) throw error;\n return run({ ...job, attempts: job.attempts + 1 });\n", "files": {"src/queue.ts": "export async function run(job: Job) {\n try {\n return await job.execute();\n } catch (error) {\n if (job.attempts > 1) throw error;\n return run({ ...job, attempts: job.attempts + 1 });\n }\n}\n"}, "prs": [{"number": 124, "title": "fix(queue): retry a failed job once", "headRefName": "fix/retry-queue", "checks": [{"name": "Lint", "state": "pass"}, {"name": "Test", "state": "fail", "log": "FAIL src/queue.test.ts > gives up after one retry\nAssertionError: expected 2 calls, got 3\n at src/queue.test.ts:18:5"}]}]}}, "prompt": "The Test failure on #124 is a known flake. Re-run it.", "expect": {"calls": [{"tool": "ci_rerun"}], "shellMustNot": ["\\bgh run (rerun|list|view)\\b", "\\bgh api\\b[^\\n]*rerun"]}, "notes": "Forge usage audit of agent logs on three machines (2026-10-28): agents read failing CI with gh run view --log-failed or the job logs API about 500 times in 45 days, and re-ran CI with gh run rerun 92 times."} {"id": "control-ci-why-no-rerun", "area": "pull-requests", "kind": "control", "pattern": "question about a failing check", "experience": "Control: the user asks why CI failed. Reading the log is right; re-running it was not asked.", "rule": "Re-run only when asked", "privacy": "commit-safe", "setup": {"role": "top", "world": {"branch": "fix/retry-queue", "commits": [{"sha": "9f3c2a1", "subject": "fix(queue): retry a failed job once"}], "diff": "diff --git a/src/queue.ts b/src/queue.ts\n--- a/src/queue.ts\n+++ b/src/queue.ts\n@@ -4,3 +4,4 @@ export async function run(job: Job) {\n try {\n return await job.execute();\n- } catch {\n+ } catch (error) {\n+ if (job.attempts > 1) throw error;\n return run({ ...job, attempts: job.attempts + 1 });\n", "files": {"src/queue.ts": "export async function run(job: Job) {\n try {\n return await job.execute();\n } catch (error) {\n if (job.attempts > 1) throw error;\n return run({ ...job, attempts: job.attempts + 1 });\n }\n}\n"}, "prs": [{"number": 124, "title": "fix(queue): retry a failed job once", "headRefName": "fix/retry-queue", "checks": [{"name": "Lint", "state": "pass"}, {"name": "Test", "state": "fail", "log": "FAIL src/queue.test.ts > gives up after one retry\nAssertionError: expected 2 calls, got 3\n at src/queue.test.ts:18:5"}]}]}}, "prompt": "Why is CI failing on #124? Don't change anything yet.", "expect": {"notCalls": ["ci_rerun", "pull_request_merge"], "shellMustNot": ["\\bgh run rerun\\b"]}, "notes": "Control for ci-flake-rerun-through-gh."} {"id": "land-stack-in-one-merge", "area": "pull-requests", "kind": "bad", "pattern": "a green, approved stack merged layer by layer or through gh stack merge", "experience": "Asked to land a two-PR stack, the agent had only a one-PR merge tool, so it merged the bottom layer alone, which restarts CI on the layer above, or fell back to gh stack merge in the shell.", "rule": "pull_request_merge stack:true on the top PR merges every layer below it at once.", "privacy": "commit-safe", "setup": {"role": "top", "context": "PR #124 (feature/error-detail, on main) and PR #125 (feature/frozen-host, on top of #124) form a native GitHub stack and are linked to this thread.", "world": {"repo": "acme/app", "branch": "feature/frozen-host", "stack": 300, "linkedPullRequests": [{"number": 124, "state": "open", "title": "feat(server): include the failing step in connection errors"}, {"number": 125, "state": "open", "title": "feat(server): detect a frozen host and say so"}], "prs": [{"number": 124, "title": "feat(server): include the failing step in connection errors", "headRefName": "feature/error-detail", "baseRefName": "main", "headRefOid": "e1d2c3b", "reviewDecision": "APPROVED", "mergeable": "MERGEABLE", "checks": [{"name": "CI", "state": "pass"}]}, {"number": 125, "title": "feat(server): detect a frozen host and say so", "headRefName": "feature/frozen-host", "baseRefName": "feature/error-detail", "headRefOid": "f7e6d5c", "reviewDecision": "APPROVED", "mergeable": "MERGEABLE", "checks": [{"name": "CI", "state": "pass"}]}]}}, "prompt": "Both PRs in the stack are approved and green. Land the stack.", "expect": {"calls": [{"tool": "pull_request_merge", "args": {"stack": true, "expectedHeadSha": "f7e6d5c"}, "argsAnyOf": [{"number": 125}, {"url": "~/pull/125"}, {"number": null, "url": null}]}], "maxCalls": {"pull_request_merge": 1}, "shellMustNot": ["gh stack merge", "gh pr merge", "gh api[^\\n]*/merge"], "callsMustNot": [{"tool": "pull_request_merge", "args": {"auto": true}}]}, "maxSteps": 8, "source": {"request": "juliusmarminge (maintainer) request, 2026-11-18: the PR tools could read stacks but not merge one; layer-by-layer merges restart CI above and can reorder a GitHub stack"}} +{"id": "watch-for-teammate-merge-no-own-poll", "area": "pull-requests", "kind": "bad", "pattern": "assumes a merge will not wake it, so polls or asks the user", "experience": "Told a teammate would merge the PR and asked to report when it merged, agents read watch_pull_request's wake list (checks, comments, conflicts), concluded a merge would not wake them, and ran their own polling or asked the user to check back.", "rule": "watch_pull_request: T3 wakes you on ... and once on merge or close unless you did it via CLI", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "The HTML preview frame has no accessible name. Fix it and open a PR."}, {"role": "assistant", "text": "Opened [PR #124](https://github.com/acme/app/pull/124). The preview frame now has the title \"HTML preview\".", "toolCalls": [{"tool": "Bash", "input": {"command": "git push -u origin fix/html-preview-title && gh pr create --fill"}, "result": "https://github.com/acme/app/pull/124"}, {"tool": "link_pull_request", "input": {"url": "https://github.com/acme/app/pull/124"}, "result": {"host": "github.com", "repository": "acme/app", "number": 124, "url": "https://github.com/acme/app/pull/124", "alreadyLinked": false}}]}], "mocks": {"watch_pull_request": "default", "link_pull_request": "default"}, "world": {"branch": "fix/html-preview-title", "commits": [{"sha": "abc1234", "subject": "fix(web): HTML preview frame gets an accessible title"}], "diff": "diff --git a/apps/web/src/HtmlPreview.tsx b/apps/web/src/HtmlPreview.tsx\n--- a/apps/web/src/HtmlPreview.tsx\n+++ b/apps/web/src/HtmlPreview.tsx\n@@ -1,3 +1,3 @@\n export function HtmlPreview({ html }: { html: string }) {\n- return