From 8fe84fcfb774e689bf71dac1b28187e7cddfa1e2 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Fri, 20 Nov 2026 13:27:48 -0800 Subject: [PATCH 1/4] feat(server): pull_request_read returns a compact summary by default The default read returned the full body and every check run (median 15 KB), while agents still fell back to gh for review threads and comments. It now returns core state, a checks summary (counts plus failing names) and the unresolved review thread count; `include` adds body, checks, threads, comments or diff. Co-Authored-By: Claude Opus 5.5 --- .../toolkits/pullRequests/handlers.test.ts | 74 +++++--- .../src/mcp/toolkits/pullRequests/handlers.ts | 169 ++++++++++-------- .../src/mcp/toolkits/pullRequests/tools.ts | 94 +++++----- evals/mcp-tool-use/cases/pull-requests.ndjson | 16 +- evals/mcp-tool-use/run.test.ts | 16 +- evals/mcp-tool-use/run.ts | 116 ++++++------ 6 files changed, 267 insertions(+), 218 deletions(-) diff --git a/apps/server/src/mcp/toolkits/pullRequests/handlers.test.ts b/apps/server/src/mcp/toolkits/pullRequests/handlers.test.ts index eb56ddfbed044..7a69d8d9a939a 100644 --- a/apps/server/src/mcp/toolkits/pullRequests/handlers.test.ts +++ b/apps/server/src/mcp/toolkits/pullRequests/handlers.test.ts @@ -847,7 +847,20 @@ describe("source control tools", () => { changedFiles: 1, labels: [], reviewers: [], - checks: [{ name: "CI", status: "success", description: null, url: null }], + checks: [ + { name: "CI", status: "success", description: null, url: null }, + { name: "Lint", status: "failure", description: null, url: null }, + { name: "E2E", status: "cancelled", description: null, url: null }, + { name: "Deploy", status: "pending", description: null, url: null }, + { + name: "Approval", + status: "action-required", + description: null, + url: null, + placeholder: true, + }, + { name: "Docs", status: "skipped", description: null, url: null }, + ], } as unknown as PullRequestDetail; const reviewThread = (id: string, isResolved: boolean) => ({ id, @@ -866,34 +879,47 @@ describe("source control tools", () => { commits: [], }; - it.effect("reads the thread's branch PR, only the sections asked, without resolved threads", () => + it.effect("reads the thread's branch PR compactly, adding only the sections asked", () => Effect.gen(function* () { const asked: Array = []; + let reviewThreadsTruncated = false; const harness = yield* makeHarness({ thread: branchThread, pullRequests: { detail: (ref) => Effect.sync(() => asked.push(ref)).pipe(Effect.as(detail)), - activity: () => Effect.succeed(activity), + activity: () => Effect.sync(() => ({ ...activity, reviewThreadsTruncated })), }, }); - const result = yield* harness.call("pull_request_read", { - sections: ["summary", "conversation"], + const compact = yield* harness.call("pull_request_read", {}); + expect(compact).toMatchObject({ + headSha: "abc123", + checks: { passed: 2, pending: 2, failed: ["Lint", "E2E"] }, + unresolvedThreads: 1, }); - expect(asked).toEqual([ - { - projectId: PROJECT_ID, - host: "github.com", - repository: "t3tools/t3code", - number: 42, - allowStale: false, - }, - ]); - expect(result.summary?.headSha).toBe("abc123"); - expect(result.checks).toBeUndefined(); - expect(result.conversation?.reviewThreads.map((thread) => thread.id)).toEqual([ + expect(compact.body).toBeUndefined(); + expect(compact.checkRuns).toBeUndefined(); + expect(compact.conversation).toBeUndefined(); + expect(compact.unresolvedThreadsTruncated).toBeUndefined(); + const result = yield* harness.call("pull_request_read", { include: ["body", "threads"] }); + expect(asked[1]).toEqual({ + projectId: PROJECT_ID, + host: "github.com", + repository: "t3tools/t3code", + number: 42, + allowStale: false, + }); + expect(result.body).toBe("Adds tools."); + expect(result.checkRuns).toBeUndefined(); + expect(result.conversation?.comments).toBeUndefined(); + expect(result.conversation?.reviewThreads?.map((thread) => thread.id)).toEqual([ "open-thread", ]); - expect(result.conversation?.hiddenResolvedThreads).toBe(1); + // Unfetched threads may be unresolved, so the compact count says it is partial. + reviewThreadsTruncated = true; + expect(yield* harness.call("pull_request_read", {})).toMatchObject({ + unresolvedThreads: 1, + unresolvedThreadsTruncated: true, + }); }), ); @@ -910,6 +936,7 @@ describe("source control tools", () => { const harness = yield* makeHarness({ thread: branchThread, pullRequests: { + detail: () => Effect.succeed(detail), activity: () => Effect.sync(() => ({ ...activity, @@ -942,7 +969,7 @@ describe("source control tools", () => { ), }, }); - const first = yield* harness.call("pull_request_read", { sections: ["conversation"] }); + const first = yield* harness.call("pull_request_read", { include: ["threads"] }); expect(first.conversation?.reviewThreads).toMatchObject([ { id: "first", moreComments: false, comments: [{}, { body: "reply" }] }, ]); @@ -951,12 +978,12 @@ describe("source control tools", () => { // The agent resolves what it read; the next page still starts at the next thread. firstResolved = true; const second = yield* harness.call("pull_request_read", { cursor: cursor! }); - expect(second.summary).toBeUndefined(); - expect(second.conversation?.reviewThreads.map((thread) => thread.id)).toEqual(["second"]); + expect(second.conversation?.comments).toBeUndefined(); + expect(second.conversation?.reviewThreads?.map((thread) => thread.id)).toEqual(["second"]); expect(second.conversation?.nextCursor).toBeNull(); // A cursor past the current end finishes instead of repeating an empty page. const stale = Base64Url.encode( - JSON.stringify({ threads: 99, comments: 99, includeResolved: false }), + JSON.stringify({ threads: 99, comments: null, includeResolved: false }), ); const past = yield* harness.call("pull_request_read", { cursor: stale }); expect(past.conversation).toMatchObject({ reviewThreads: [], nextCursor: null }); @@ -991,10 +1018,11 @@ describe("source control tools", () => { pullRequests: { summary: () => Effect.succeed({ state: "open", headBranch } as never), detail: (ref) => Effect.sync(() => asked.push(ref)).pipe(Effect.as(detail)), + activity: () => Effect.succeed(activity), }, }); const harness = yield* opened("tools"); - yield* harness.call("pull_request_read", { sections: ["summary"] }); + yield* harness.call("pull_request_read", {}); expect(asked).toMatchObject([{ number: 43 }]); // Before sync, the host says which branch it came from; another branch's is not this one. diff --git a/apps/server/src/mcp/toolkits/pullRequests/handlers.ts b/apps/server/src/mcp/toolkits/pullRequests/handlers.ts index fffed7dd4f42b..9bb29aeac9036 100644 --- a/apps/server/src/mcp/toolkits/pullRequests/handlers.ts +++ b/apps/server/src/mcp/toolkits/pullRequests/handlers.ts @@ -5,6 +5,7 @@ import { PullRequestListCursors, type PullRequestRef, type PullRequestReviewCommentDraft, + type PullRequestCheck, type PullRequestReviewThread, type OrchestrationProjectShell, type PullRequestState, @@ -70,9 +71,10 @@ import { const ReadCursor = Schema.Union([ Schema.Struct({ diff: TrimmedNonEmptyString }), + // Null marks a list the read did not ask for. Schema.Struct({ - threads: NonNegativeInt, - comments: NonNegativeInt, + threads: Schema.NullOr(NonNegativeInt), + comments: Schema.NullOr(NonNegativeInt), includeResolved: Schema.Boolean, }), ]); @@ -116,7 +118,7 @@ const REVIEW_DIFF_PAGES = 10; function conversationPage( activity: PullRequestActivity, - start: { readonly threads: number; readonly comments: number }, + start: { readonly threads: number | null; readonly comments: number | null }, includeResolved: boolean, ) { // Offsets index the unfiltered lists, whose order resolving a thread does not change. @@ -124,8 +126,8 @@ function conversationPage( const pageThreads: Array = []; const pageComments: Array = []; let size = 0; - let thread = Math.min(start.threads, reviewThreads.length); - let comment = Math.min(start.comments, comments.length); + let thread = start.threads === null ? null : Math.min(start.threads, reviewThreads.length); + let comment = start.comments === null ? null : Math.min(start.comments, comments.length); const fits = (item: unknown) => { const next = size + JSON.stringify(item).length; if (next > CONVERSATION_PAGE_CHARS && pageThreads.length + pageComments.length > 0) { @@ -135,7 +137,7 @@ function conversationPage( return true; }; // Review threads come first: they hold the feedback an agent acts on. - while (thread < reviewThreads.length) { + while (thread !== null && thread < reviewThreads.length) { const next = reviewThreads[thread]!; if (includeResolved || !next.isResolved) { if (!fits(next)) break; @@ -143,22 +145,33 @@ function conversationPage( } thread += 1; } - if (thread === reviewThreads.length) { - while (comment < comments.length && fits(comments[comment])) { + const threadsDone = thread === null || thread === reviewThreads.length; + if (threadsDone) { + while (comment !== null && comment < comments.length && fits(comments[comment])) { pageComments.push(comments[comment++]!); } } - const done = thread === reviewThreads.length && comment === comments.length; + const done = threadsDone && (comment === null || comment === comments.length); return { threads: pageThreads, comments: pageComments, - hiddenResolvedThreads: includeResolved - ? 0 - : reviewThreads.filter((candidate) => candidate.isResolved).length, next: done ? null : { threads: thread, comments: comment, includeResolved }, }; } +function checksSummary(checks: ReadonlyArray) { + let passed = 0; + let pending = 0; + const failed: Array = []; + for (const check of checks) { + if (check.placeholder === true) pending += 1; + else if (check.status === "failure" || check.status === "cancelled") failed.push(check.name); + else if (check.status === "pending" || check.status === "action-required") pending += 1; + else passed += 1; + } + return { passed, pending, failed }; +} + function entryOf( link: ThreadPullRequestLink, chains: ReturnType, @@ -570,10 +583,8 @@ const make = Effect.gen(function* () { const [detail, activity, diff] = yield* Effect.all( [ // Agents pin merges to this headSha, so it must not come from a stale snapshot. - wants("summary") || wants("checks") - ? pullRequests.detail({ ...ref, allowStale: false }).pipe(Effect.asSome) - : Effect.succeedNone, - wants("conversation") ? pullRequests.activity(ref).pipe(Effect.asSome) : Effect.succeedNone, + pullRequests.detail({ ...ref, allowStale: false }), + pullRequests.activity(ref), wants("diff") ? pullRequests .diff({ ...ref, ...(diffCursor === undefined ? {} : { cursor: diffCursor }) }) @@ -582,70 +593,77 @@ const make = Effect.gen(function* () { ], { concurrency: "unbounded" }, ).pipe(Effect.mapError(forgeFailure)); - const page = Option.map(activity, (value) => - conversationPage( - value, - cursor !== undefined && "threads" in cursor ? cursor : { threads: 0, comments: 0 }, - includeResolved, - ), - ); + const wantsThreads = wants("threads"); + const wantsComments = wants("comments"); + const page = + wantsThreads || wantsComments + ? Option.some( + conversationPage( + activity, + cursor !== undefined && "threads" in cursor + ? cursor + : { threads: wantsThreads ? 0 : null, comments: wantsComments ? 0 : null }, + includeResolved, + ), + ) + : Option.none(); const threadRests = Option.isSome(page) ? yield* Effect.forEach(page.value.threads, (thread) => restOfThread(ref, thread), { concurrency: 4, }) : []; const result: Omit = { - ...(Option.isSome(detail) && wants("summary") - ? { - summary: { - title: detail.value.title, - body: detail.value.body, - state: detail.value.state, - isDraft: detail.value.isDraft, - author: detail.value.author?.login ?? null, - headBranch: detail.value.headBranch, - baseBranch: detail.value.baseBranch, - headSha: detail.value.headSha ?? null, - mergeability: detail.value.mergeability, - behindBy: detail.value.behindBy ?? null, - autoMergeEnabled: detail.value.autoMergeEnabled ?? null, - additions: detail.value.additions, - deletions: detail.value.deletions, - changedFiles: detail.value.changedFiles, - labels: detail.value.labels.map((label) => label.name), - reviewers: detail.value.reviewers.map((reviewer) => reviewer.login), - }, - } - : {}), - ...(Option.isSome(detail) && wants("checks") ? { checks: detail.value.checks } : {}), - ...(Option.isSome(activity) && Option.isSome(page) + title: detail.title, + state: detail.state, + isDraft: detail.isDraft, + author: detail.author?.login ?? null, + headBranch: detail.headBranch, + baseBranch: detail.baseBranch, + headSha: detail.headSha ?? null, + mergeability: detail.mergeability, + behindBy: detail.behindBy ?? null, + autoMergeEnabled: detail.autoMergeEnabled ?? null, + checks: checksSummary(detail.checks), + unresolvedThreads: activity.reviewThreads.filter((thread) => !thread.isResolved).length, + ...(activity.reviewThreadsTruncated === true ? { unresolvedThreadsTruncated: true } : {}), + ...(wants("body") ? { body: detail.body } : {}), + ...(wants("checks") ? { checkRuns: detail.checks } : {}), + ...(Option.isSome(page) ? { conversation: { - comments: page.value.comments.map((comment) => ({ - id: comment.id, - kind: comment.kind, - author: comment.author?.login ?? null, - body: comment.body, - createdAt: comment.createdAt, - path: comment.path, - reviewState: comment.reviewState, - })), - reviewThreads: page.value.threads.map((thread, index) => ({ - id: thread.id, - path: thread.path, - line: thread.line, - isResolved: thread.isResolved, - isOutdated: thread.isOutdated, - comments: [...thread.comments, ...threadRests[index]!.comments].map((comment) => ({ - author: comment.author?.login ?? null, - body: comment.body, - createdAt: comment.createdAt, - })), - moreComments: threadRests[index]!.moreComments, - })), - hiddenResolvedThreads: page.value.hiddenResolvedThreads, - truncated: - activity.value.commentsTruncated || activity.value.reviewThreadsTruncated === true, + ...(wantsThreads + ? { + reviewThreads: page.value.threads.map((thread, index) => ({ + id: thread.id, + path: thread.path, + line: thread.line, + isResolved: thread.isResolved, + isOutdated: thread.isOutdated, + comments: [...thread.comments, ...threadRests[index]!.comments].map( + (comment) => ({ + author: comment.author?.login ?? null, + body: comment.body, + createdAt: comment.createdAt, + }), + ), + moreComments: threadRests[index]!.moreComments, + })), + } + : {}), + ...(wantsComments + ? { + comments: page.value.comments.map((comment) => ({ + id: comment.id, + kind: comment.kind, + author: comment.author?.login ?? null, + body: comment.body, + createdAt: comment.createdAt, + path: comment.path, + reviewState: comment.reviewState, + })), + } + : {}), + truncated: activity.commentsTruncated || activity.reviewThreadsTruncated === true, nextCursor: page.value.next === null ? null : readCursors.encode(page.value.next), }, } @@ -958,10 +976,13 @@ const make = Effect.gen(function* () { // A cursor continues the one section it came from, with that read's options. const sections: ReadonlyArray = cursor === undefined - ? (input.sections ?? ["summary", "checks"]) + ? (input.include ?? []) : "diff" in cursor ? ["diff"] - : ["conversation"]; + : [ + ...(cursor.threads === null ? [] : (["threads"] as const)), + ...(cursor.comments === null ? [] : (["comments"] as const)), + ]; const includeResolved = cursor !== undefined && "includeResolved" in cursor ? cursor.includeResolved diff --git a/apps/server/src/mcp/toolkits/pullRequests/tools.ts b/apps/server/src/mcp/toolkits/pullRequests/tools.ts index e8cfddb9e0919..7c0d172469143 100644 --- a/apps/server/src/mcp/toolkits/pullRequests/tools.ts +++ b/apps/server/src/mcp/toolkits/pullRequests/tools.ts @@ -498,9 +498,10 @@ export const ThreadPullRequestInput = Schema.Struct({ export type ThreadPullRequestInput = typeof ThreadPullRequestInput.Type; export const PullRequestReadSection = Schema.Literals([ - "summary", + "body", "checks", - "conversation", + "threads", + "comments", "diff", ]); export type PullRequestReadSection = typeof PullRequestReadSection.Type; @@ -513,50 +514,53 @@ const ReadComment = Schema.Struct({ export const ReadPullRequestResult = Schema.Struct({ ...PullRequestIdentity, - summary: Schema.optional( - Schema.Struct({ - title: Schema.String, - body: Schema.String, - state: PullRequestState, - isDraft: Schema.Boolean, - author: Schema.NullOr(Schema.String), - headBranch: Schema.String, - baseBranch: Schema.String, - headSha: Schema.NullOr(Schema.String), - mergeability: PullRequestMergeability, - behindBy: Schema.NullOr(Schema.Int), - autoMergeEnabled: Schema.NullOr(Schema.Boolean), - additions: Schema.Int, - deletions: Schema.Int, - changedFiles: Schema.Int, - labels: Schema.Array(Schema.String), - reviewers: Schema.Array(Schema.String), - }), - ), - checks: Schema.optional(Schema.Array(PullRequestCheck)), + title: Schema.String, + state: PullRequestState, + isDraft: Schema.Boolean, + author: Schema.NullOr(Schema.String), + headBranch: Schema.String, + baseBranch: Schema.String, + headSha: Schema.NullOr(Schema.String), + mergeability: PullRequestMergeability, + behindBy: Schema.NullOr(Schema.Int), + autoMergeEnabled: Schema.NullOr(Schema.Boolean), + checks: Schema.Struct({ + passed: Schema.Int, + pending: Schema.Int, + failed: Schema.Array(Schema.String), + }), + unresolvedThreads: Schema.Int, + unresolvedThreadsTruncated: Schema.optional(Schema.Literal(true)).annotate({ + description: "unresolvedThreads is a lower bound.", + }), + body: Schema.optional(Schema.String), + checkRuns: Schema.optional(Schema.Array(PullRequestCheck)), conversation: Schema.optional( Schema.Struct({ - comments: Schema.Array( - Schema.Struct({ - ...ReadComment.fields, - id: Schema.String, - kind: PullRequestCommentKind, - path: Schema.NullOr(Schema.String), - reviewState: Schema.NullOr(Schema.String), - }), + reviewThreads: Schema.optional( + Schema.Array( + Schema.Struct({ + id: Schema.String, + path: Schema.String, + line: Schema.NullOr(Schema.Int), + isResolved: Schema.Boolean, + isOutdated: Schema.Boolean, + comments: Schema.Array(ReadComment), + moreComments: Schema.Boolean, + }), + ), ), - reviewThreads: Schema.Array( - Schema.Struct({ - id: Schema.String, - path: Schema.String, - line: Schema.NullOr(Schema.Int), - isResolved: Schema.Boolean, - isOutdated: Schema.Boolean, - comments: Schema.Array(ReadComment), - moreComments: Schema.Boolean, - }), + comments: Schema.optional( + Schema.Array( + Schema.Struct({ + ...ReadComment.fields, + id: Schema.String, + kind: PullRequestCommentKind, + path: Schema.NullOr(Schema.String), + reviewState: Schema.NullOr(Schema.String), + }), + ), ), - hiddenResolvedThreads: Schema.Int, truncated: Schema.Boolean, nextCursor: Schema.NullOr(Schema.String), }), @@ -617,12 +621,12 @@ export const MergePullRequestResult = Schema.Struct({ export type MergePullRequestResult = typeof MergePullRequestResult.Type; const ReadPullRequestTool = Tool.make("pull_request_read", { - description: "Read a PR's state, checks, review comments, or diff.", + description: "Read a PR's state and check counts. `include` adds detail.", parameters: Schema.Struct({ ...ThreadPullRequestInput.fields, - sections: Schema.optional( + include: Schema.optional( Schema.Array(PullRequestReadSection).annotate({ - description: "Default: summary, checks. conversation has review threads.", + description: "threads: unresolved review threads.", }), ), includeResolved: Schema.optional(Schema.Boolean), diff --git a/evals/mcp-tool-use/cases/pull-requests.ndjson b/evals/mcp-tool-use/cases/pull-requests.ndjson index 62fdf96238ea1..3ea28bb130703 100644 --- a/evals/mcp-tool-use/cases/pull-requests.ndjson +++ b/evals/mcp-tool-use/cases/pull-requests.ndjson @@ -39,17 +39,17 @@ {"id": "schedule-fresh-session-per-run", "area": "pull-requests", "kind": "guard", "pattern": "recurring task that should not accumulate context", "experience": "A recurring triage task kept one thread so the user could read its history, but each run carried every earlier run's context into the next.", "rule": "schedule_task `freshSession: true` keeps runs in one thread while each run starts a fresh session.", "privacy": "commit-safe", "setup": {"role": "top", "context": "Today is Monday 2026-11-02, 10:00 local time.", "mocks": {"schedule_task": {"scheduledTaskId": "scheduled-task-7", "title": "Triage new issues", "enabled": true, "projectId": "project-1", "boundThreadId": null, "freshSession": true, "schedule": {"type": "fixed_time", "timeOfDay": "09:00"}, "nextRunAt": "2026-11-03T09:00:00-08:00", "lastRunStatus": "never"}}}, "prompt": "Every morning at 9, triage new issues in acme/app. Keep all the runs in one thread so I can scroll back, but each run should start clean without remembering the previous days.", "expect": {"calls": [{"tool": "schedule_task", "args": {"schedule.type": "fixed_time", "freshSession": true}}], "notCalls": ["delegate_task", "run_scheduled_task_now"]}, "source": {"request": "juliusmarminge (maintainer) request, 2026-11-03: automations in one thread with separate provider sessions"}, "notes": "schedule-one-off-tomorrow is the control: an ordinary schedule must not set freshSession."} {"id": "babysit-merge-after-approval-until-merged", "area": "pull-requests", "kind": "bad", "pattern": "babysit until a teammate approves, then merge, watching only until checks finish", "experience": "Asked to babysit a PR and merge it once a teammate approved, the agent watched it only until checks finished, so the thread left Working as soon as checks passed while the agent still owned the merge.", "rule": "until: merged when you will act on the PR yourself, such as merging it after someone else approves", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "The HTML preview frame has no accessible name. Fix it and open a PR."}, {"role": "assistant", "text": "Opened [PR #124](https://github.com/acme/app/pull/124). The preview frame now has the title \"HTML preview\".", "toolCalls": [{"tool": "Bash", "input": {"command": "git push -u origin fix/html-preview-title && gh pr create --fill"}, "result": "https://github.com/acme/app/pull/124"}, {"tool": "link_pull_request", "input": {"url": "https://github.com/acme/app/pull/124"}, "result": {"host": "github.com", "repository": "acme/app", "number": 124, "url": "https://github.com/acme/app/pull/124", "alreadyLinked": false}}]}], "mocks": {"watch_pull_request": "default", "link_pull_request": "default"}, "world": {"branch": "fix/html-preview-title", "commits": [{"sha": "abc1234", "subject": "fix(web): HTML preview frame gets an accessible title"}], "diff": "diff --git a/apps/web/src/HtmlPreview.tsx b/apps/web/src/HtmlPreview.tsx\n--- a/apps/web/src/HtmlPreview.tsx\n+++ b/apps/web/src/HtmlPreview.tsx\n@@ -1,3 +1,3 @@\n export function HtmlPreview({ html }: { html: string }) {\n- return