From 696f8b4ce1344e39d0f03ddfa8a257ff04e80733 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 8 Nov 2026 06:43:20 +0000 Subject: [PATCH 01/10] test(server): reproduce server-browser tab wedge (#32849) These tests reproduce pingdotgg/t3code#32849 and fail on main: - An agent action whose Playwright call never settles is not answered by the server at its timeout, so the broker evicts the shared host. - A stuck tab's resized event blocks the manager-event mirror, so another tab's color scheme change never applies. - SessionControl.system() on a closed control throws synchronously instead of rejecting, which escapes the mirror's .catch. AI-assisted; verified with the tests and tools listed above. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/preview/ServerBrowser.test.ts | 74 +++++++++++++++++++ .../server/src/preview/SessionControl.test.ts | 11 +++ 2 files changed, 85 insertions(+) diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index 3925baa1a426e..9e7eae784715b 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -2399,3 +2399,77 @@ it.live("a full server takes back the slot of the longest idle agent tab nobody }), ).pipe(Effect.provide(layer)), ); + +/** Makes the tab's next color scheme change hang in Playwright; resolves once it has started. */ +const stallColorScheme = (page: ReturnType["page"]) => { + const started = Promise.withResolvers(); + page.emulateMedia.mockImplementationOnce(() => { + started.resolve(); + return new Promise(() => {}); + }); + return Effect.promise(() => started.promise); +}; + +it.effect("a stuck agent action times out on the server and later calls on its tab fail fast", () => + Effect.scoped( + Effect.gen(function* () { + const { broker, tabId } = yield* ready; + const stalled = stallColorScheme(contexts[0]!.page); + const stuck = yield* broker + .invoke({ + scope, + tabId, + operation: "setColorScheme", + input: { colorScheme: "dark" }, + timeoutMs: 1_000, + }) + .pipe(Effect.flip, Effect.forkScoped); + yield* stalled; + // The broker evicts a host that has not answered by timeoutMs plus its one second grace. + yield* TestClock.adjust("2 seconds"); + expect(yield* Fiber.join(stuck)).toMatchObject({ _tag: "PreviewAutomationTimeoutError" }); + // The server answered in time, so the host every tab shares stays connected. + const status = yield* broker + .invoke({ scope, tabId, operation: "status", input: {} }) + .pipe(Effect.exit); + expect(status).toMatchObject({ _tag: "Success" }); + const busy = yield* broker + .invoke({ scope, tabId, operation: "click", input: { ref: "e1" }, timeoutMs: 1_000 }) + .pipe(Effect.flip); + expect(busy.message).toMatch(/busy|still running/i); + }), + ).pipe(Effect.provide(layer)), +); + +it.effect("a tab stuck in an agent action does not hold back setting changes for other tabs", () => + Effect.scoped( + Effect.gen(function* () { + const { broker, tabId } = yield* ready; + const manager = yield* Manager.PreviewManager; + const other = yield* broker.invoke({ + scope, + operation: "open", + input: { reuseExistingTab: false, show: false }, + }); + const stalled = stallColorScheme(contexts[0]!.page); + yield* broker + .invoke({ + scope, + tabId, + operation: "setColorScheme", + input: { colorScheme: "dark" }, + timeoutMs: 1_000, + }) + .pipe(Effect.ignore, Effect.forkScoped); + // The stuck action has already published its own tab's resized event. + yield* stalled; + yield* manager.adjust({ + threadId: scope.thread.threadId, + tabId: PreviewTabId.make(other.tabId!), + colorScheme: "dark", + }); + yield* TestClock.adjust("2 seconds"); + expect(contexts[1]!.page.emulateMedia).toHaveBeenCalledWith({ colorScheme: "dark" }); + }), + ).pipe(Effect.provide(layer)), +); diff --git a/apps/server/src/preview/SessionControl.test.ts b/apps/server/src/preview/SessionControl.test.ts index f47be6319cb81..c92d31b1c7702 100644 --- a/apps/server/src/preview/SessionControl.test.ts +++ b/apps/server/src/preview/SessionControl.test.ts @@ -344,6 +344,17 @@ describe("SessionControl", () => { expect(control.generation).toBe(1); }); + it("rejects system work on a closed tab instead of throwing", async () => { + const control = new SessionControl("agent"); + await control.close(); + // Callers handle failures on the returned promise; a synchronous throw escapes them. + let applied: Promise | undefined; + expect(() => { + applied = control.system(async () => {}); + }).not.toThrow(); + await expect(applied).rejects.toThrow("closed"); + }); + it("lets any agent act on a tab no agent opened, but only while no human controls it", async () => { const control = new SessionControl(null); await expect(control.agent("agent-a", async () => "acted")).resolves.toBe("acted"); From 79e297d3cc4456d218ade8e7d54bf6d356bf9a1a Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 8 Nov 2026 06:51:24 +0000 Subject: [PATCH 02/10] fix(server): a stuck browser action no longer wedges its tab or other tabs An agent action whose Playwright call never settled was never answered by the server, so the broker evicted the shared server-browser host. The manager-event mirror also awaited the stuck tab's queued setting change, which stopped closes, opens and setting changes on every other tab. The server now answers each request at its timeoutMs on the Effect clock and aborts the request's signal. The abandoned run keeps its tab's queue, so runs never overlap. While it is overdue, new agent actions on that tab are refused at once with a busy reason that says how to recover, unless a dialog is pending. Queued calls skip when their signal is aborted. The mirror no longer awaits per-tab work, and SessionControl.system() rejects on a closed tab instead of throwing. Fixes pingdotgg/t3code#32849 AI-assisted; verified with the tests and tools listed above. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/preview/ServerBrowser.test.ts | 56 +++++++++++++++++++ apps/server/src/preview/ServerBrowser.ts | 32 ++++++++--- apps/server/src/preview/SessionControl.ts | 30 ++++++++-- packages/contracts/src/previewAutomation.ts | 3 + 4 files changed, 107 insertions(+), 14 deletions(-) diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index 9e7eae784715b..85c4e2a42d6a1 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -2473,3 +2473,59 @@ it.effect("a tab stuck in an agent action does not hold back setting changes for }), ).pipe(Effect.provide(layer)), ); + +it.effect( + "calls queued behind a stuck agent action time out on their own, and dialogs come first", + () => + Effect.scoped( + Effect.gen(function* () { + const { broker, tabId } = yield* ready; + const page = contexts[0]!.page; + const stalled = stallColorScheme(page); + const stuck = yield* broker + .invoke({ + scope, + tabId, + operation: "setColorScheme", + input: { colorScheme: "dark" }, + timeoutMs: 1_000, + }) + .pipe(Effect.flip, Effect.forkScoped); + yield* stalled; + const queued = yield* broker + .invoke({ + scope, + tabId, + operation: "click", + input: { ref: "e1" }, + timeoutMs: 1_000, + }) + .pipe(Effect.flip, Effect.forkScoped); + // The host handles requests in arrival order, so the click is queued once this answers. + yield* broker.invoke({ scope, tabId, operation: "status", input: {} }); + yield* TestClock.adjust("2 seconds"); + expect(yield* Fiber.join(stuck)).toMatchObject({ _tag: "PreviewAutomationTimeoutError" }); + expect(yield* Fiber.join(queued)).toMatchObject({ _tag: "PreviewAutomationTimeoutError" }); + page.emit("dialog", { + type: () => "alert", + message: () => "Saved", + defaultValue: () => "", + accept: vi.fn(async () => {}), + dismiss: vi.fn(async () => {}), + }); + const blocked = yield* broker + .invoke({ + scope, + tabId, + operation: "click", + input: { ref: "e1" }, + timeoutMs: 1_000, + }) + .pipe(Effect.flip); + expect(blocked).toMatchObject({ + _tag: "PreviewAutomationControlInterruptedError", + reason: "dialogPending", + }); + }), + ).pipe(Effect.provide(layer)), +); diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 7f128dc5d4eb7..f7ecf6232b9c8 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -1970,7 +1970,7 @@ const make = Effect.gen(function* () { const runOperation = async ( request: PreviewAutomationRequest, - deadlineMs: number, + signal: AbortSignal, ): Promise => { const input = request.input; switch (request.operation) { @@ -2095,7 +2095,7 @@ const make = Effect.gen(function* () { } return statusWithTitle(tab, request.agentSessionId); }, - deadlineMs, + signal, ); } case "recordingStop": { @@ -2178,6 +2178,12 @@ const make = Effect.gen(function* () { return executeTabOperation(tab, request); }); } + // An action blocked by a dialog outlives its request; point the agent at the dialog. + if (tab.dialog && tab.control.overdue) + throw new BrowserControlInterrupted({ + message: "A browser dialog is pending. Read preview_status and use preview_dialog first.", + reason: "dialogPending", + }); return tab.control.agent( agentSessionId, async () => { @@ -2198,7 +2204,7 @@ const make = Effect.gen(function* () { if (generation !== tab.control.generation) ServerBrowserPage.invalidateRefs(tab.page); } }, - deadlineMs, + signal, ); }; @@ -2298,12 +2304,21 @@ const make = Effect.gen(function* () { const handleRequest = (connectionId: string, request: PreviewAutomationRequest) => Effect.tryPromise({ - try: () => - runOperation(request, performance.now() + request.timeoutMs).finally(() => - markUsed(request), - ), + try: (signal) => runOperation(request, signal).finally(() => markUsed(request)), catch: ServerBrowserPage.toOperationError, }).pipe( + // Answer before the broker evicts this host; the abandoned run still holds its tab's queue. + Effect.timeoutOrElse({ + duration: request.timeoutMs, + orElse: () => + Effect.fail( + new ServerBrowserPage.ServerBrowserOperationError({ + tag: "PreviewAutomationTimeoutError", + message: `The browser did not finish ${request.operation} in ${request.timeoutMs}ms.`, + detail: { timeoutMs: request.timeoutMs }, + }), + ), + }), Effect.match({ onSuccess: (result) => ({ ok: true as const, result }), onFailure: (error) => ({ @@ -2344,8 +2359,9 @@ const make = Effect.gen(function* () { closedSessions.delete(key); return; } + // Queued per tab at call time, so one stuck tab cannot hold back the others. if (event.type === "resized") { - await tab.control + void tab.control .system(async () => { if ( !viewportSettingsEqual(event.snapshot.viewport ?? FILL_PREVIEW_VIEWPORT, tab.setting) diff --git a/apps/server/src/preview/SessionControl.ts b/apps/server/src/preview/SessionControl.ts index f19f566ed30bb..ccb0ed92e2b7b 100644 --- a/apps/server/src/preview/SessionControl.ts +++ b/apps/server/src/preview/SessionControl.ts @@ -34,6 +34,7 @@ export class SessionControl { private pending: Promise = Promise.resolve(); private readonly handoff = new Set>(); private owner: string | null = null; + private running: AbortSignal | undefined; private epoch = 0; private closed = false; @@ -88,7 +89,12 @@ export class SessionControl { this.onGenerationChange(); } - private async action(allowed: () => boolean, run: () => Promise, deadlineMs?: number) { + /** An agent run outlived its request; it keeps the queue until it settles. */ + get overdue() { + return this.running?.aborted === true; + } + + private async action(allowed: () => boolean, run: () => Promise, signal?: AbortSignal) { this.assertOpen(); if (!allowed()) throw new BrowserControlInterrupted({ message: "You do not control this browser tab." }); @@ -96,13 +102,18 @@ export class SessionControl { return this.enqueue(async () => { this.assertOpen(); if (epoch !== this.epoch || !allowed()) throw new BrowserControlInterrupted(); - if (deadlineMs !== undefined && performance.now() >= deadlineMs) { + if (signal?.aborted) { throw new DOMException( "This browser request expired before it could start.", "TimeoutError", ); } - return run(); + this.running = signal; + try { + return await run(); + } finally { + this.running = undefined; + } }); } @@ -110,7 +121,7 @@ export class SessionControl { return this.agentId === null || this.agentId === agentId; } - agent(agentId: string, run: () => Promise, deadlineMs?: number) { + agent(agentId: string, run: () => Promise, signal?: AbortSignal) { if (!this.agentMayAct(agentId)) return Promise.reject( new BrowserControlInterrupted({ @@ -125,7 +136,14 @@ export class SessionControl { reason: "humanControl", }), ); - return this.action(() => this.agentMayAct(agentId) && this.owner === null, run, deadlineMs); + if (this.overdue) + return Promise.reject( + new BrowserControlInterrupted({ + message: "An earlier action on this tab timed out and is still running.", + reason: "busy", + }), + ); + return this.action(() => this.agentMayAct(agentId) && this.owner === null, run, signal); } agentCanActNow(agentId: string) { @@ -138,7 +156,7 @@ export class SessionControl { return run(); } - system(run: () => Promise) { + async system(run: () => Promise) { this.assertOpen(); return this.enqueue(async () => { this.assertOpen(); diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index 8c038f4fe81cc..5ef8ff6e7926e 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -792,6 +792,7 @@ export const PreviewAutomationControlReason = Schema.Literals([ "interrupted", "dialogPending", "tabLimit", + "busy", ]); export type PreviewAutomationControlReason = typeof PreviewAutomationControlReason.Type; @@ -814,6 +815,8 @@ export class PreviewAutomationControlInterruptedError extends Schema.TaggedError return "A browser dialog is pending. Read preview_status and resolve it with preview_dialog."; if (this.reason === "tabLimit") return "Too many server browser tabs are open. Close one with t3_preview_close, or reuse a tabId from preview_status tabs."; + if (this.reason === "busy") + return "This browser tab is busy with an earlier action that timed out. Open a new tab with preview_open and reuseExistingTab=false, or close this one with t3_preview_close."; if (this.reason === "closed") return "This browser tab is closed. Call preview_open."; if (this.reason === "interrupted") return "Browser control changed. Take a fresh snapshot before trying again."; From 3da8321f8c2131487ee0724c7f013157fc72a5e0 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 8 Nov 2026 07:14:32 +0000 Subject: [PATCH 03/10] fix(server): keep operation timeout reasons and mark stuck recording stops busy The server's request deadline now fires a little after the operation's own timeout, inside the broker's response grace, so a click's named-target reason and other operation errors still reach the agent. A recording stop that outlives its request leaves the tab busy while it still runs late. A pending dialog only overrides the busy refusal for a caller that may act on the tab. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/preview/ServerBrowser.test.ts | 60 +++++++++++++++++++ apps/server/src/preview/ServerBrowser.ts | 13 ++-- apps/server/src/preview/SessionControl.ts | 24 ++++++-- 3 files changed, 89 insertions(+), 8 deletions(-) diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index 85c4e2a42d6a1..7b6f59b4d7f9a 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -2529,3 +2529,63 @@ it.effect( }), ).pipe(Effect.provide(layer)), ); + +it.effect("an action that fails just after its timeout still reports its own failure", () => + Effect.scoped( + Effect.gen(function* () { + const { broker, tabId } = yield* ready; + const failing = Promise.withResolvers(); + const started = Promise.withResolvers(); + contexts[0]!.page.emulateMedia.mockImplementationOnce(() => { + started.resolve(); + return failing.promise; + }); + const failed = yield* broker + .invoke({ + scope, + tabId, + operation: "setColorScheme", + input: { colorScheme: "dark" }, + timeoutMs: 1_000, + }) + .pipe(Effect.flip, Effect.forkScoped); + yield* Effect.promise(() => started.promise); + // Operations time themselves out a little after the request arrives; their reason wins. + yield* TestClock.adjust("1100 millis"); + failing.reject(new Error("Target page crashed")); + expect(yield* Fiber.join(failed)).toMatchObject({ _tag: "PreviewAutomationExecutionError" }); + }), + ).pipe(Effect.provide(layer)), +); + +it.effect("a recording stop that never finishes leaves its tab busy", () => + Effect.scoped( + Effect.gen(function* () { + const { broker, tabId } = yield* ready; + const context = contexts[0]!; + yield* broker.invoke({ scope, tabId, operation: "recordingStart", input: {} }); + const started = Promise.withResolvers(); + context.sessions.at(-1)!.detach.mockImplementationOnce(() => { + started.resolve(); + return new Promise(() => {}); + }); + const stuck = yield* broker + .invoke({ scope, tabId, operation: "recordingStop", input: {}, timeoutMs: 1_000 }) + .pipe(Effect.flip, Effect.forkScoped); + yield* Effect.promise(() => started.promise); + yield* TestClock.adjust("2 seconds"); + expect(yield* Fiber.join(stuck)).toMatchObject({ _tag: "PreviewAutomationTimeoutError" }); + const busy = yield* broker + .invoke({ scope, tabId, operation: "click", input: { ref: "e1" }, timeoutMs: 1_000 }) + .pipe(Effect.flip, Effect.forkScoped); + // The host answers in arrival order, so a refused click has its answer before this one. + yield* broker.invoke({ scope, tabId, operation: "status", input: {} }); + // A click that queued instead of being refused would time out here. + yield* TestClock.adjust("2 seconds"); + expect(yield* Fiber.join(busy)).toMatchObject({ + _tag: "PreviewAutomationControlInterruptedError", + reason: "busy", + }); + }), + ).pipe(Effect.provide(layer)), +); diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index f7ecf6232b9c8..3aa74e0893b89 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -127,6 +127,8 @@ const EDITABLE_AT_POINT_SCRIPT = `(x, y) => { }`; const UNATTACHED_FILL_VIEWPORT = { width: 1280, height: 800 } as const; const NAVIGATION_TIMEOUT_MS = 15_000; +// Must stay under the broker's 1 s host response grace. +const HOST_ANSWER_MARGIN_MS = 500; const VIEWER_NAVIGATION_OPTIONS = { waitUntil: "commit", timeout: NAVIGATION_TIMEOUT_MS } as const; const ACTION_TIMELINE_LIMIT = 50; @@ -2131,7 +2133,9 @@ const make = Effect.gen(function* () { } // A stop cannot act twice (a retry finds no recording); // dropping a late one would leave capture running. - return tab.control.agent(request.agentSessionId ?? "", () => stopRecording(tab)); + return tab.control.agent(request.agentSessionId ?? "", () => stopRecording(tab), signal, { + runIfExpired: true, + }); } } const tab = await requireTab(request); @@ -2179,7 +2183,7 @@ const make = Effect.gen(function* () { }); } // An action blocked by a dialog outlives its request; point the agent at the dialog. - if (tab.dialog && tab.control.overdue) + if (tab.dialog && tab.control.overdue && tab.control.agentCanActNow(agentSessionId)) throw new BrowserControlInterrupted({ message: "A browser dialog is pending. Read preview_status and use preview_dialog first.", reason: "dialogPending", @@ -2307,9 +2311,10 @@ const make = Effect.gen(function* () { try: (signal) => runOperation(request, signal).finally(() => markUsed(request)), catch: ServerBrowserPage.toOperationError, }).pipe( - // Answer before the broker evicts this host; the abandoned run still holds its tab's queue. + // Lets the operation report its own timeout first, yet answers before the broker's + // response grace evicts this host. The abandoned run still holds its tab's queue. Effect.timeoutOrElse({ - duration: request.timeoutMs, + duration: request.timeoutMs + HOST_ANSWER_MARGIN_MS, orElse: () => Effect.fail( new ServerBrowserPage.ServerBrowserOperationError({ diff --git a/apps/server/src/preview/SessionControl.ts b/apps/server/src/preview/SessionControl.ts index ccb0ed92e2b7b..8f6cc0aebf6f6 100644 --- a/apps/server/src/preview/SessionControl.ts +++ b/apps/server/src/preview/SessionControl.ts @@ -94,7 +94,12 @@ export class SessionControl { return this.running?.aborted === true; } - private async action(allowed: () => boolean, run: () => Promise, signal?: AbortSignal) { + private async action( + allowed: () => boolean, + run: () => Promise, + signal?: AbortSignal, + runIfExpired = false, + ) { this.assertOpen(); if (!allowed()) throw new BrowserControlInterrupted({ message: "You do not control this browser tab." }); @@ -102,7 +107,7 @@ export class SessionControl { return this.enqueue(async () => { this.assertOpen(); if (epoch !== this.epoch || !allowed()) throw new BrowserControlInterrupted(); - if (signal?.aborted) { + if (signal?.aborted && !runIfExpired) { throw new DOMException( "This browser request expired before it could start.", "TimeoutError", @@ -121,7 +126,13 @@ export class SessionControl { return this.agentId === null || this.agentId === agentId; } - agent(agentId: string, run: () => Promise, signal?: AbortSignal) { + /** `runIfExpired` still starts work whose request expired while queued, for work that must not be dropped. */ + agent( + agentId: string, + run: () => Promise, + signal?: AbortSignal, + { runIfExpired = false }: { readonly runIfExpired?: boolean } = {}, + ) { if (!this.agentMayAct(agentId)) return Promise.reject( new BrowserControlInterrupted({ @@ -143,7 +154,12 @@ export class SessionControl { reason: "busy", }), ); - return this.action(() => this.agentMayAct(agentId) && this.owner === null, run, signal); + return this.action( + () => this.agentMayAct(agentId) && this.owner === null, + run, + signal, + runIfExpired, + ); } agentCanActNow(agentId: string) { From e0409994ee8f635d57a4565e7b35540ebdc41f3d Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 8 Nov 2026 07:31:04 +0000 Subject: [PATCH 04/10] fix(server): expire queued browser work at the request timeout The run's abort signal now fires at the request's own timeoutMs on the Effect clock, while the server still answers at timeoutMs plus its margin. Queued work whose request expired never starts, as before, and a started run past its timeout still marks the tab busy. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/preview/ServerBrowser.test.ts | 36 +++++++++++++++++++ apps/server/src/preview/ServerBrowser.ts | 35 +++++++++++------- 2 files changed, 59 insertions(+), 12 deletions(-) diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index 7b6f59b4d7f9a..e47a953c33d25 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -2589,3 +2589,39 @@ it.effect("a recording stop that never finishes leaves its tab busy", () => }), ).pipe(Effect.provide(layer)), ); + +it.effect("a queued action whose timeout passes while it waits never runs", () => + Effect.scoped( + Effect.gen(function* () { + const { broker, tabId } = yield* ready; + const page = contexts[0]!.page; + const started = Promise.withResolvers(); + const finish = Promise.withResolvers(); + page.emulateMedia.mockImplementationOnce(() => { + started.resolve(); + return finish.promise; + }); + const slow = yield* broker + .invoke({ + scope, + tabId, + operation: "setColorScheme", + input: { colorScheme: "dark" }, + timeoutMs: 3_000, + }) + .pipe(Effect.forkScoped); + yield* Effect.promise(() => started.promise); + const queued = yield* broker + .invoke({ scope, tabId, operation: "click", input: { ref: "e1" }, timeoutMs: 1_000 }) + .pipe(Effect.flip, Effect.forkScoped); + // The host answers in arrival order, so the click is queued once this answers. + yield* broker.invoke({ scope, tabId, operation: "status", input: {} }); + // Past the click's timeout but before the server answers it. + yield* TestClock.adjust("1200 millis"); + finish.resolve(); + yield* Fiber.join(slow); + expect(yield* Fiber.join(queued)).toMatchObject({ _tag: "PreviewAutomationTimeoutError" }); + expect(page.locator).not.toHaveBeenCalled(); + }), + ).pipe(Effect.provide(layer)), +); diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 3aa74e0893b89..3820431fa0eee 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -2306,9 +2306,19 @@ const make = Effect.gen(function* () { } }; - const handleRequest = (connectionId: string, request: PreviewAutomationRequest) => - Effect.tryPromise({ - try: (signal) => runOperation(request, signal).finally(() => markUsed(request)), + const handleRequest = Effect.fn("ServerBrowser.handleRequest")(function* ( + connectionId: string, + request: PreviewAutomationRequest, + ) { + // Queued work expires at the request's timeout; a started run past it marks its tab busy. + const expiry = yield* Scope.make(); + const signal = yield* Effect.abortSignal.pipe(Scope.provide(expiry)); + yield* Effect.sleep(request.timeoutMs).pipe( + Effect.andThen(Scope.close(expiry, Exit.void)), + Effect.forkChild, + ); + const outcome = yield* Effect.tryPromise({ + try: () => runOperation(request, signal).finally(() => markUsed(request)), catch: ServerBrowserPage.toOperationError, }).pipe( // Lets the operation report its own timeout first, yet answers before the broker's @@ -2324,6 +2334,7 @@ const make = Effect.gen(function* () { }), ), }), + Effect.ensuring(Scope.close(expiry, Exit.void)), Effect.match({ onSuccess: (result) => ({ ok: true as const, result }), onFailure: (error) => ({ @@ -2335,16 +2346,16 @@ const make = Effect.gen(function* () { }, }), }), - Effect.flatMap((outcome) => - broker.respond({ - clientId: SERVER_HOST_CLIENT_ID, - connectionId, - requestId: request.requestId, - ...outcome, - }), - ), - Effect.ignore, ); + yield* broker + .respond({ + clientId: SERVER_HOST_CLIENT_ID, + connectionId, + requestId: request.requestId, + ...outcome, + }) + .pipe(Effect.ignore); + }); const mirrorManagerEvent = (event: PreviewEvent) => Effect.promise(async () => { From b088b88d19fb0edc43f754da2e1d199bda70d6d8 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 10 Nov 2026 11:20:12 +0000 Subject: [PATCH 05/10] fix(server): a recording stop on a stuck tab reports a pending dialog first Co-Authored-By: Claude Opus 5.5 --- apps/server/src/preview/ServerBrowser.test.ts | 15 +++++++++++++-- apps/server/src/preview/ServerBrowser.ts | 17 +++++++++++------ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index 97b65c0d2729c..ccf32dfea9f2e 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -2578,6 +2578,10 @@ it.effect("a tab stuck in an agent action does not hold back setting changes for input: { reuseExistingTab: false, show: false }, }); const stalled = stallColorScheme(contexts[0]!.page); + const applied = Promise.withResolvers(); + contexts[1]!.page.emulateMedia.mockImplementationOnce(async (options: unknown) => { + applied.resolve(options); + }); yield* broker .invoke({ scope, @@ -2594,8 +2598,7 @@ it.effect("a tab stuck in an agent action does not hold back setting changes for tabId: PreviewTabId.make(other.tabId!), colorScheme: "dark", }); - yield* TestClock.adjust("2 seconds"); - expect(contexts[1]!.page.emulateMedia).toHaveBeenCalledWith({ colorScheme: "dark" }); + expect(yield* Effect.promise(() => applied.promise)).toEqual({ colorScheme: "dark" }); }), ).pipe(Effect.provide(layer)), ); @@ -2607,6 +2610,7 @@ it.effect( Effect.gen(function* () { const { broker, tabId } = yield* ready; const page = contexts[0]!.page; + yield* broker.invoke({ scope, tabId, operation: "recordingStart", input: {} }); const stalled = stallColorScheme(page); const stuck = yield* broker .invoke({ @@ -2652,6 +2656,13 @@ it.effect( _tag: "PreviewAutomationControlInterruptedError", reason: "dialogPending", }); + const stop = yield* broker + .invoke({ scope, tabId, operation: "recordingStop", input: {}, timeoutMs: 1_000 }) + .pipe(Effect.flip); + expect(stop).toMatchObject({ + _tag: "PreviewAutomationControlInterruptedError", + reason: "dialogPending", + }); }), ).pipe(Effect.provide(layer)), ); diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index b415e1e7506a8..7071a0d40c3d2 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -1930,6 +1930,15 @@ const make = Effect.gen(function* () { ); }; + // Work blocked by a dialog outlives its request; point the agent at the dialog, not at `busy`. + const requireNoOverdueDialog = (tab: ServerTab, agentSessionId: string) => { + if (tab.dialog && tab.control.overdue && tab.control.agentCanActNow(agentSessionId)) + throw new BrowserControlInterrupted({ + message: "A browser dialog is pending. Read preview_status and use preview_dialog first.", + reason: "dialogPending", + }); + }; + const requireTab = async (request: PreviewAutomationRequest) => { if (!request.agentSessionId) throw new BrowserControlInterrupted({ @@ -2137,6 +2146,7 @@ const make = Effect.gen(function* () { message: "No recording is active for this thread.", }); } + requireNoOverdueDialog(tab, request.agentSessionId ?? ""); // A stop cannot act twice (a retry finds no recording); // dropping a late one would leave capture running. return tab.control.agent(request.agentSessionId ?? "", () => stopRecording(tab), signal, { @@ -2188,12 +2198,7 @@ const make = Effect.gen(function* () { return executeTabOperation(tab, request); }); } - // An action blocked by a dialog outlives its request; point the agent at the dialog. - if (tab.dialog && tab.control.overdue && tab.control.agentCanActNow(agentSessionId)) - throw new BrowserControlInterrupted({ - message: "A browser dialog is pending. Read preview_status and use preview_dialog first.", - reason: "dialogPending", - }); + requireNoOverdueDialog(tab, agentSessionId); return tab.control.agent( agentSessionId, async () => { From baf65eec05c713cec009f407ff1af24ba42e36a6 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 10 Nov 2026 11:37:52 +0000 Subject: [PATCH 06/10] test(evals): busy preview tab recovery case and an ordinary-timeout control The busy control reason added for #32849 is new agent-facing tool-result text. preview-busy-tab-opens-new-tab resumes a thread right after preview_act returned the busy error, and passes when the first tab-touching preview call is preview_open with reuseExistingTab=false or t3_preview_close on the stuck tab. preview-wait-timeout-keeps-tab is its control: after a plain wait_for timeout the agent keeps working in the same tab. The runner now replays a prior result that decodes as a contracts PreviewAutomationError as T3 sends it, an error result carrying the class's own message, so the case never copies the text. A new firstCall check grades the first call among a set of tools against alternatives. Co-Authored-By: Claude Opus 5.5 --- evals/mcp-tool-use/README.md | 3 +- .../mcp-tool-use/cases/visual-preview.ndjson | 2 ++ evals/mcp-tool-use/run.test.ts | 30 ++++++++++++++++++- evals/mcp-tool-use/run.ts | 24 ++++++++++++++- 4 files changed, 56 insertions(+), 3 deletions(-) diff --git a/evals/mcp-tool-use/README.md b/evals/mcp-tool-use/README.md index 7df44b7f82165..34e2520dc331d 100644 --- a/evals/mcp-tool-use/README.md +++ b/evals/mcp-tool-use/README.md @@ -27,8 +27,9 @@ Cases are JSON lines in `cases/`, one file per area. The case schema in `run.ts` - `kind`: `bad` reproduces a mistake, `guard` protects correct behavior, `control` is ordinary work. - `experience`: the real mistake, in a sentence. `source`: where it happened. - `setup`: the fake thread, its `world` (repo, branch, files, PRs, pages), `priorTurns`, and `mocks` for tool results. +- `setup.priorTurns[].toolCalls[].result`: a T3 preview error (`{"_tag": "PreviewAutomationControlInterruptedError", ...}`, every field the contracts class needs) replays as T3 sends it: an error result with the class's own message. - `setup.priorTurns[].toolCalls[].resultFile`: load a repository-relative file as the tool result instead of copying its contents into the case. An empty `prompt` resumes a top-level thread after its last prior tool result without adding a user message. Side-chat and subagent setups require a nonempty prompt. -- `expect`: the checks, such as `calls`, `callsMustNot`, `maxCalls`, `order`, `shellMust`, `textMust`, `finalTextMust` (the last text, once the turn ended), and `linked`. A call's `args` match partially: `~text` is a case-insensitive substring, `"<=N"` / `">=N"` bound a number, `null` matches a missing key, and keys may be dotted paths. Device cases also use `device` to check every child brief, forbid raw UI commands, bound failed device actions per user turn, and require artifact paths in completed final answers. Secrets cases use `secret` (see [Secrets cases](#secrets-cases)). +- `expect`: the checks, such as `calls`, `callsMustNot`, `maxCalls`, `order`, `firstCall` (the first call of any of its `tools` matches one of its `calls`), `shellMust`, `textMust`, `finalTextMust` (the last text, once the turn ended), and `linked`. A call's `args` match partially: `~text` is a case-insensitive substring, `"<=N"` / `">=N"` bound a number, `null` matches a missing key, and keys may be dotted paths. Device cases also use `device` to check every child brief, forbid raw UI commands, bound failed device actions per user turn, and require artifact paths in completed final answers. Secrets cases use `secret` (see [Secrets cases](#secrets-cases)). Check that the case builds with `--dry-run --case `. Then run it on the text before your fix, where it must fail, and on the text after, where it must pass. diff --git a/evals/mcp-tool-use/cases/visual-preview.ndjson b/evals/mcp-tool-use/cases/visual-preview.ndjson index edd66f26966e9..811837e1263cf 100644 --- a/evals/mcp-tool-use/cases/visual-preview.ndjson +++ b/evals/mcp-tool-use/cases/visual-preview.ndjson @@ -34,3 +34,5 @@ {"id": "browser-status-reuses-existing-tab", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "pattern": "usable Browser tab needs no new tab", "experience": "An already paired, usable tab should be reused during verification.", "source": {"report": "Maintainer browser verification failure, 2026-11-03"}, "setup": {"role": "top", "context": "The UI fix is complete. The test server is healthy at http://localhost:5173 and its browser profile is already paired. The current tab can be automated.", "mocks": {"preview_status": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 0}}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://localhost:5173/"}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/", "screenshotPath": "/tmp/after.png"}}, "priorTurns": [{"role": "user", "text": "Use the Browser panel to verify the UI fix and capture an after screenshot."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "cat .agents/skills/test-t3-app/SKILL.md"}, "result": "", "resultFile": ".agents/skills/test-t3-app/SKILL.md"}]}, {"role": "assistant", "toolCalls": [{"tool": "preview_status", "input": {}, "result": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 0}}}]}], "harness": "claude"}, "prompt": "", "expect": {"calls": [{"tool": "preview_snapshot"}], "callsMustNot": [{"tool": "preview_open", "args": {"reuseExistingTab": false}}]}, "maxSteps": 10, "notes": "Checks reuse of an existing usable tab. Screenshot saving is outside this control. Prior tool-result replay is validated on the Claude harness; Codex output wrapping needs separate validation."} {"id": "browser-resume-after-unavailable-status", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "closed Browser panel treated as unavailable automation", "experience": "The agent read test-t3-app, saw available false with no tab, and asked the user to open the panel instead of calling preview_open. Opening a tab succeeded on the next turn.", "source": {"report": "Maintainer browser verification failure, 2026-11-03; fresh skill read followed by unavailable status. Opus replay reproduces the stop on the old skill."}, "setup": {"role": "top", "context": "Repository instructions: Before web or desktop UI verification, read .agents/skills/test-t3-app/SKILL.md and follow it. If a skill causes you to stop or leave requested work unfinished, link the SKILL.md, quote the relevant instruction, and explain how it applies.", "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://localhost:5173/"}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/", "screenshotPath": "/tmp/after.png"}}, "world": {"repo": "acme/app", "branch": "fix/permissions-popup", "status": "", "files": {"apps/web/src/components/ComposerContextPermissions.tsx": "export function ComposerContextPermissions() { return ; }\n"}, "description": "The requested fix is already committed as a1b2c3d on fix/permissions-popup, ahead of main. It changes ComposerContextPermissions.tsx to open the permissions popup above its trigger. Dependencies are installed. No development server is running and no .t3 directory exists.", "commits": [{"sha": "a1b2c3d", "subject": "fix(web): open composer permissions above its button"}]}, "priorTurns": [{"role": "user", "text": "When I click Full access in the composer, open its permissions menu directly above that button."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "git commit -am \"fix(web): open composer permissions above its button\""}, "result": "[fix/permissions-popup a1b2c3d] fix(web): open composer permissions above its button\n 1 file changed, 3 insertions(+), 1 deletion(-)"}]}, {"role": "assistant", "text": "Implemented. Clicking Full access opens the permissions menu above the button. Focused checks pass. Full-client visual verification remains undone."}, {"role": "user", "text": "before/after screenshot"}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "cat .agents/skills/test-t3-app/SKILL.md"}, "result": "", "resultFile": ".agents/skills/test-t3-app/SKILL.md"}]}, {"role": "assistant", "text": "I'll use the test-t3-app skill to capture the before and after states in the app."}, {"role": "assistant", "toolCalls": [{"tool": "preview_status", "input": {}, "result": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}}]}], "shellMocks": [{"match": "git status|ls .*\\.t3", "output": "ls: .t3: No such file or directory"}], "harness": "claude"}, "prompt": "", "expect": {"calls": [{"tool": "preview_open"}]}, "maxSteps": 8, "notes": "Resume at the failed decision after reading the skill and receiving available=false. Opus reproduces the premature stop; Sonnet did not reliably reproduce it. This checks recovery through preview_open, not completion of screenshots. Prior tool-result replay is validated on the Claude harness; Codex output wrapping needs separate validation."} {"id": "revise-page-resends-html", "area": "visual-preview", "kind": "bad", "pattern": "revising a page on disk resends its full HTML to html_preview and html_render", "experience": "The pages already existed as generated files on disk, but the agent retyped each one into html_preview and again into html_render, about four minutes per call. To save time it published fixes without previewing them again.", "source": {"discussion": "https://github.com/pingdotgg/t3code/discussions/23303"}, "rule": "edit the .html file and pass its path to html_preview and html_render instead of resending its html", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "The new-worktree toggle is ugly. Come up with four better options and mock them."}, {"role": "assistant", "text": "Here are four options.", "toolCalls": [{"tool": "Write", "input": {"file_path": "/tmp/design/index.html", "content": "\n

A. Checkbox under the composer

Ask anything
\n

B. Toggle in the toolbar

Ask anything
\n

C. Split send button

Ask anything
Send in new worktree
\n

D. Two visible choices

Ask anything
\n\n"}, "result": "File created successfully at: /tmp/design/index.html"}, {"tool": "html_preview", "input": {"html": "\n

A. Checkbox under the composer

Ask anything
\n

B. Toggle in the toolbar

Ask anything
\n

C. Split send button

Ask anything
Send in new worktree
\n

D. Two visible choices

Ask anything
\n\n"}, "result": "{\"width\":728,\"contentHeight\":1840,\"capturedHeight\":1840,\"consoleMessages\":[]}"}, {"tool": "html_render", "input": {"html": "\n

A. Checkbox under the composer

Ask anything
\n

B. Toggle in the toolbar

Ask anything
\n

C. Split send button

Ask anything
Send in new worktree
\n

D. Two visible choices

Ask anything
\n\n", "title": "New worktree: four options", "height": 1840}, "result": "{\"htmlRender\":{\"attachmentId\":\"att-1\",\"title\":\"New worktree: four options\",\"height\":1840},\"message\":\"Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say.\"}"}]}], "mocks": {"html_preview": {"width": 728, "contentHeight": 1840, "capturedHeight": 1840, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "New worktree: four options", "height": 1840}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}, "world": {"files": {"/tmp/design/index.html": "\n

A. Checkbox under the composer

Ask anything
\n

B. Toggle in the toolbar

Ask anything
\n

C. Split send button

Ask anything
Send in new worktree
\n

D. Two visible choices

Ask anything
\n\n"}}}, "prompt": "Make the Send in new worktree button in option C blue, and show me the page again.", "expect": {"calls": [{"tool": "html_preview", "args": {"path": "/tmp/design/index.html", "html": null}}, {"tool": "html_render", "args": {"path": "/tmp/design/index.html", "html": null}}], "callsMustNot": [{"tool": "html_preview", "args": {"html": "~Ask anything"}}, {"tool": "html_render", "args": {"html": "~Ask anything"}}], "order": ["html_preview", "html_render"]}, "notes": "On main neither tool has a path parameter."} +{"id": "preview-busy-tab-opens-new-tab", "area": "visual-preview", "kind": "bad", "pattern": "preview tab stuck behind a timed-out action; the next preview call retries the same tab", "experience": "A server-browser action never finished, so every later call on that tab timed out with the same generic error and the agent kept retrying the stuck tab.", "rule": "On a busy tab, continue in a new tab (preview_open with reuseExistingTab=false) or close it with t3_preview_close; do not retry it.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#32849: a browser action that never finishes wedges its tab; the agent only sees a generic timeout and keeps retrying the same tab"}, "setup": {"role": "top", "context": "The dev server is healthy at http://localhost:5173. The agent opened preview tab tab_1 (owner agent) earlier in this turn.", "priorTurns": [{"role": "user", "text": "Open http://localhost:5173/settings in the browser, switch it to dark mode, click Save, and check that the settings-saved toast still reads well in dark mode."}, {"role": "assistant", "toolCalls": [{"tool": "preview_open", "input": {"url": "http://localhost:5173/settings"}, "result": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "loading": false}}, {"tool": "preview_set_appearance", "input": {"tabId": "tab_1", "colorScheme": "dark"}, "result": {"_tag": "PreviewAutomationTimeoutError", "operation": "setColorScheme", "environmentId": "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee", "threadId": "thread-settings-check", "providerSessionId": "session-1", "providerInstanceId": "claudeAgent", "clientId": "server-browser", "connectionId": "conn-1", "requestId": "req-2", "tabId": "tab_1", "timeoutMs": 15000}}, {"tool": "preview_act", "input": {"tabId": "tab_1", "action": "click", "locator": "aria-ref=e12", "timeoutMs": 3000}, "result": {"_tag": "PreviewAutomationControlInterruptedError", "operation": "click", "environmentId": "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee", "threadId": "thread-settings-check", "providerSessionId": "session-1", "providerInstanceId": "claudeAgent", "clientId": "server-browser", "connectionId": "conn-1", "requestId": "req-3", "tabId": "tab_1", "timeoutMs": 3000, "remoteTag": "BrowserControlInterrupted", "remoteMessageLength": 0, "cause": null, "reason": "busy"}}]}], "mocks": {"preview_open": {"available": true, "visible": false, "tabId": "tab_2", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "loading": false}, "preview_status": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "loading": false, "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/settings", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 1}}, "t3_preview_list": {"tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/settings", "owner": "agent"}]}, "preview_snapshot": {"available": true, "tabId": "tab_2", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "snapshot": "- heading \"Settings\" [level=1]\n- switch \"Dark mode\" [checked] [ref=e7]\n- button \"Save\" [ref=e12]"}}}, "prompt": "", "expect": {"firstCall": {"tools": ["preview_act", "preview_resize", "preview_navigate", "preview_snapshot", "preview_wait_for", "preview_evaluate", "preview_set_appearance", "preview_open", "t3_preview_close"], "calls": [{"tool": "preview_open", "args": {"reuseExistingTab": false}}, {"tool": "t3_preview_close", "args": {"tabId": "tab_1"}}]}}, "maxSteps": 6, "notes": "The prior preview_act result is the contracts busy error, rendered with its real message. preview_status and t3_preview_list are reads, so they may come first."} +{"id": "preview-wait-timeout-keeps-tab", "area": "visual-preview", "kind": "control", "pattern": "an ordinary wait_for timeout; the agent keeps working in the same tab", "experience": "Guards #32849's busy-tab recovery: a plain timeout on a responsive tab is a page result, not a reason to abandon the tab.", "rule": "After an ordinary preview timeout, inspect or retry in the same tab.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#32849 (control)"}, "setup": {"role": "top", "context": "The dev server is healthy at http://localhost:5173. The agent opened preview tab tab_1 (owner agent) earlier in this turn.", "priorTurns": [{"role": "user", "text": "Open http://localhost:5173/settings in the browser, switch it to dark mode, click Save, and check that the settings-saved toast still reads well in dark mode."}, {"role": "assistant", "toolCalls": [{"tool": "preview_open", "input": {"url": "http://localhost:5173/settings"}, "result": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "loading": false}}, {"tool": "preview_set_appearance", "input": {"tabId": "tab_1", "colorScheme": "dark"}, "result": {"available": true, "tabId": "tab_1", "colorScheme": "dark"}}, {"tool": "preview_act", "input": {"tabId": "tab_1", "action": "click", "locator": "aria-ref=e12"}, "result": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/settings"}}, {"tool": "preview_wait_for", "input": {"tabId": "tab_1", "text": "Settings saved", "timeoutMs": 5000}, "result": {"_tag": "PreviewAutomationTimeoutError", "operation": "waitFor", "environmentId": "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee", "threadId": "thread-settings-check", "providerSessionId": "session-1", "providerInstanceId": "claudeAgent", "clientId": "server-browser", "connectionId": "conn-1", "requestId": "req-4", "tabId": "tab_1", "timeoutMs": 5000}}]}], "mocks": {"preview_open": {"available": true, "visible": false, "tabId": "tab_2", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "loading": false}, "preview_status": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "loading": false, "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/settings", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 1}}, "t3_preview_list": {"tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/settings", "owner": "agent"}]}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "snapshot": "- heading \"Settings\" [level=1]\n- switch \"Dark mode\" [checked] [ref=e7]\n- button \"Save\" [ref=e12]\n- status: \"Saved\""}}}, "prompt": "", "expect": {"firstCall": {"tools": ["preview_act", "preview_resize", "preview_navigate", "preview_snapshot", "preview_wait_for", "preview_evaluate", "preview_set_appearance", "preview_open", "t3_preview_close"], "calls": [{"tool": "preview_act|preview_snapshot|preview_wait_for|preview_evaluate|preview_navigate|preview_resize|preview_set_appearance"}]}}, "maxSteps": 6} diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts index 6e9413ed9141e..e48ccb329ee2b 100644 --- a/evals/mcp-tool-use/run.test.ts +++ b/evals/mcp-tool-use/run.test.ts @@ -7,7 +7,12 @@ import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; import { FetchHttpClient } from "effect/http"; import { assert, expect, it } from "@effect/vitest"; -import { ComposerContextId, EnvironmentId, ThreadId } from "@t3tools/contracts"; +import { + ComposerContextId, + EnvironmentId, + PreviewAutomationControlInterruptedError, + ThreadId, +} from "@t3tools/contracts"; import { projectComposerContextForProvider } from "@t3tools/shared/composerContextReferences"; import { @@ -244,6 +249,10 @@ it("passes a good run and names every rule a bad run breaks", () => { callsMustNot: [{ tool: "delegate_task", args: { runtimeMode: "approval-required" } }], notCalls: ["t3_thread_send"], order: ["orchestrator_capabilities", "delegate_task"], + firstCall: { + tools: ["delegate_task", "t3_thread_send"], + calls: [{ tool: "t3_thread_send" }, { tool: "delegate_task", args: { task: "~FINDING" } }], + }, minCalls: { orchestrator_capabilities: 1 }, maxCalls: { task_status: 0 }, endsTurnAfter: "delegate_task", @@ -287,6 +296,7 @@ it("passes a good run and names every rule a bad run breaks", () => { 'callsMustNot delegate_task {"runtimeMode":"approval-required"}', "notCalls t3_thread_send", "order orchestrator_capabilities > delegate_task", + 'firstCall t3_thread_send or delegate_task {"task":"~FINDING"}', "minCalls orchestrator_capabilities >= 1", "maxCalls task_status <= 0", "endsTurnAfter delegate_task", @@ -578,6 +588,24 @@ it("lists link tools only while a case turns the beta on, and sends chips as T3 assert.include(prompt?.text, "environmentId: environment-vps"); }); +const decodeInterrupted = Schema.decodeUnknownSync(PreviewAutomationControlInterruptedError); +it("replays an earlier preview error as T3's error result, with its contracts message", () => { + const base = buildBase(template, { model: "claude-opus-5-5", keepUserInstructions: false }); + const busyCase = NodeFS.readFileSync( + new URL("./cases/visual-preview.ndjson", import.meta.url), + "utf8", + ) + .trim() + .split("\n") + .map((line) => decodeDeviceCase(line)) + .find((c) => c.id === "preview-busy-tab-opens-new-tab")!; + const prior = busyCase.setup.priorTurns!.at(-1)!.toolCalls!.at(-1)!; + const error = decodeInterrupted(prior.result); + assert.equal(error.reason, "busy"); + const replayed = buildMessages(makeSample(busyCase, base)).at(-1)?.content.at(-1); + assert.deepInclude(replayed, { content: error.message, is_error: true }); +}); + it("replays earlier capability results with the model under test as the caller", () => { const base = buildBase(template, { model: "claude-opus-5-5", keepUserInstructions: false }); const capabilities = { diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts index 5a04303123db4..b59e52ace9821 100644 --- a/evals/mcp-tool-use/run.ts +++ b/evals/mcp-tool-use/run.ts @@ -18,6 +18,7 @@ import { DeviceOperationError, DeviceSummary, OrchestratorMcpTaskRole, + PreviewAutomationError, ProviderInstanceId, } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; @@ -190,6 +191,10 @@ const McpToolUseExpect = Schema.Struct({ notCalls: Schema.optional(Schema.Array(Schema.String)), /** First calls of these tools happen in this order. */ order: Schema.optional(Schema.Array(Schema.String)), + /** The first call of any tool in `tools` matches one of `calls`. */ + firstCall: Schema.optional( + Schema.Struct({ tools: Schema.Array(Schema.String), calls: Schema.Array(McpToolUseCall) }), + ), minCalls: Schema.optional(Schema.Record(Schema.String, Schema.Number)), maxCalls: Schema.optional(Schema.Record(Schema.String, Schema.Number)), /** @@ -388,6 +393,10 @@ const toolResult = (id: string, content: unknown): Block => ({ tool_use_id: id, content: valueText(content), }); +const decodePreviewError = Schema.decodeUnknownOption(PreviewAutomationError); +/** A prior result that is a T3 preview error, `{"_tag": "PreviewAutomationTimeoutError", ...}`. */ +const previewFailure = (value: unknown) => + Option.getOrUndefined(Option.map(decodePreviewError(value), (error) => error.message)); const texts = (content: typeof Content.Type) => typeof content === "string" ? [content] : content.flatMap((block) => block.text ?? []); const isT3Name = (name: string) => name.startsWith(T3_PREFIX) || name.startsWith("mcp__t3_code__"); @@ -1008,7 +1017,14 @@ export const buildMessages = (sample: Sample) => { normalizeToolName(tool.name) === "orchestrator_capabilities" ? capabilities(sample, result) : (sample.priorResults.get(prior) ?? result); - push("user", toolResult(id, replayed)); + // T3 sends a preview tool's failure as an error result with only its message. + const failure = previewFailure(replayed); + push( + "user", + failure === undefined + ? toolResult(id, replayed) + : { ...toolResult(id, failure), is_error: true }, + ); } if (turn.text) push(turn.role, text(turn.text)); } @@ -1782,6 +1798,12 @@ export const gradeSample = (expect: McpToolUseCase["expect"], run: SampleRun, pa const inOrder = firsts.every((at, i) => at > (firsts[i - 1] ?? -1)); add(`order ${expect.order.join(" > ")}`, inOrder); } + if (expect.firstCall !== undefined) { + const { tools, calls: allowed } = expect.firstCall; + const first = calls.find((call) => tools.includes(call.tool)); + const ok = first !== undefined && allowed.some((call) => found(call, [first])); + add(`firstCall ${allowed.map(label).join(" or ")}`, ok); + } for (const [t, n] of Object.entries(expect.minCalls ?? {})) add(`minCalls ${t} >= ${n}`, count(t) >= n); for (const [t, n] of Object.entries(expect.maxCalls ?? {})) From 7d371399233f867230e75d07138409b736b4f002 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 10 Nov 2026 11:53:16 +0000 Subject: [PATCH 07/10] test(server): type the cross-tab mirror milestone mock Co-Authored-By: Claude Opus 5.5 --- apps/server/src/preview/ServerBrowser.test.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index ccf32dfea9f2e..b1a269c822571 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -2578,10 +2578,8 @@ it.effect("a tab stuck in an agent action does not hold back setting changes for input: { reuseExistingTab: false, show: false }, }); const stalled = stallColorScheme(contexts[0]!.page); - const applied = Promise.withResolvers(); - contexts[1]!.page.emulateMedia.mockImplementationOnce(async (options: unknown) => { - applied.resolve(options); - }); + const applied = Promise.withResolvers(); + contexts[1]!.page.emulateMedia.mockImplementationOnce(async () => applied.resolve()); yield* broker .invoke({ scope, @@ -2598,7 +2596,8 @@ it.effect("a tab stuck in an agent action does not hold back setting changes for tabId: PreviewTabId.make(other.tabId!), colorScheme: "dark", }); - expect(yield* Effect.promise(() => applied.promise)).toEqual({ colorScheme: "dark" }); + yield* Effect.promise(() => applied.promise); + expect(contexts[1]!.page.emulateMedia).toHaveBeenCalledWith({ colorScheme: "dark" }); }), ).pipe(Effect.provide(layer)), ); From cbda26eb67b4edabcc0aad6588ebc6bf72484722 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 10 Nov 2026 12:13:05 +0000 Subject: [PATCH 08/10] test: busy-tab eval cases grade the tab acted on; a late queued recording stop still runs Co-Authored-By: Claude Opus 5.5 --- apps/server/src/preview/ServerBrowser.test.ts | 7 ++++++ .../mcp-tool-use/cases/visual-preview.ndjson | 4 ++-- evals/mcp-tool-use/run.test.ts | 24 +++++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index b1a269c822571..32075e8ca69c2 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -2731,6 +2731,7 @@ it.effect("a queued action whose timeout passes while it waits never runs", () = Effect.gen(function* () { const { broker, tabId } = yield* ready; const page = contexts[0]!.page; + yield* broker.invoke({ scope, tabId, operation: "recordingStart", input: {} }); const started = Promise.withResolvers(); const finish = Promise.withResolvers(); page.emulateMedia.mockImplementationOnce(() => { @@ -2750,6 +2751,9 @@ it.effect("a queued action whose timeout passes while it waits never runs", () = const queued = yield* broker .invoke({ scope, tabId, operation: "click", input: { ref: "e1" }, timeoutMs: 1_000 }) .pipe(Effect.flip, Effect.forkScoped); + const stop = yield* broker + .invoke({ scope, tabId, operation: "recordingStop", input: {}, timeoutMs: 1_000 }) + .pipe(Effect.exit, Effect.forkScoped); // The host answers in arrival order, so the click is queued once this answers. yield* broker.invoke({ scope, tabId, operation: "status", input: {} }); // Past the click's timeout but before the server answers it. @@ -2758,6 +2762,9 @@ it.effect("a queued action whose timeout passes while it waits never runs", () = yield* Fiber.join(slow); expect(yield* Fiber.join(queued)).toMatchObject({ _tag: "PreviewAutomationTimeoutError" }); expect(page.locator).not.toHaveBeenCalled(); + // A late stop still runs, or capture would keep going with no recording to stop. + yield* Fiber.join(stop); + expect(contexts[0]!.sessions.at(-1)!.detach).toHaveBeenCalled(); }), ).pipe(Effect.provide(layer)), ); diff --git a/evals/mcp-tool-use/cases/visual-preview.ndjson b/evals/mcp-tool-use/cases/visual-preview.ndjson index 811837e1263cf..33854939c12cb 100644 --- a/evals/mcp-tool-use/cases/visual-preview.ndjson +++ b/evals/mcp-tool-use/cases/visual-preview.ndjson @@ -34,5 +34,5 @@ {"id": "browser-status-reuses-existing-tab", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "pattern": "usable Browser tab needs no new tab", "experience": "An already paired, usable tab should be reused during verification.", "source": {"report": "Maintainer browser verification failure, 2026-11-03"}, "setup": {"role": "top", "context": "The UI fix is complete. The test server is healthy at http://localhost:5173 and its browser profile is already paired. The current tab can be automated.", "mocks": {"preview_status": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 0}}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://localhost:5173/"}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/", "screenshotPath": "/tmp/after.png"}}, "priorTurns": [{"role": "user", "text": "Use the Browser panel to verify the UI fix and capture an after screenshot."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "cat .agents/skills/test-t3-app/SKILL.md"}, "result": "", "resultFile": ".agents/skills/test-t3-app/SKILL.md"}]}, {"role": "assistant", "toolCalls": [{"tool": "preview_status", "input": {}, "result": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 0}}}]}], "harness": "claude"}, "prompt": "", "expect": {"calls": [{"tool": "preview_snapshot"}], "callsMustNot": [{"tool": "preview_open", "args": {"reuseExistingTab": false}}]}, "maxSteps": 10, "notes": "Checks reuse of an existing usable tab. Screenshot saving is outside this control. Prior tool-result replay is validated on the Claude harness; Codex output wrapping needs separate validation."} {"id": "browser-resume-after-unavailable-status", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "closed Browser panel treated as unavailable automation", "experience": "The agent read test-t3-app, saw available false with no tab, and asked the user to open the panel instead of calling preview_open. Opening a tab succeeded on the next turn.", "source": {"report": "Maintainer browser verification failure, 2026-11-03; fresh skill read followed by unavailable status. Opus replay reproduces the stop on the old skill."}, "setup": {"role": "top", "context": "Repository instructions: Before web or desktop UI verification, read .agents/skills/test-t3-app/SKILL.md and follow it. If a skill causes you to stop or leave requested work unfinished, link the SKILL.md, quote the relevant instruction, and explain how it applies.", "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://localhost:5173/"}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/", "screenshotPath": "/tmp/after.png"}}, "world": {"repo": "acme/app", "branch": "fix/permissions-popup", "status": "", "files": {"apps/web/src/components/ComposerContextPermissions.tsx": "export function ComposerContextPermissions() { return ; }\n"}, "description": "The requested fix is already committed as a1b2c3d on fix/permissions-popup, ahead of main. It changes ComposerContextPermissions.tsx to open the permissions popup above its trigger. Dependencies are installed. No development server is running and no .t3 directory exists.", "commits": [{"sha": "a1b2c3d", "subject": "fix(web): open composer permissions above its button"}]}, "priorTurns": [{"role": "user", "text": "When I click Full access in the composer, open its permissions menu directly above that button."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "git commit -am \"fix(web): open composer permissions above its button\""}, "result": "[fix/permissions-popup a1b2c3d] fix(web): open composer permissions above its button\n 1 file changed, 3 insertions(+), 1 deletion(-)"}]}, {"role": "assistant", "text": "Implemented. Clicking Full access opens the permissions menu above the button. Focused checks pass. Full-client visual verification remains undone."}, {"role": "user", "text": "before/after screenshot"}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "cat .agents/skills/test-t3-app/SKILL.md"}, "result": "", "resultFile": ".agents/skills/test-t3-app/SKILL.md"}]}, {"role": "assistant", "text": "I'll use the test-t3-app skill to capture the before and after states in the app."}, {"role": "assistant", "toolCalls": [{"tool": "preview_status", "input": {}, "result": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}}]}], "shellMocks": [{"match": "git status|ls .*\\.t3", "output": "ls: .t3: No such file or directory"}], "harness": "claude"}, "prompt": "", "expect": {"calls": [{"tool": "preview_open"}]}, "maxSteps": 8, "notes": "Resume at the failed decision after reading the skill and receiving available=false. Opus reproduces the premature stop; Sonnet did not reliably reproduce it. This checks recovery through preview_open, not completion of screenshots. Prior tool-result replay is validated on the Claude harness; Codex output wrapping needs separate validation."} {"id": "revise-page-resends-html", "area": "visual-preview", "kind": "bad", "pattern": "revising a page on disk resends its full HTML to html_preview and html_render", "experience": "The pages already existed as generated files on disk, but the agent retyped each one into html_preview and again into html_render, about four minutes per call. To save time it published fixes without previewing them again.", "source": {"discussion": "https://github.com/pingdotgg/t3code/discussions/23303"}, "rule": "edit the .html file and pass its path to html_preview and html_render instead of resending its html", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "The new-worktree toggle is ugly. Come up with four better options and mock them."}, {"role": "assistant", "text": "Here are four options.", "toolCalls": [{"tool": "Write", "input": {"file_path": "/tmp/design/index.html", "content": "\n

A. Checkbox under the composer

Ask anything
\n

B. Toggle in the toolbar

Ask anything
\n

C. Split send button

Ask anything
Send in new worktree
\n

D. Two visible choices

Ask anything
\n\n"}, "result": "File created successfully at: /tmp/design/index.html"}, {"tool": "html_preview", "input": {"html": "\n

A. Checkbox under the composer

Ask anything
\n

B. Toggle in the toolbar

Ask anything
\n

C. Split send button

Ask anything
Send in new worktree
\n

D. Two visible choices

Ask anything
\n\n"}, "result": "{\"width\":728,\"contentHeight\":1840,\"capturedHeight\":1840,\"consoleMessages\":[]}"}, {"tool": "html_render", "input": {"html": "\n

A. Checkbox under the composer

Ask anything
\n

B. Toggle in the toolbar

Ask anything
\n

C. Split send button

Ask anything
Send in new worktree
\n

D. Two visible choices

Ask anything
\n\n", "title": "New worktree: four options", "height": 1840}, "result": "{\"htmlRender\":{\"attachmentId\":\"att-1\",\"title\":\"New worktree: four options\",\"height\":1840},\"message\":\"Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say.\"}"}]}], "mocks": {"html_preview": {"width": 728, "contentHeight": 1840, "capturedHeight": 1840, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "New worktree: four options", "height": 1840}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}, "world": {"files": {"/tmp/design/index.html": "\n

A. Checkbox under the composer

Ask anything
\n

B. Toggle in the toolbar

Ask anything
\n

C. Split send button

Ask anything
Send in new worktree
\n

D. Two visible choices

Ask anything
\n\n"}}}, "prompt": "Make the Send in new worktree button in option C blue, and show me the page again.", "expect": {"calls": [{"tool": "html_preview", "args": {"path": "/tmp/design/index.html", "html": null}}, {"tool": "html_render", "args": {"path": "/tmp/design/index.html", "html": null}}], "callsMustNot": [{"tool": "html_preview", "args": {"html": "~Ask anything"}}, {"tool": "html_render", "args": {"html": "~Ask anything"}}], "order": ["html_preview", "html_render"]}, "notes": "On main neither tool has a path parameter."} -{"id": "preview-busy-tab-opens-new-tab", "area": "visual-preview", "kind": "bad", "pattern": "preview tab stuck behind a timed-out action; the next preview call retries the same tab", "experience": "A server-browser action never finished, so every later call on that tab timed out with the same generic error and the agent kept retrying the stuck tab.", "rule": "On a busy tab, continue in a new tab (preview_open with reuseExistingTab=false) or close it with t3_preview_close; do not retry it.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#32849: a browser action that never finishes wedges its tab; the agent only sees a generic timeout and keeps retrying the same tab"}, "setup": {"role": "top", "context": "The dev server is healthy at http://localhost:5173. The agent opened preview tab tab_1 (owner agent) earlier in this turn.", "priorTurns": [{"role": "user", "text": "Open http://localhost:5173/settings in the browser, switch it to dark mode, click Save, and check that the settings-saved toast still reads well in dark mode."}, {"role": "assistant", "toolCalls": [{"tool": "preview_open", "input": {"url": "http://localhost:5173/settings"}, "result": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "loading": false}}, {"tool": "preview_set_appearance", "input": {"tabId": "tab_1", "colorScheme": "dark"}, "result": {"_tag": "PreviewAutomationTimeoutError", "operation": "setColorScheme", "environmentId": "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee", "threadId": "thread-settings-check", "providerSessionId": "session-1", "providerInstanceId": "claudeAgent", "clientId": "server-browser", "connectionId": "conn-1", "requestId": "req-2", "tabId": "tab_1", "timeoutMs": 15000}}, {"tool": "preview_act", "input": {"tabId": "tab_1", "action": "click", "locator": "aria-ref=e12", "timeoutMs": 3000}, "result": {"_tag": "PreviewAutomationControlInterruptedError", "operation": "click", "environmentId": "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee", "threadId": "thread-settings-check", "providerSessionId": "session-1", "providerInstanceId": "claudeAgent", "clientId": "server-browser", "connectionId": "conn-1", "requestId": "req-3", "tabId": "tab_1", "timeoutMs": 3000, "remoteTag": "BrowserControlInterrupted", "remoteMessageLength": 0, "cause": null, "reason": "busy"}}]}], "mocks": {"preview_open": {"available": true, "visible": false, "tabId": "tab_2", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "loading": false}, "preview_status": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "loading": false, "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/settings", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 1}}, "t3_preview_list": {"tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/settings", "owner": "agent"}]}, "preview_snapshot": {"available": true, "tabId": "tab_2", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "snapshot": "- heading \"Settings\" [level=1]\n- switch \"Dark mode\" [checked] [ref=e7]\n- button \"Save\" [ref=e12]"}}}, "prompt": "", "expect": {"firstCall": {"tools": ["preview_act", "preview_resize", "preview_navigate", "preview_snapshot", "preview_wait_for", "preview_evaluate", "preview_set_appearance", "preview_open", "t3_preview_close"], "calls": [{"tool": "preview_open", "args": {"reuseExistingTab": false}}, {"tool": "t3_preview_close", "args": {"tabId": "tab_1"}}]}}, "maxSteps": 6, "notes": "The prior preview_act result is the contracts busy error, rendered with its real message. preview_status and t3_preview_list are reads, so they may come first."} -{"id": "preview-wait-timeout-keeps-tab", "area": "visual-preview", "kind": "control", "pattern": "an ordinary wait_for timeout; the agent keeps working in the same tab", "experience": "Guards #32849's busy-tab recovery: a plain timeout on a responsive tab is a page result, not a reason to abandon the tab.", "rule": "After an ordinary preview timeout, inspect or retry in the same tab.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#32849 (control)"}, "setup": {"role": "top", "context": "The dev server is healthy at http://localhost:5173. The agent opened preview tab tab_1 (owner agent) earlier in this turn.", "priorTurns": [{"role": "user", "text": "Open http://localhost:5173/settings in the browser, switch it to dark mode, click Save, and check that the settings-saved toast still reads well in dark mode."}, {"role": "assistant", "toolCalls": [{"tool": "preview_open", "input": {"url": "http://localhost:5173/settings"}, "result": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "loading": false}}, {"tool": "preview_set_appearance", "input": {"tabId": "tab_1", "colorScheme": "dark"}, "result": {"available": true, "tabId": "tab_1", "colorScheme": "dark"}}, {"tool": "preview_act", "input": {"tabId": "tab_1", "action": "click", "locator": "aria-ref=e12"}, "result": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/settings"}}, {"tool": "preview_wait_for", "input": {"tabId": "tab_1", "text": "Settings saved", "timeoutMs": 5000}, "result": {"_tag": "PreviewAutomationTimeoutError", "operation": "waitFor", "environmentId": "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee", "threadId": "thread-settings-check", "providerSessionId": "session-1", "providerInstanceId": "claudeAgent", "clientId": "server-browser", "connectionId": "conn-1", "requestId": "req-4", "tabId": "tab_1", "timeoutMs": 5000}}]}], "mocks": {"preview_open": {"available": true, "visible": false, "tabId": "tab_2", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "loading": false}, "preview_status": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "loading": false, "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/settings", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 1}}, "t3_preview_list": {"tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/settings", "owner": "agent"}]}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "snapshot": "- heading \"Settings\" [level=1]\n- switch \"Dark mode\" [checked] [ref=e7]\n- button \"Save\" [ref=e12]\n- status: \"Saved\""}}}, "prompt": "", "expect": {"firstCall": {"tools": ["preview_act", "preview_resize", "preview_navigate", "preview_snapshot", "preview_wait_for", "preview_evaluate", "preview_set_appearance", "preview_open", "t3_preview_close"], "calls": [{"tool": "preview_act|preview_snapshot|preview_wait_for|preview_evaluate|preview_navigate|preview_resize|preview_set_appearance"}]}}, "maxSteps": 6} +{"id": "preview-busy-tab-opens-new-tab", "area": "visual-preview", "kind": "bad", "pattern": "preview tab stuck behind a timed-out action; the next preview call retries the same tab", "experience": "A server-browser action never finished, so every later call on that tab timed out with the same generic error and the agent kept retrying the stuck tab.", "rule": "On a busy tab, continue in a new tab (preview_open with reuseExistingTab=false) or close it with t3_preview_close; do not retry it.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#32849: a browser action that never finishes wedges its tab; the agent only sees a generic timeout and keeps retrying the same tab"}, "setup": {"role": "top", "context": "The dev server is healthy at http://localhost:5173. The agent opened preview tab tab_1 (owner agent) earlier in this turn.", "priorTurns": [{"role": "user", "text": "Open http://localhost:5173/settings in the browser, switch it to dark mode, click Save, and check that the settings-saved toast still reads well in dark mode."}, {"role": "assistant", "toolCalls": [{"tool": "preview_open", "input": {"url": "http://localhost:5173/settings"}, "result": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "loading": false}}, {"tool": "preview_set_appearance", "input": {"tabId": "tab_1", "colorScheme": "dark"}, "result": {"_tag": "PreviewAutomationTimeoutError", "operation": "setColorScheme", "environmentId": "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee", "threadId": "thread-settings-check", "providerSessionId": "session-1", "providerInstanceId": "claudeAgent", "clientId": "server-browser", "connectionId": "conn-1", "requestId": "req-2", "tabId": "tab_1", "timeoutMs": 15000}}, {"tool": "preview_act", "input": {"tabId": "tab_1", "action": "click", "locator": "aria-ref=e12", "timeoutMs": 3000}, "result": {"_tag": "PreviewAutomationControlInterruptedError", "operation": "click", "environmentId": "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee", "threadId": "thread-settings-check", "providerSessionId": "session-1", "providerInstanceId": "claudeAgent", "clientId": "server-browser", "connectionId": "conn-1", "requestId": "req-3", "tabId": "tab_1", "timeoutMs": 3000, "remoteTag": "BrowserControlInterrupted", "remoteMessageLength": 0, "cause": null, "reason": "busy"}}]}], "mocks": {"preview_open": {"available": true, "visible": false, "tabId": "tab_2", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "loading": false}, "preview_status": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "loading": false, "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/settings", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 1}}, "t3_preview_list": {"tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/settings", "owner": "agent"}]}, "preview_snapshot": {"available": true, "tabId": "tab_2", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "snapshot": "- heading \"Settings\" [level=1]\n- switch \"Dark mode\" [checked] [ref=e7]\n- button \"Save\" [ref=e12]"}}}, "prompt": "", "expect": {"firstCall": {"tools": ["preview_act", "preview_resize", "preview_navigate", "preview_snapshot", "preview_wait_for", "preview_evaluate", "preview_set_appearance", "preview_open", "t3_preview_close"], "calls": [{"tool": "preview_open", "args": {"reuseExistingTab": false}}, {"tool": "t3_preview_close", "args": {"tabId": "tab_1"}}]}, "callsMustNot": [{"tool": "preview_act|preview_resize|preview_navigate|preview_snapshot|preview_wait_for|preview_evaluate|preview_set_appearance", "args": {"tabId": "tab_1"}}]}, "maxSteps": 6, "notes": "The prior preview_act result is the contracts busy error, rendered with its real message. preview_status and t3_preview_list are reads, so they may come first."} +{"id": "preview-wait-timeout-keeps-tab", "area": "visual-preview", "kind": "control", "pattern": "an ordinary wait_for timeout; the agent keeps working in the same tab", "experience": "Guards #32849's busy-tab recovery: a plain timeout on a responsive tab is a page result, not a reason to abandon the tab.", "rule": "After an ordinary preview timeout, inspect or retry in the same tab.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#32849 (control)"}, "setup": {"role": "top", "context": "The dev server is healthy at http://localhost:5173. The agent opened preview tab tab_1 (owner agent) earlier in this turn.", "priorTurns": [{"role": "user", "text": "Open http://localhost:5173/settings in the browser, switch it to dark mode, click Save, and check that the settings-saved toast still reads well in dark mode."}, {"role": "assistant", "toolCalls": [{"tool": "preview_open", "input": {"url": "http://localhost:5173/settings"}, "result": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "loading": false}}, {"tool": "preview_set_appearance", "input": {"tabId": "tab_1", "colorScheme": "dark"}, "result": {"available": true, "tabId": "tab_1", "colorScheme": "dark"}}, {"tool": "preview_act", "input": {"tabId": "tab_1", "action": "click", "locator": "aria-ref=e12"}, "result": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/settings"}}, {"tool": "preview_wait_for", "input": {"tabId": "tab_1", "text": "Settings saved", "timeoutMs": 5000}, "result": {"_tag": "PreviewAutomationTimeoutError", "operation": "waitFor", "environmentId": "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee", "threadId": "thread-settings-check", "providerSessionId": "session-1", "providerInstanceId": "claudeAgent", "clientId": "server-browser", "connectionId": "conn-1", "requestId": "req-4", "tabId": "tab_1", "timeoutMs": 5000}}]}], "mocks": {"preview_open": {"available": true, "visible": false, "tabId": "tab_2", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "loading": false}, "preview_status": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "loading": false, "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/settings", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 1}}, "t3_preview_list": {"tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/settings", "owner": "agent"}]}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "snapshot": "- heading \"Settings\" [level=1]\n- switch \"Dark mode\" [checked] [ref=e7]\n- button \"Save\" [ref=e12]\n- status: \"Saved\""}}}, "prompt": "", "expect": {"firstCall": {"tools": ["preview_act", "preview_resize", "preview_navigate", "preview_snapshot", "preview_wait_for", "preview_evaluate", "preview_set_appearance", "preview_open", "t3_preview_close"], "calls": [{"tool": "preview_act|preview_resize|preview_navigate|preview_snapshot|preview_wait_for|preview_evaluate|preview_set_appearance", "args": {"tabId": "tab_1"}}, {"tool": "preview_act|preview_resize|preview_navigate|preview_snapshot|preview_wait_for|preview_evaluate|preview_set_appearance", "args": {"tabId": null}}]}}, "maxSteps": 6} diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts index e48ccb329ee2b..b708c50cbe757 100644 --- a/evals/mcp-tool-use/run.test.ts +++ b/evals/mcp-tool-use/run.test.ts @@ -606,6 +606,30 @@ it("replays an earlier preview error as T3's error result, with its contracts me assert.deepInclude(replayed, { content: error.message, is_error: true }); }); +it("grades busy-tab recovery by the tab the agent acts on, not just the tool", () => { + const cases = NodeFS.readFileSync( + new URL("./cases/visual-preview.ndjson", import.meta.url), + "utf8", + ) + .trim() + .split("\n") + .map((line) => decodeDeviceCase(line)); + const passes = (id: string, calls: Array<[string, Record]>) => + gradeSample(cases.find((c) => c.id === id)!.expect, { + calls: calls.map(([tool, input], step) => ({ step, tool, input })), + text: "", + stopReason: "end_turn", + }).every((check) => check.pass); + const busy = "preview-busy-tab-opens-new-tab"; + const fresh: [string, Record] = ["preview_open", { reuseExistingTab: false }]; + assert.isTrue(passes(busy, [["preview_status", {}], fresh, ["preview_act", { tabId: "tab_2" }]])); + assert.isFalse(passes(busy, [fresh, ["preview_act", { tabId: "tab_1" }]])); + const control = "preview-wait-timeout-keeps-tab"; + assert.isTrue(passes(control, [["preview_snapshot", { tabId: "tab_1" }]])); + assert.isTrue(passes(control, [["preview_snapshot", {}]])); + assert.isFalse(passes(control, [["preview_snapshot", { tabId: "tab_2" }]])); +}); + it("replays earlier capability results with the model under test as the caller", () => { const base = buildBase(template, { model: "claude-opus-5-5", keepUserInstructions: false }); const capabilities = { From aaf544d2849899f308f3ef52ad322620c9af21ae Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 10 Nov 2026 12:23:50 +0000 Subject: [PATCH 09/10] test(server): queue the late recording stop behind the click Co-Authored-By: Claude Opus 5.5 --- apps/server/src/preview/ServerBrowser.test.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index 32075e8ca69c2..55568dfc042d2 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -2732,6 +2732,11 @@ it.effect("a queued action whose timeout passes while it waits never runs", () = const { broker, tabId } = yield* ready; const page = contexts[0]!.page; yield* broker.invoke({ scope, tabId, operation: "recordingStart", input: {} }); + const stopRan = Promise.withResolvers(); + contexts[0]!.sessions.at(-1)!.detach.mockImplementationOnce(() => { + stopRan.resolve(); + return new Promise(() => {}); + }); const started = Promise.withResolvers(); const finish = Promise.withResolvers(); page.emulateMedia.mockImplementationOnce(() => { @@ -2751,11 +2756,12 @@ it.effect("a queued action whose timeout passes while it waits never runs", () = const queued = yield* broker .invoke({ scope, tabId, operation: "click", input: { ref: "e1" }, timeoutMs: 1_000 }) .pipe(Effect.flip, Effect.forkScoped); - const stop = yield* broker - .invoke({ scope, tabId, operation: "recordingStop", input: {}, timeoutMs: 1_000 }) - .pipe(Effect.exit, Effect.forkScoped); // The host answers in arrival order, so the click is queued once this answers. yield* broker.invoke({ scope, tabId, operation: "status", input: {} }); + yield* broker + .invoke({ scope, tabId, operation: "recordingStop", input: {}, timeoutMs: 1_000 }) + .pipe(Effect.ignore, Effect.forkScoped); + yield* broker.invoke({ scope, tabId, operation: "status", input: {} }); // Past the click's timeout but before the server answers it. yield* TestClock.adjust("1200 millis"); finish.resolve(); @@ -2763,8 +2769,7 @@ it.effect("a queued action whose timeout passes while it waits never runs", () = expect(yield* Fiber.join(queued)).toMatchObject({ _tag: "PreviewAutomationTimeoutError" }); expect(page.locator).not.toHaveBeenCalled(); // A late stop still runs, or capture would keep going with no recording to stop. - yield* Fiber.join(stop); - expect(contexts[0]!.sessions.at(-1)!.detach).toHaveBeenCalled(); + yield* Effect.promise(() => stopRan.promise); }), ).pipe(Effect.provide(layer)), ); From b55ddf9a2b1fa4f6be0ef62cca3d8dacdc6268a1 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 10 Nov 2026 19:32:17 +0000 Subject: [PATCH 10/10] test(server): drop preview tests that repeat covered cases AI-assisted; verified with the tests and tools listed above. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/preview/ServerBrowser.test.ts | 110 ++++++++---------- evals/mcp-tool-use/run.test.ts | 24 ---- 2 files changed, 48 insertions(+), 86 deletions(-) diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index 55568dfc042d2..61af124efb55c 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -2602,68 +2602,54 @@ it.effect("a tab stuck in an agent action does not hold back setting changes for ).pipe(Effect.provide(layer)), ); -it.effect( - "calls queued behind a stuck agent action time out on their own, and dialogs come first", - () => - Effect.scoped( - Effect.gen(function* () { - const { broker, tabId } = yield* ready; - const page = contexts[0]!.page; - yield* broker.invoke({ scope, tabId, operation: "recordingStart", input: {} }); - const stalled = stallColorScheme(page); - const stuck = yield* broker - .invoke({ - scope, - tabId, - operation: "setColorScheme", - input: { colorScheme: "dark" }, - timeoutMs: 1_000, - }) - .pipe(Effect.flip, Effect.forkScoped); - yield* stalled; - const queued = yield* broker - .invoke({ - scope, - tabId, - operation: "click", - input: { ref: "e1" }, - timeoutMs: 1_000, - }) - .pipe(Effect.flip, Effect.forkScoped); - // The host handles requests in arrival order, so the click is queued once this answers. - yield* broker.invoke({ scope, tabId, operation: "status", input: {} }); - yield* TestClock.adjust("2 seconds"); - expect(yield* Fiber.join(stuck)).toMatchObject({ _tag: "PreviewAutomationTimeoutError" }); - expect(yield* Fiber.join(queued)).toMatchObject({ _tag: "PreviewAutomationTimeoutError" }); - page.emit("dialog", { - type: () => "alert", - message: () => "Saved", - defaultValue: () => "", - accept: vi.fn(async () => {}), - dismiss: vi.fn(async () => {}), - }); - const blocked = yield* broker - .invoke({ - scope, - tabId, - operation: "click", - input: { ref: "e1" }, - timeoutMs: 1_000, - }) - .pipe(Effect.flip); - expect(blocked).toMatchObject({ - _tag: "PreviewAutomationControlInterruptedError", - reason: "dialogPending", - }); - const stop = yield* broker - .invoke({ scope, tabId, operation: "recordingStop", input: {}, timeoutMs: 1_000 }) - .pipe(Effect.flip); - expect(stop).toMatchObject({ - _tag: "PreviewAutomationControlInterruptedError", - reason: "dialogPending", - }); - }), - ).pipe(Effect.provide(layer)), +it.effect("a dialog on a stuck tab is reported before the tab is busy", () => + Effect.scoped( + Effect.gen(function* () { + const { broker, tabId } = yield* ready; + const page = contexts[0]!.page; + yield* broker.invoke({ scope, tabId, operation: "recordingStart", input: {} }); + const stalled = stallColorScheme(page); + const stuck = yield* broker + .invoke({ + scope, + tabId, + operation: "setColorScheme", + input: { colorScheme: "dark" }, + timeoutMs: 1_000, + }) + .pipe(Effect.flip, Effect.forkScoped); + yield* stalled; + yield* TestClock.adjust("2 seconds"); + expect(yield* Fiber.join(stuck)).toMatchObject({ _tag: "PreviewAutomationTimeoutError" }); + page.emit("dialog", { + type: () => "alert", + message: () => "Saved", + defaultValue: () => "", + accept: vi.fn(async () => {}), + dismiss: vi.fn(async () => {}), + }); + const blocked = yield* broker + .invoke({ + scope, + tabId, + operation: "click", + input: { ref: "e1" }, + timeoutMs: 1_000, + }) + .pipe(Effect.flip); + expect(blocked).toMatchObject({ + _tag: "PreviewAutomationControlInterruptedError", + reason: "dialogPending", + }); + const stop = yield* broker + .invoke({ scope, tabId, operation: "recordingStop", input: {}, timeoutMs: 1_000 }) + .pipe(Effect.flip); + expect(stop).toMatchObject({ + _tag: "PreviewAutomationControlInterruptedError", + reason: "dialogPending", + }); + }), + ).pipe(Effect.provide(layer)), ); it.effect("an action that fails just after its timeout still reports its own failure", () => diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts index b708c50cbe757..e48ccb329ee2b 100644 --- a/evals/mcp-tool-use/run.test.ts +++ b/evals/mcp-tool-use/run.test.ts @@ -606,30 +606,6 @@ it("replays an earlier preview error as T3's error result, with its contracts me assert.deepInclude(replayed, { content: error.message, is_error: true }); }); -it("grades busy-tab recovery by the tab the agent acts on, not just the tool", () => { - const cases = NodeFS.readFileSync( - new URL("./cases/visual-preview.ndjson", import.meta.url), - "utf8", - ) - .trim() - .split("\n") - .map((line) => decodeDeviceCase(line)); - const passes = (id: string, calls: Array<[string, Record]>) => - gradeSample(cases.find((c) => c.id === id)!.expect, { - calls: calls.map(([tool, input], step) => ({ step, tool, input })), - text: "", - stopReason: "end_turn", - }).every((check) => check.pass); - const busy = "preview-busy-tab-opens-new-tab"; - const fresh: [string, Record] = ["preview_open", { reuseExistingTab: false }]; - assert.isTrue(passes(busy, [["preview_status", {}], fresh, ["preview_act", { tabId: "tab_2" }]])); - assert.isFalse(passes(busy, [fresh, ["preview_act", { tabId: "tab_1" }]])); - const control = "preview-wait-timeout-keeps-tab"; - assert.isTrue(passes(control, [["preview_snapshot", { tabId: "tab_1" }]])); - assert.isTrue(passes(control, [["preview_snapshot", {}]])); - assert.isFalse(passes(control, [["preview_snapshot", { tabId: "tab_2" }]])); -}); - it("replays earlier capability results with the model under test as the caller", () => { const base = buildBase(template, { model: "claude-opus-5-5", keepUserInstructions: false }); const capabilities = {