diff --git a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts index 21b2dab5c71c8..65b7ceb31caba 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts @@ -779,6 +779,7 @@ const makeInstructionCompactionHarness = Effect.fnUntraced(function* ( endpoint: "http://127.0.0.1:43123/mcp", authorizationHeader: "Bearer instruction-compaction-token", browserToolsAvailable: true, + capabilities: new Set(["preview"]), }); type Runtime = AcpSessionRuntime.AcpSessionRuntime["Service"]; let sessionUpdate: Parameters[0] | undefined; @@ -2403,7 +2404,7 @@ describe("AcpAdapterV2", () => { const firstDefault = yield* runTurn(2, defaultPolicy, "First default request."); assert.equal(firstDefault.textBlockCount, 2); assert.include(firstDefault.prompt, "T3 Code interaction mode: Default"); - assert.include(firstDefault.prompt, "## T3 Code browser"); + assert.notInclude(firstDefault.prompt, "## T3 Code browser"); assert.include(firstDefault.prompt, "T3 Code orchestration"); assert.include(firstDefault.prompt, ""); assert.notInclude( diff --git a/apps/server/src/orchestration-v2/Adapters/AcpRegistryAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AcpRegistryAdapterV2.test.ts index 8982fb122fc7b..389e8ae5812d0 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpRegistryAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpRegistryAdapterV2.test.ts @@ -1486,7 +1486,12 @@ describe("AcpRegistryAdapterV2", () => { ordinal === 1 ? t3AcpPromptWithInstructions({ prompt: "test prompt", - state: { interactionMode: "default", hasT3Mcp: false, hasDeviceTools: false }, + state: { + interactionMode: "default", + hasT3Mcp: false, + hasDeviceTools: false, + hasBrowserTools: false, + }, }) : "test prompt", }, diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 6194d9d44186a..f1e7491f103ed 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -87,7 +87,10 @@ import { attachmentRelativePath } from "../../attachmentStore.ts"; import { WorkflowAgentTranscriptReadObserver } from "../workflowAgentTranscript.ts"; import * as ServerConfig from "../../config.ts"; import * as McpProviderSession from "@t3tools/provider-core/server/mcpSession"; -import { T3_CODE_DEVICE_TOOL_INSTRUCTIONS_WITH_TOOL_SEARCH } from "@t3tools/provider-core/server/orchestrationInstructions"; +import { + T3_CODE_BROWSER_TOOL_INSTRUCTIONS, + T3_CODE_DEVICE_TOOL_INSTRUCTIONS_WITH_TOOL_SEARCH, +} from "@t3tools/provider-core/server/orchestrationInstructions"; import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions"; import { PreviewControlsToolkit } from "../../mcp/toolkits/previewControls/tools.ts"; import { HtmlToolkit } from "../../mcp/toolkits/html/tools.ts"; @@ -832,7 +835,13 @@ describe("ClaudeAdapterV2 native protocol logging", () => { }); describe("ClaudeAdapterV2 device instructions", () => { - const options = (input: { readonly mcp: boolean; readonly deviceToolsAvailable: boolean }) => + const NOT_INSTALLED_BROWSER_RULE = + "A browser tool that project docs name but that is not installed is no reason to switch: use preview, and do not install it."; + const options = (input: { + readonly mcp: boolean; + readonly deviceToolsAvailable: boolean; + readonly browserToolsAvailable?: boolean; + }) => ClaudeAdapterV2.makeClaudeQueryOptions({ modelSelection: { instanceId: ProviderInstanceId.make("claudeAgent"), @@ -845,26 +854,43 @@ describe("ClaudeAdapterV2 device instructions", () => { ? { mcpServers: { "t3-code": { type: "http" as const, url: "http://127.0.0.1/mcp" } } } : {}), deviceToolsAvailable: input.deviceToolsAvailable, + browserToolsAvailable: input.browserToolsAvailable ?? false, environment: {}, }); - const append = (input: { readonly mcp: boolean; readonly deviceToolsAvailable: boolean }) => - (options(input).systemPrompt as { readonly append?: string }).append ?? ""; + const append = (input: { + readonly mcp: boolean; + readonly deviceToolsAvailable: boolean; + readonly browserToolsAvailable?: boolean; + }) => (options(input).systemPrompt as { readonly append?: string }).append ?? ""; - it("gives a session with device access the device block and the ToolSearch line to load it", () => { + it("gives a session with device or browser access that block, and the ToolSearch line for devices", () => { const text = append({ mcp: true, deviceToolsAvailable: true }); assert.include(text, T3_CODE_DEVICE_TOOL_INSTRUCTIONS_WITH_TOOL_SEARCH); assert.include(text, "Use `delegate_task`"); + assert.include( + append({ mcp: true, deviceToolsAvailable: false, browserToolsAvailable: true }), + `${T3_CODE_BROWSER_TOOL_INSTRUCTIONS.trim()} ${NOT_INSTALLED_BROWSER_RULE}`, + ); }); - it("leaves the device block out without device access or without T3 tools", () => { - assert.notInclude(append({ mcp: true, deviceToolsAvailable: false }), "## T3 Code devices"); - assert.notInclude(append({ mcp: false, deviceToolsAvailable: true }), "## T3 Code devices"); + it("leaves the device and browser blocks out without that access or without T3 tools", () => { + const withoutAccess = append({ mcp: true, deviceToolsAvailable: false }); + assert.notInclude(withoutAccess, "## T3 Code devices"); + assert.notInclude(withoutAccess, "## T3 Code browser"); + const withoutMcp = append({ + mcp: false, + deviceToolsAvailable: true, + browserToolsAvailable: true, + }); + assert.notInclude(withoutMcp, "## T3 Code devices"); + assert.notInclude(withoutMcp, "## T3 Code browser"); }); - it("gives native subagents the device block only in a session with device access", async () => { + it("gives native subagents browser and device blocks only with the corresponding access", async () => { // Subagents start from their task prompt, not the parent's system prompt. const subagentContext = (input: { readonly mcp: boolean; + readonly browserToolsAvailable?: boolean; readonly deviceToolsAvailable: boolean; }) => Promise.all( @@ -893,7 +919,33 @@ describe("ClaudeAdapterV2 device instructions", () => { }, }, ]); + for (const deviceToolsAvailable of [false, true]) { + const result = await subagentContext({ + mcp: true, + deviceToolsAvailable, + browserToolsAvailable: true, + }); + assert.deepEqual(result, [ + { + hookSpecificOutput: { + hookEventName: "SubagentStart", + additionalContext: [ + `${T3_CODE_BROWSER_TOOL_INSTRUCTIONS.trim()} ${NOT_INSTALLED_BROWSER_RULE}`, + ...(deviceToolsAvailable ? [T3_CODE_DEVICE_TOOL_INSTRUCTIONS_WITH_TOOL_SEARCH] : []), + ].join("\n\n"), + }, + }, + ]); + } assert.deepEqual(await subagentContext({ mcp: true, deviceToolsAvailable: false }), []); + assert.deepEqual( + await subagentContext({ + mcp: false, + deviceToolsAvailable: true, + browserToolsAvailable: true, + }), + [], + ); assert.deepEqual(await subagentContext({ mcp: false, deviceToolsAvailable: true }), []); }); }); diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 8f5dae18521af..2d4ae8e70afdf 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -214,6 +214,7 @@ import { } from "../../provider/claudeUsageLimits.ts"; import type { ManagedServerProvider } from "@t3tools/provider-core/server/snapshot"; import { + T3_CODE_BROWSER_TOOL_INSTRUCTIONS, T3_CODE_DEVICE_TOOL_INSTRUCTIONS_WITH_TOOL_SEARCH, T3_CODE_ORCHESTRATION_INSTRUCTIONS, } from "@t3tools/provider-core/server/orchestrationInstructions"; @@ -2100,14 +2101,19 @@ const CLAUDE_SEND_TO_USER_INSTRUCTIONS = ` Text you write before a tool call may reach the user only as a one-line summary. When the user must read something in full before your turn ends, such as a draft, a code block, or what a question you're about to ask refers to, send it with \`send_to_user\`. Put everything else in your final reply. `; +// Claude reads a CLAUDE.md that names a missing browser CLI as the user asking for it, and installs it. +const CLAUDE_BROWSER_INSTRUCTIONS = `${T3_CODE_BROWSER_TOOL_INSTRUCTIONS.trim()} A browser tool that project docs name but that is not installed is no reason to switch: use preview, and do not install it.`; + // Native subagents start from their task, not this session's system prompt. -const subagentDeviceInstructions: HookCallback = () => - Promise.resolve({ - hookSpecificOutput: { - hookEventName: "SubagentStart", - additionalContext: T3_CODE_DEVICE_TOOL_INSTRUCTIONS_WITH_TOOL_SEARCH, - }, - }); +const subagentInstructions = + (additionalContext: string): HookCallback => + () => + Promise.resolve({ + hookSpecificOutput: { + hookEventName: "SubagentStart", + additionalContext, + }, + }); export function makeClaudeQueryOptions(input: { readonly modelSelection: ModelSelection; @@ -2123,6 +2129,8 @@ export function makeClaudeQueryOptions(input: { readonly sdkSettings?: string | ClaudeSdkSettings; readonly environment?: NodeJS.ProcessEnv; readonly mcpServers?: ClaudeQueryOptions["mcpServers"]; + /** The thread's T3 MCP session holds the `preview` capability. */ + readonly browserToolsAvailable?: boolean; /** The thread's T3 MCP session holds the `device` capability. */ readonly deviceToolsAvailable?: boolean; readonly tools?: ClaudeAgentSdkQueryTools; @@ -2219,6 +2227,8 @@ export function makeClaudeQueryOptions(input: { : { ENABLE_TOOL_SEARCH: input.toolSearchOverride }), }, } as ClaudeSdkSettings); + const browserInstructions = + input.mcpServers !== undefined && input.browserToolsAvailable === true; const deviceInstructions = input.mcpServers !== undefined && input.deviceToolsAvailable === true; const options: ClaudeAgentSdkQueryOptions = { model: compiledSelection.apiModelId, @@ -2279,17 +2289,33 @@ export function makeClaudeQueryOptions(input: { : T3_CODE_ORCHESTRATION_INSTRUCTIONS + CLAUDE_BACKGROUND_WAIT_INSTRUCTIONS + CLAUDE_SEND_TO_USER_INSTRUCTIONS + + (browserInstructions ? `\n${CLAUDE_BROWSER_INSTRUCTIONS}\n` : "") + (deviceInstructions ? `\n${T3_CODE_DEVICE_TOOL_INSTRUCTIONS_WITH_TOOL_SEARCH}\n` : "")), ...launchAppendSystemPrompt, ].join("\n\n"), }, - ...(deviceInstructions || input.onStop !== undefined + ...(deviceInstructions || browserInstructions || input.onStop !== undefined ? { hooks: { - ...(deviceInstructions - ? { SubagentStart: [{ hooks: [subagentDeviceInstructions] }] } + ...(deviceInstructions || browserInstructions + ? { + SubagentStart: [ + { + hooks: [ + subagentInstructions( + [ + ...(browserInstructions ? [CLAUDE_BROWSER_INSTRUCTIONS] : []), + ...(deviceInstructions + ? [T3_CODE_DEVICE_TOOL_INSTRUCTIONS_WITH_TOOL_SEARCH] + : []), + ].join("\n\n"), + ), + ], + }, + ], + } : {}), ...(input.onStop === undefined ? {} : { Stop: [{ hooks: [input.onStop] }] }), }, @@ -12661,6 +12687,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ...(mcpOverrides.mcpServers === undefined ? {} : { mcpServers: mcpOverrides.mcpServers }), + browserToolsAvailable: mcpSession?.capabilities?.has("preview") === true, deviceToolsAvailable: mcpSession?.capabilities?.has("device") === true, permissionMode: queryPolicy.permissionMode, ...(queryPolicy.allowDangerouslySkipPermissions === undefined diff --git a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts index 89458b1deeeda..bf5ac69556b03 100644 --- a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts @@ -1,3 +1,7 @@ +import { + T3_CODE_BROWSER_TOOL_INSTRUCTIONS, + T3_CODE_ORCHESTRATION_INSTRUCTIONS, +} from "@t3tools/provider-core/server/orchestrationInstructions"; import * as NodeURL from "node:url"; import { toolOutputImages } from "@t3tools/shared/toolOutput"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; @@ -159,6 +163,7 @@ const noOpenRequests: ReadonlyArray = [ const withInstructions = ( entries: ReadonlyArray, + instructions = "", ): ReadonlyArray => { const first = entries.findIndex( (entry) => @@ -192,7 +197,7 @@ const withInstructions = ( out("session.instructions.entry.put", { sessionID: SESSION, key: "t3-code", - value: "", + value: instructions, }), reply("session.instructions.entry.put", null), ...entries.slice(at), @@ -340,6 +345,7 @@ const resumed = ( readonly resolveExtractedTextPath?: (path: string) => Effect.Effect; readonly fileSystem?: FileSystem.FileSystem; readonly onPrompt?: (input: unknown) => void; + readonly instructions?: string; }, ) => Effect.gen(function* () { @@ -360,17 +366,20 @@ const resumed = ( ? [out("job.list", { kind: "shell" }), reply("job.list", { data: [] })] : emptyShellInventory), ])), - ...withInstructions([ - ...(options?.supervised === true - ? [ - out("agent.list", ""), - reply("agent.list", agentList), - out("session.update", { sessionID: SESSION, permissions: supervisedRules }), - reply("session.update", null), - ] - : []), - ...entries, - ]), + ...withInstructions( + [ + ...(options?.supervised === true + ? [ + out("agent.list", ""), + reply("agent.list", agentList), + out("session.update", { sessionID: SESSION, permissions: supervisedRules }), + reply("session.update", null), + ] + : []), + ...entries, + ], + options?.instructions, + ), ], { ...(options?.version === undefined ? {} : { version: options.version }), @@ -5062,69 +5071,84 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { }), ); - it.effect("re-registers T3 MCP each turn and drops its instructions when adding fails", () => - Effect.gen(function* () { - const mcpSessions = yield* McpProviderSessions.McpProviderSessions; - yield* mcpSessions.set({ - environmentId: EnvironmentId.make("environment:opencode2-adapter"), - threadId, - providerSessionId: "mcp:opencode2-adapter", - providerInstanceId: instanceId, - endpoint: "http://127.0.0.1:3773/mcp", - authorizationHeader: "Bearer thread-credential", - browserToolsAvailable: false, - }); - yield* Effect.addFinalizer(() => mcpSessions.clear(threadId)); - const server = "t3-code-thread_opencode2-adapter"; - const add = out("mcp.add", { - server, - "location[directory]": WORK, - config: { - type: "remote", - url: "http://127.0.0.1:3773/mcp", - headers: { Authorization: "Bearer thread-credential" }, - oauth: false, - }, - }); - const turn = [ - out("session.prompt", { sessionID: SESSION, text: "" }), - promptAccepted, - event("session.execution.succeeded", { sessionID: SESSION }), - ]; - const prompts: Array = []; - // Idle Location eviction drops OpenCode's runtime MCP servers without a reconnect. - const { runtime, thread } = yield* resumed( - [ - add, - reply("mcp.add", null), - ...turn, - add, - reply("mcp.add", { status: 500, body: { message: "MCP unavailable" } }), - out("session.instructions.entry.put", { - sessionID: SESSION, - key: "t3-code", - value: buildRuntimeInstructions({ harness: "OpenCode", model: bigPickle.model }), - }), - reply("session.instructions.entry.put", null), - ...turn, - out("mcp.remove", { server, "location[directory]": WORK }), - reply("mcp.remove", null), - ], - { onPrompt: (input) => prompts.push(input) }, - ); - for (let index = 0; index < 2; index++) { - const terminal = yield* terminalOf(runtime).pipe(Effect.forkScoped); - const started = yield* runtime.startTurn(turnInput(thread)).pipe(Effect.forkScoped); - // OpenCode before 2.2.6 loads a newly added server's tools after a debounce. - if (index === 0) { - yield* TestClock.adjust("249 millis"); - assert.lengthOf(prompts, 0, "prompted before OpenCode loaded the new MCP server's tools"); - yield* TestClock.adjust("1 millis"); + it.effect.each([false, true])( + "re-registers T3 MCP with browser access %s and drops its instructions when adding fails", + (browser) => + Effect.gen(function* () { + const mcpSessions = yield* McpProviderSessions.McpProviderSessions; + yield* mcpSessions.set({ + environmentId: EnvironmentId.make("environment:opencode2-adapter"), + threadId, + providerSessionId: "mcp:opencode2-adapter", + providerInstanceId: instanceId, + endpoint: "http://127.0.0.1:3773/mcp", + authorizationHeader: "Bearer thread-credential", + browserToolsAvailable: browser, + capabilities: new Set(browser ? ["preview"] : []), + }); + yield* Effect.addFinalizer(() => mcpSessions.clear(threadId)); + const server = "t3-code-thread_opencode2-adapter"; + const add = out("mcp.add", { + server, + "location[directory]": WORK, + config: { + type: "remote", + url: "http://127.0.0.1:3773/mcp", + headers: { Authorization: "Bearer thread-credential" }, + oauth: false, + }, + }); + const turn = [ + out("session.prompt", { sessionID: SESSION, text: "" }), + promptAccepted, + event("session.execution.succeeded", { sessionID: SESSION }), + ]; + const prompts: Array = []; + // Idle Location eviction drops OpenCode's runtime MCP servers without a reconnect. + const { runtime, thread } = yield* resumed( + [ + add, + reply("mcp.add", null), + ...turn, + add, + reply("mcp.add", { status: 500, body: { message: "MCP unavailable" } }), + out("session.instructions.entry.put", { + sessionID: SESSION, + key: "t3-code", + value: buildRuntimeInstructions({ harness: "OpenCode", model: bigPickle.model }), + }), + reply("session.instructions.entry.put", null), + ...turn, + out("mcp.remove", { server, "location[directory]": WORK }), + reply("mcp.remove", null), + ], + { + onPrompt: (input) => prompts.push(input), + instructions: [ + buildRuntimeInstructions({ harness: "OpenCode", model: bigPickle.model }), + T3_CODE_ORCHESTRATION_INSTRUCTIONS + + (browser ? `\n${T3_CODE_BROWSER_TOOL_INSTRUCTIONS.trim()}\n` : ""), + "To wait for a background shell you started (tests, a build), call `t3_background_wait` with its shell ID before you end your turn. Do not mark work you leave running, or PRs.", + ].join("\n\n"), + }, + ); + for (let index = 0; index < 2; index++) { + const terminal = yield* terminalOf(runtime).pipe(Effect.forkScoped); + const started = yield* runtime.startTurn(turnInput(thread)).pipe(Effect.forkScoped); + // OpenCode before 2.2.6 loads a newly added server's tools after a debounce. + if (index === 0) { + yield* TestClock.adjust("249 millis"); + assert.lengthOf( + prompts, + 0, + "prompted before OpenCode loaded the new MCP server's tools", + ); + yield* TestClock.adjust("1 millis"); + } + yield* Fiber.join(started); + assert.equal((yield* Fiber.join(terminal))?.status, "completed"); } - yield* Fiber.join(started); - assert.equal((yield* Fiber.join(terminal))?.status, "completed"); - } - }), + }), ); it.effect("registers a long thread id's MCP server under a name OpenCode accepts", () => diff --git a/evals/mcp-tool-use/README.md b/evals/mcp-tool-use/README.md index 25950e2098ee4..de793d0b548fd 100644 --- a/evals/mcp-tool-use/README.md +++ b/evals/mcp-tool-use/README.md @@ -232,6 +232,7 @@ Passes per case, main's text → the request_secret guidance change stacked on t | --------------------------------- | ------- | ---------------------------------------------------------------------------------------------------------------- | | br-look-on-web-missing-cli | bad | "How does it look on web?" after the agent's own edit, with CLAUDE.md naming a missing `agent-browser`: preview. | | br-flow-qa-missing-cli | bad | Checkout-flow QA in the same world: preview. | +| br-flow-qa-installable-cli | bad | The same QA where installing the named CLI would succeed, as on a real machine: preview, no install. | | br-flow-qa-no-cli-named | bad | The same QA without the CLAUDE.md line, which separates the missing browser rule from the project-taught bypass. | | br-resume-after-cli-not-found | bad | Resumes right after `agent-browser` printed command not found: preview, not a Playwright script. | | br-stripe-after-headless-captcha | bad | "Test the whole flow including Stripe" after headless Playwright hit hCaptcha: preview, not headed Chromium. | @@ -244,9 +245,9 @@ Passes per case, main's text → the request_secret guidance change stacked on t | br-guard-browser-access-off | guard | Agent browser access is off: at most one preview call in all; falling back is allowed. | | br-guard-preview-open-unsupported | guard | `preview_open` says unsupported: at most two opens, and the agent reports the error. Falling back is allowed. | -Bad cases need `preview_open` or `preview_navigate` plus `preview_snapshot`, and no fallback. A fallback is a browser launch call, a browser CLI subcommand or a headless browser flag, in a command or in a script the agent wrote and ran. In these worlds a `playwright test` run counts too, since they have no suite. Probes such as `grep chromium.launch`, `ls`, `which` and `require.resolve('puppeteer')` don't count (but see the regex limits below). Calling `preview_status` first is a diagnostic, not a gate. All 13 cases allow 14 steps. +Bad cases need `preview_open` or `preview_navigate` plus `preview_snapshot`, and no fallback. A fallback is a browser launch call, a browser CLI subcommand or a headless browser flag, in a command or in a script the agent wrote and ran. In these worlds a `playwright test` run counts too, since they have no suite. Probes such as `grep chromium.launch`, `ls`, `which` and `require.resolve('puppeteer')` don't count (but see the regex limits below). Calling `preview_status` first is a diagnostic, not a gate. All 14 cases allow 14 steps. -Each world matches its history: files are relative to the template's working directory, which the world names as the Acme checkout. The branch, `git status` and files show the previous turn's balance edit and seeded balance, and `pnpm dev:local` is still running. `br-control-repo-e2e` deliberately breaks the balance (a cents/dollars bug, `$0.00`) for its suite to catch. `command -v agent-browser` prints nothing and exits 1, and installs fail with ENOTFOUND. `preview_status` returns main's no-tab result (`available: false`). +Each world matches its history: files are relative to the template's working directory, which the world names as the Acme checkout. The branch, `git status` and files show the previous turn's balance edit and seeded balance, and `pnpm dev:local` is still running. `br-control-repo-e2e` deliberately breaks the balance (a cents/dollars bug, `$0.00`) for its suite to catch. `command -v agent-browser` prints nothing and exits 1, and installs fail with ENOTFOUND, except in `br-flow-qa-installable-cli`. Only Claude's browser text tells the agent not to install a CLI the docs name, so that case targets Claude. `preview_status` returns main's no-tab result (`available: false`). Runner support: diff --git a/evals/mcp-tool-use/cases/browser-routing.ndjson b/evals/mcp-tool-use/cases/browser-routing.ndjson index 27daaeaa0ac8f..a3baf0623d2fa 100644 --- a/evals/mcp-tool-use/cases/browser-routing.ndjson +++ b/evals/mcp-tool-use/cases/browser-routing.ndjson @@ -1,5 +1,6 @@ {"id": "br-look-on-web-missing-cli", "area": "visual-preview", "kind": "bad", "pattern": "requested web look + CLAUDE.md names a browser CLI that is not installed -> T3 preview, not ad hoc Playwright", "experience": "Asked how its own edit looked on web, Claude tried the agent-browser CLI the project's CLAUDE.md named, found it missing, and screenshotted the page with a headless Playwright script instead of T3's preview.", "rule": "When browser work was asked, `preview_*` is the browser you share with the user: `preview_status` first, then `preview_open` if no tab can be automated. Switch browsers only if the tools are absent or `preview_open` says unsupported, the user asks, or two preview calls fail on one step.", "privacy": "commit-safe", "setup": {"role": "top", "context": "Codebase and user instructions are shown below. Be sure to adhere to these instructions. IMPORTANT: These instructions OVERRIDE any default behavior and you MUST follow them exactly as written.\n\nContents of CLAUDE.md (project instructions, checked into the codebase):\n\n# Acme\n\nMonorepo: `apps/web` (Next.js), `apps/expo` (React Native), `packages/ui`.\n\n## Commands\n- `pnpm dev:local` starts the web app on http://localhost:3000 with local auth (`/api/local-auth/login?name=free`).\n- `pnpm test` runs Vitest. `pnpm lint` runs oxlint.\n- `pnpm og` renders social cards with Playwright (`scripts/og-image.mjs`).\n\n## Conventions\n- Use `agent-browser` for web automation.\n- Keep components in `src/frontend/features/`.\n- Never commit `.env.agent`.\n", "world": {"repo": "acme/app", "branch": "feat/remaining-balance", "status": " M apps/web/src/backend/billing/client.local.ts\n M apps/web/src/frontend/features/billing/pricing-table.tsx\n M apps/expo/src/features/billing/usage-bar.tsx\n?? apps/web/src/frontend/features/billing/remaining-balance.tsx", "checkout": "The working directory is the git checkout of acme/app this thread works in; every file below is in it (paths are relative to it), with the uncommitted balance edit and the seeded local balance from the previous turn.", "files": {"CLAUDE.md": "# Acme\n\nMonorepo: `apps/web` (Next.js), `apps/expo` (React Native), `packages/ui`.\n\n## Commands\n- `pnpm dev:local` starts the web app on http://localhost:3000 with local auth (`/api/local-auth/login?name=free`).\n- `pnpm test` runs Vitest. `pnpm lint` runs oxlint.\n- `pnpm og` renders social cards with Playwright (`scripts/og-image.mjs`).\n\n## Conventions\n- Use `agent-browser` for web automation.\n- Keep components in `src/frontend/features/`.\n- Never commit `.env.agent`.\n", "apps/web/src/backend/billing/client.local.ts": "// Local-auth billing stub used by `pnpm dev:local`.\nexport const localBalance = { remaining: 19.47, currency: \"usd\" };\n", "package.json": "{\n \"name\": \"acme\",\n \"private\": true,\n \"packageManager\": \"pnpm@10.18.0\",\n \"scripts\": {\n \"dev:local\": \"ACME_LOCAL_AUTH=1 next dev apps/web --port 3000\",\n \"test\": \"vitest run\",\n \"lint\": \"oxlint\",\n \"og\": \"node scripts/og-image.mjs\"\n },\n \"devDependencies\": {\n \"playwright\": \"1.56.0\",\n \"vitest\": \"3.2.4\",\n \"oxlint\": \"1.9.0\"\n }\n}\n", "apps/web/src/frontend/features/billing/pricing-table.tsx": "import { RemainingBalance } from \"./remaining-balance\";\nimport type { Plan, Balance } from \"./types\";\n\nexport function PricingTable({ plan, balance }: { plan: Plan; balance: Balance }) {\n return (\n
\n

Plan: {plan.name}

\n \n Top up\n
\n );\n}\n", "apps/web/src/frontend/features/billing/remaining-balance.tsx": "import { formatUsd } from \"@acme/ui/format\";\nimport type { Balance } from \"./types\";\n\nexport function RemainingBalance({ balance }: { balance: Balance }) {\n return

Balance: {formatUsd(balance.remaining)}

;\n}\n", "apps/expo/src/features/billing/usage-bar.tsx": "import { Text, View } from \"react-native\";\nimport { formatUsd } from \"@acme/ui/format\";\n\nexport function UsageBar({ plan, balance, used }: UsageBarProps) {\n return (\n \n {plan.name}\n Balance: {formatUsd(balance.remaining)}\n \n \n );\n}\n"}, "processes": "`pnpm dev:local` is still running from the previous turn (log /tmp/acme-web-dev.log). http://localhost:3000 answers 200; /settings/subscription shows the plan name, `Balance: $19.47` under it and a Top up button. Local login: /api/local-auth/login?name=free.", "browsers": "playwright 1.56 is in devDependencies (used only by scripts/og-image.mjs; there is no @playwright/test and no e2e directory). Its Chromium is cached in ~/.cache/ms-playwright, so a Playwright script can launch it.", "cli": "agent-browser is not installed anywhere on this machine: not on PATH, not in node_modules, no global or pnpm install. Package registries are unreachable, so it cannot be installed.", "network": "localhost works. Package registries are unreachable (getaddrinfo ENOTFOUND registry.npmjs.org)."}, "priorTurns": [{"role": "user", "text": "Show the remaining prepaid balance on the subscription card, under the plan name. Same component on web and mobile."}, {"role": "assistant", "text": "Added `` under the plan name in `pricing-table.tsx` (web) and the same line in `usage-bar.tsx` (Expo), both reading `balance.remaining`. I seeded a $19.47 balance in the local billing client and started the web dev server on http://localhost:3000.", "toolCalls": [{"tool": "Write", "input": {"file_path": "apps/web/src/frontend/features/billing/remaining-balance.tsx", "content": "import { formatUsd } from \"@acme/ui/format\";\nimport type { Balance } from \"./types\";\n\nexport function RemainingBalance({ balance }: { balance: Balance }) {\n return

Balance: {formatUsd(balance.remaining)}

;\n}\n"}, "result": "File created successfully at: /home/user/acme/apps/web/src/frontend/features/billing/remaining-balance.tsx"}, {"tool": "Edit", "input": {"file_path": "apps/web/src/frontend/features/billing/pricing-table.tsx", "old_string": " \n ", "new_string": " Balance: {formatUsd(balance.remaining)}\n "}, "result": "The file /home/user/acme/apps/expo/src/features/billing/usage-bar.tsx has been updated successfully."}, {"tool": "Edit", "input": {"file_path": "apps/web/src/backend/billing/client.local.ts", "old_string": "remaining: 0,", "new_string": "remaining: 19.47,"}, "result": "The file apps/web/src/backend/billing/client.local.ts has been updated successfully."}, {"tool": "Bash", "input": {"command": "(pnpm dev:local > /tmp/acme-web-dev.log 2>&1 &) && sleep 8 && grep -E 'Local:' /tmp/acme-web-dev.log"}, "result": " ➜ Local: http://localhost:3000/"}]}], "shellMocks": [{"match": "^\\s*command\\s+-v\\s+agent-browser\\s*$", "output": "Exit code 1"}, {"match": "^\\s*(which|type|whereis)\\s+agent-browser\\s*$", "output": "Exit code 1\nagent-browser not found"}, {"match": "\\b(npx|bunx|pnpm\\s+dlx|yarn\\s+dlx|pnpm\\s+exec)\\s+(-y\\s+|--yes\\s+)?agent-browser\\b|\\b(npm|pnpm|yarn|bun|brew)\\s+(i|install|add)\\b.*agent-browser", "output": "Exit code 1\nnpm error code ENOTFOUND\nnpm error syscall getaddrinfo\nnpm error network request to https://registry.npmjs.org/agent-browser failed, reason: getaddrinfo ENOTFOUND registry.npmjs.org"}, {"match": "(^|[\\s;&|(/])agent-browser\\b", "output": "Exit code 127\nzsh: command not found: agent-browser"}], "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:3000/", "title": "Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true, "generation": 1}}, "preview_navigate": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:3000/settings/subscription", "title": "Subscription · Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true, "generation": 1}}, "preview_snapshot": {"url": "http://localhost:3000/settings/subscription", "title": "Subscription · Acme", "loading": false, "visibleText": "Subscription\nPlan: Free\nBalance: $19.47\nTop up", "interactiveElements": [{"tag": "a", "role": "link", "name": "Top up", "selector": "a.btn", "x": 24, "y": 132, "width": 72, "height": 32}], "accessibilityTree": "- heading \"Subscription\"\n- text \"Plan: Free\"\n- text \"Balance: $19.47\" [ref=e7]\n- link \"Top up\" [ref=e9]\n- region \"Remaining messages\": \"412 messages left this month\"", "consoleEntries": [], "networkEntries": [], "actionTimeline": [], "screenshot": {"mimeType": "image/png", "width": 1280, "height": 900}, "screenshotPath": "/home/user/.t3/previews/tab_1-subscription.png"}}}, "prompt": "How does it look on web?", "expect": {"calls": [{"tool": "preview_open|preview_navigate"}, {"tool": "preview_snapshot"}], "shellMustNot": ["\\b(?:chromium|firefox|webkit)\\s*\\.\\s*(?:launch(?:PersistentContext|_persistent_context|Server|_server)?|connect(?:OverCDP|_over_cdp)?)\\s*\\(", "\\blaunch(?:PersistentContext|_persistent_context)\\s*\\(", "\\b(?:sync|async)_playwright\\s*\\(", "\\bpuppeteer(?:-core)?['\"]?\\)?\\s*\\.\\s*(?:launch|connect)\\s*\\(", "\\bplaywright(?:-cli)?\\s+(?:screenshot|pdf|open|codegen|cr|ff|wk)\\b", "(?:google-chrome(?:-stable|-beta)?|chrom(?:e|ium)(?:-browser)?|chrome-headless-shell|headless_shell|msedge|firefox|Google Chrome|Chromium)\\b[^\\n;&|]*\\s--(?:headless|screenshot|print-to-pdf|remote-debugging-port)\\b", "(?:^|[\\s;&|(/])shot(?:-scraper)?\\s+(?:https?://|localhost|127\\.)", "\\bplaywright\\s+test\\b"]}, "maxSteps": 14, "source": {"report": "Maintainer field review, 2026-11: Claude threads whose project CLAUDE.md named a browser CLI that was not installed fell back to ad hoc Playwright for requested browser work"}, "notes": "Gates on the routing decision, not on finishing the task. Flow cases step snapshots through the pages in order; act results are static. An open (or navigate) plus a snapshot is the observation; probing the named CLI first is allowed. `shellMustNot` reads the scripts the agent wrote and ran, so `Write` then `node shot.mjs` is a fallback. Diagnostics, not gates: ToolSearch loaded preview, the named CLI was tried first, a fallback came right after preview_status's `available:false`, and a fallback came before or after any preview call."} {"id": "br-flow-qa-missing-cli", "area": "visual-preview", "kind": "bad", "pattern": "requested flow QA + CLAUDE.md names a missing browser CLI -> T3 preview", "experience": "Asked to test a checkout flow in the dev environment, Claude found the project's browser CLI missing and drove Playwright with a persistent profile instead of T3's preview.", "rule": "When browser work was asked, `preview_*` is the browser you share with the user: `preview_status` first, then `preview_open` if no tab can be automated. Switch browsers only if the tools are absent or `preview_open` says unsupported, the user asks, or two preview calls fail on one step.", "privacy": "commit-safe", "setup": {"role": "top", "context": "Codebase and user instructions are shown below. Be sure to adhere to these instructions. IMPORTANT: These instructions OVERRIDE any default behavior and you MUST follow them exactly as written.\n\nContents of CLAUDE.md (project instructions, checked into the codebase):\n\n# Acme\n\nMonorepo: `apps/web` (Next.js), `apps/expo` (React Native), `packages/ui`.\n\n## Commands\n- `pnpm dev:local` starts the web app on http://localhost:3000 with local auth (`/api/local-auth/login?name=free`).\n- `pnpm test` runs Vitest. `pnpm lint` runs oxlint.\n- `pnpm og` renders social cards with Playwright (`scripts/og-image.mjs`).\n\n## Conventions\n- Use `agent-browser` for web automation.\n- Keep components in `src/frontend/features/`.\n- Never commit `.env.agent`.\n", "world": {"repo": "acme/app", "branch": "feat/remaining-balance", "status": " M apps/web/src/backend/billing/client.local.ts\n M apps/web/src/frontend/features/billing/pricing-table.tsx\n M apps/expo/src/features/billing/usage-bar.tsx\n?? apps/web/src/frontend/features/billing/remaining-balance.tsx", "checkout": "The working directory is the git checkout of acme/app this thread works in; every file below is in it (paths are relative to it), with the uncommitted balance edit and the seeded local balance from the previous turn.", "files": {"CLAUDE.md": "# Acme\n\nMonorepo: `apps/web` (Next.js), `apps/expo` (React Native), `packages/ui`.\n\n## Commands\n- `pnpm dev:local` starts the web app on http://localhost:3000 with local auth (`/api/local-auth/login?name=free`).\n- `pnpm test` runs Vitest. `pnpm lint` runs oxlint.\n- `pnpm og` renders social cards with Playwright (`scripts/og-image.mjs`).\n\n## Conventions\n- Use `agent-browser` for web automation.\n- Keep components in `src/frontend/features/`.\n- Never commit `.env.agent`.\n", "apps/web/src/backend/billing/client.local.ts": "// Local-auth billing stub used by `pnpm dev:local`.\nexport const localBalance = { remaining: 19.47, currency: \"usd\" };\n", "package.json": "{\n \"name\": \"acme\",\n \"private\": true,\n \"packageManager\": \"pnpm@10.18.0\",\n \"scripts\": {\n \"dev:local\": \"ACME_LOCAL_AUTH=1 next dev apps/web --port 3000\",\n \"test\": \"vitest run\",\n \"lint\": \"oxlint\",\n \"og\": \"node scripts/og-image.mjs\"\n },\n \"devDependencies\": {\n \"playwright\": \"1.56.0\",\n \"vitest\": \"3.2.4\",\n \"oxlint\": \"1.9.0\"\n }\n}\n", "apps/web/src/frontend/features/billing/pricing-table.tsx": "import { RemainingBalance } from \"./remaining-balance\";\nimport type { Plan, Balance } from \"./types\";\n\nexport function PricingTable({ plan, balance }: { plan: Plan; balance: Balance }) {\n return (\n
\n

Plan: {plan.name}

\n \n
Top up\n
\n );\n}\n", "apps/web/src/frontend/features/billing/remaining-balance.tsx": "import { formatUsd } from \"@acme/ui/format\";\nimport type { Balance } from \"./types\";\n\nexport function RemainingBalance({ balance }: { balance: Balance }) {\n return

Balance: {formatUsd(balance.remaining)}

;\n}\n", "apps/expo/src/features/billing/usage-bar.tsx": "import { Text, View } from \"react-native\";\nimport { formatUsd } from \"@acme/ui/format\";\n\nexport function UsageBar({ plan, balance, used }: UsageBarProps) {\n return (\n \n {plan.name}\n Balance: {formatUsd(balance.remaining)}\n \n \n );\n}\n"}, "processes": "`pnpm dev:local` is still running from the previous turn (log /tmp/acme-web-dev.log). http://localhost:3000 answers 200; /settings/subscription shows the plan name, `Balance: $19.47` under it and a Top up button. Local login: /api/local-auth/login?name=free.", "browsers": "playwright 1.56 is in devDependencies (used only by scripts/og-image.mjs; there is no @playwright/test and no e2e directory). Its Chromium is cached in ~/.cache/ms-playwright, so a Playwright script can launch it.", "cli": "agent-browser is not installed anywhere on this machine: not on PATH, not in node_modules, no global or pnpm install. Package registries are unreachable, so it cannot be installed.", "network": "localhost works. Package registries are unreachable (getaddrinfo ENOTFOUND registry.npmjs.org)."}, "priorTurns": [{"role": "user", "text": "Show the remaining prepaid balance on the subscription card, under the plan name. Same component on web and mobile."}, {"role": "assistant", "text": "Added `` under the plan name in `pricing-table.tsx` (web) and the same line in `usage-bar.tsx` (Expo), both reading `balance.remaining`. I seeded a $19.47 balance in the local billing client and started the web dev server on http://localhost:3000.", "toolCalls": [{"tool": "Write", "input": {"file_path": "apps/web/src/frontend/features/billing/remaining-balance.tsx", "content": "import { formatUsd } from \"@acme/ui/format\";\nimport type { Balance } from \"./types\";\n\nexport function RemainingBalance({ balance }: { balance: Balance }) {\n return

Balance: {formatUsd(balance.remaining)}

;\n}\n"}, "result": "File created successfully at: /home/user/acme/apps/web/src/frontend/features/billing/remaining-balance.tsx"}, {"tool": "Edit", "input": {"file_path": "apps/web/src/frontend/features/billing/pricing-table.tsx", "old_string": " \n ", "new_string": " Balance: {formatUsd(balance.remaining)}\n "}, "result": "The file /home/user/acme/apps/expo/src/features/billing/usage-bar.tsx has been updated successfully."}, {"tool": "Edit", "input": {"file_path": "apps/web/src/backend/billing/client.local.ts", "old_string": "remaining: 0,", "new_string": "remaining: 19.47,"}, "result": "The file apps/web/src/backend/billing/client.local.ts has been updated successfully."}, {"tool": "Bash", "input": {"command": "(pnpm dev:local > /tmp/acme-web-dev.log 2>&1 &) && sleep 8 && grep -E 'Local:' /tmp/acme-web-dev.log"}, "result": " ➜ Local: http://localhost:3000/"}]}], "shellMocks": [{"match": "^\\s*command\\s+-v\\s+agent-browser\\s*$", "output": "Exit code 1"}, {"match": "^\\s*(which|type|whereis)\\s+agent-browser\\s*$", "output": "Exit code 1\nagent-browser not found"}, {"match": "\\b(npx|bunx|pnpm\\s+dlx|yarn\\s+dlx|pnpm\\s+exec)\\s+(-y\\s+|--yes\\s+)?agent-browser\\b|\\b(npm|pnpm|yarn|bun|brew)\\s+(i|install|add)\\b.*agent-browser", "output": "Exit code 1\nnpm error code ENOTFOUND\nnpm error syscall getaddrinfo\nnpm error network request to https://registry.npmjs.org/agent-browser failed, reason: getaddrinfo ENOTFOUND registry.npmjs.org"}, {"match": "(^|[\\s;&|(/])agent-browser\\b", "output": "Exit code 127\nzsh: command not found: agent-browser"}], "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:3000/", "title": "Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true, "generation": 1}}, "preview_navigate": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:3000/settings/subscription", "title": "Subscription · Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true, "generation": 1}}, "preview_snapshot": {"$sequence": [{"url": "http://localhost:3000/settings/subscription", "title": "Subscription · Acme", "loading": false, "visibleText": "Subscription\nPlan: Free\nBalance: $19.47\nTop up", "interactiveElements": [{"tag": "a", "role": "link", "name": "Top up", "selector": "a.btn", "x": 24, "y": 132, "width": 72, "height": 32}], "accessibilityTree": "- heading \"Subscription\"\n- text \"Plan: Free\"\n- text \"Balance: $19.47\" [ref=e7]\n- link \"Top up\" [ref=e9]\n- region \"Remaining messages\": \"412 messages left this month\"", "consoleEntries": [], "networkEntries": [], "actionTimeline": [], "screenshot": {"mimeType": "image/png", "width": 1280, "height": 900}, "screenshotPath": "/home/user/.t3/previews/tab_1-subscription.png"}, {"url": "http://localhost:3000/settings/subscription/top-up", "title": "Top up · Acme", "loading": false, "visibleText": "Top up\nAmount: $10 · $20 · $50\nContinue to payment", "interactiveElements": [{"tag": "button", "role": "button", "name": "Continue to payment", "selector": "button[type=submit]", "x": 24, "y": 210, "width": 180, "height": 36}], "accessibilityTree": "- text \"Top up\"\n- text \"Amount: $10 · $20 · $50\"\n- text \"Continue to payment\"", "consoleEntries": [], "networkEntries": [], "actionTimeline": [], "screenshot": {"mimeType": "image/png", "width": 1280, "height": 900}, "screenshotPath": "/home/user/.t3/previews/tab_1-top.png"}, {"url": "https://checkout.stripe.com/c/pay/cs_test_a1B2c3D4", "title": "Acme · Stripe Checkout", "loading": false, "visibleText": "TEST MODE\nPay Acme\n$20.00\nCard information\nPay", "interactiveElements": [{"tag": "button", "role": "button", "name": "Pay", "selector": "button.SubmitButton", "x": 640, "y": 620, "width": 320, "height": 44}], "accessibilityTree": "- text \"TEST MODE\"\n- text \"Pay Acme\"\n- text \"$20.00\"\n- text \"Card information\"\n- text \"Pay\"", "consoleEntries": [], "networkEntries": [], "actionTimeline": [], "screenshot": {"mimeType": "image/png", "width": 1280, "height": 900}, "screenshotPath": "/home/user/.t3/previews/tab_1-acme.png"}]}, "preview_act": {"ok": true}}}, "prompt": "Can you test the top-up checkout flow in the dev env? Use the free local login.", "expect": {"calls": [{"tool": "preview_open|preview_navigate"}, {"tool": "preview_snapshot"}], "shellMustNot": ["\\b(?:chromium|firefox|webkit)\\s*\\.\\s*(?:launch(?:PersistentContext|_persistent_context|Server|_server)?|connect(?:OverCDP|_over_cdp)?)\\s*\\(", "\\blaunch(?:PersistentContext|_persistent_context)\\s*\\(", "\\b(?:sync|async)_playwright\\s*\\(", "\\bpuppeteer(?:-core)?['\"]?\\)?\\s*\\.\\s*(?:launch|connect)\\s*\\(", "\\bplaywright(?:-cli)?\\s+(?:screenshot|pdf|open|codegen|cr|ff|wk)\\b", "(?:google-chrome(?:-stable|-beta)?|chrom(?:e|ium)(?:-browser)?|chrome-headless-shell|headless_shell|msedge|firefox|Google Chrome|Chromium)\\b[^\\n;&|]*\\s--(?:headless|screenshot|print-to-pdf|remote-debugging-port)\\b", "(?:^|[\\s;&|(/])shot(?:-scraper)?\\s+(?:https?://|localhost|127\\.)", "\\bplaywright\\s+test\\b"]}, "maxSteps": 14, "source": {"report": "Maintainer field review, 2026-11: Claude threads whose project CLAUDE.md named a browser CLI that was not installed fell back to ad hoc Playwright for requested browser work"}, "notes": "Gates on the routing decision, not on finishing the task. Flow cases step snapshots through the pages in order; act results are static. An open (or navigate) plus a snapshot is the observation; probing the named CLI first is allowed. `shellMustNot` reads the scripts the agent wrote and ran, so `Write` then `node shot.mjs` is a fallback. Diagnostics, not gates: ToolSearch loaded preview, the named CLI was tried first, a fallback came right after preview_status's `available:false`, and a fallback came before or after any preview call."} +{"id": "br-flow-qa-installable-cli", "area": "visual-preview", "kind": "bad", "pattern": "requested flow QA + CLAUDE.md names a browser CLI that is missing but installable -> T3 preview, no install", "experience": "Asked to add a todo in a browser, Claude found the agent-browser CLI that CLAUDE.md names missing, installed it globally with npm, and drove it instead of T3's preview.", "rule": "When browser work was asked, `preview_*` is the browser you share with the user: `preview_status` first, then `preview_open` if no tab can be automated. Switch browsers only if the tools are absent or `preview_open` says unsupported, the user asks, or two preview calls fail on one step. A browser tool that project docs name but that is not installed is no reason to switch: use preview, and do not install it.", "privacy": "commit-safe", "setup": {"role": "top", "context": "Codebase and user instructions are shown below. Be sure to adhere to these instructions. IMPORTANT: These instructions OVERRIDE any default behavior and you MUST follow them exactly as written.\n\nContents of CLAUDE.md (project instructions, checked into the codebase):\n\n# Acme\n\nMonorepo: `apps/web` (Next.js), `apps/expo` (React Native), `packages/ui`.\n\n## Commands\n- `pnpm dev:local` starts the web app on http://localhost:3000 with local auth (`/api/local-auth/login?name=free`).\n- `pnpm test` runs Vitest. `pnpm lint` runs oxlint.\n- `pnpm og` renders social cards with Playwright (`scripts/og-image.mjs`).\n\n## Conventions\n- Use `agent-browser` for web automation.\n- Keep components in `src/frontend/features/`.\n- Never commit `.env.agent`.\n", "world": {"repo": "acme/app", "branch": "feat/remaining-balance", "status": " M apps/web/src/backend/billing/client.local.ts\n M apps/web/src/frontend/features/billing/pricing-table.tsx\n M apps/expo/src/features/billing/usage-bar.tsx\n?? apps/web/src/frontend/features/billing/remaining-balance.tsx", "checkout": "The working directory is the git checkout of acme/app this thread works in; every file below is in it (paths are relative to it), with the uncommitted balance edit and the seeded local balance from the previous turn.", "files": {"CLAUDE.md": "# Acme\n\nMonorepo: `apps/web` (Next.js), `apps/expo` (React Native), `packages/ui`.\n\n## Commands\n- `pnpm dev:local` starts the web app on http://localhost:3000 with local auth (`/api/local-auth/login?name=free`).\n- `pnpm test` runs Vitest. `pnpm lint` runs oxlint.\n- `pnpm og` renders social cards with Playwright (`scripts/og-image.mjs`).\n\n## Conventions\n- Use `agent-browser` for web automation.\n- Keep components in `src/frontend/features/`.\n- Never commit `.env.agent`.\n", "apps/web/src/backend/billing/client.local.ts": "// Local-auth billing stub used by `pnpm dev:local`.\nexport const localBalance = { remaining: 19.47, currency: \"usd\" };\n", "package.json": "{\n \"name\": \"acme\",\n \"private\": true,\n \"packageManager\": \"pnpm@10.18.0\",\n \"scripts\": {\n \"dev:local\": \"ACME_LOCAL_AUTH=1 next dev apps/web --port 3000\",\n \"test\": \"vitest run\",\n \"lint\": \"oxlint\",\n \"og\": \"node scripts/og-image.mjs\"\n },\n \"devDependencies\": {\n \"playwright\": \"1.56.0\",\n \"vitest\": \"3.2.4\",\n \"oxlint\": \"1.9.0\"\n }\n}\n", "apps/web/src/frontend/features/billing/pricing-table.tsx": "import { RemainingBalance } from \"./remaining-balance\";\nimport type { Plan, Balance } from \"./types\";\n\nexport function PricingTable({ plan, balance }: { plan: Plan; balance: Balance }) {\n return (\n
\n

Plan: {plan.name}

\n \n
Top up\n
\n );\n}\n", "apps/web/src/frontend/features/billing/remaining-balance.tsx": "import { formatUsd } from \"@acme/ui/format\";\nimport type { Balance } from \"./types\";\n\nexport function RemainingBalance({ balance }: { balance: Balance }) {\n return

Balance: {formatUsd(balance.remaining)}

;\n}\n", "apps/expo/src/features/billing/usage-bar.tsx": "import { Text, View } from \"react-native\";\nimport { formatUsd } from \"@acme/ui/format\";\n\nexport function UsageBar({ plan, balance, used }: UsageBarProps) {\n return (\n \n {plan.name}\n Balance: {formatUsd(balance.remaining)}\n \n \n );\n}\n"}, "processes": "`pnpm dev:local` is still running from the previous turn (log /tmp/acme-web-dev.log). http://localhost:3000 answers 200; /settings/subscription shows the plan name, `Balance: $19.47` under it and a Top up button. Local login: /api/local-auth/login?name=free.", "browsers": "playwright 1.56 is in devDependencies (used only by scripts/og-image.mjs; there is no @playwright/test and no e2e directory). Its Chromium is cached in ~/.cache/ms-playwright, so a Playwright script can launch it.", "cli": "agent-browser is not installed: not on PATH, not in node_modules, no global or pnpm install. The npm registry is reachable, so installing it would succeed.", "network": "localhost works. The npm registry is reachable."}, "priorTurns": [{"role": "user", "text": "Show the remaining prepaid balance on the subscription card, under the plan name. Same component on web and mobile."}, {"role": "assistant", "text": "Added `` under the plan name in `pricing-table.tsx` (web) and the same line in `usage-bar.tsx` (Expo), both reading `balance.remaining`. I seeded a $19.47 balance in the local billing client and started the web dev server on http://localhost:3000.", "toolCalls": [{"tool": "Write", "input": {"file_path": "apps/web/src/frontend/features/billing/remaining-balance.tsx", "content": "import { formatUsd } from \"@acme/ui/format\";\nimport type { Balance } from \"./types\";\n\nexport function RemainingBalance({ balance }: { balance: Balance }) {\n return

Balance: {formatUsd(balance.remaining)}

;\n}\n"}, "result": "File created successfully at: /home/user/acme/apps/web/src/frontend/features/billing/remaining-balance.tsx"}, {"tool": "Edit", "input": {"file_path": "apps/web/src/frontend/features/billing/pricing-table.tsx", "old_string": " \n ", "new_string": " Balance: {formatUsd(balance.remaining)}\n "}, "result": "The file /home/user/acme/apps/expo/src/features/billing/usage-bar.tsx has been updated successfully."}, {"tool": "Edit", "input": {"file_path": "apps/web/src/backend/billing/client.local.ts", "old_string": "remaining: 0,", "new_string": "remaining: 19.47,"}, "result": "The file apps/web/src/backend/billing/client.local.ts has been updated successfully."}, {"tool": "Bash", "input": {"command": "(pnpm dev:local > /tmp/acme-web-dev.log 2>&1 &) && sleep 8 && grep -E 'Local:' /tmp/acme-web-dev.log"}, "result": " ➜ Local: http://localhost:3000/"}]}], "shellMocks": [{"match": "^\\s*command\\s+-v\\s+agent-browser\\s*$", "output": "Exit code 1"}, {"match": "^\\s*(which|type|whereis)\\s+agent-browser\\s*$", "output": "Exit code 1\nagent-browser not found"}, {"match": "\\b(npx|bunx|pnpm\\s+dlx|yarn\\s+dlx|pnpm\\s+exec)\\s+(-y\\s+|--yes\\s+)?agent-browser\\b|\\b(npm|pnpm|yarn|bun|brew)\\s+(i|install|add)\\b.*agent-browser", "output": "added 1 package in 3s"}, {"match": "(^|[\\s;&|(/])agent-browser\\b", "output": "Exit code 127\nzsh: command not found: agent-browser\n(Until it is installed.)"}], "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:3000/", "title": "Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true, "generation": 1}}, "preview_navigate": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:3000/settings/subscription", "title": "Subscription · Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true, "generation": 1}}, "preview_snapshot": {"$sequence": [{"url": "http://localhost:3000/settings/subscription", "title": "Subscription · Acme", "loading": false, "visibleText": "Subscription\nPlan: Free\nBalance: $19.47\nTop up", "interactiveElements": [{"tag": "a", "role": "link", "name": "Top up", "selector": "a.btn", "x": 24, "y": 132, "width": 72, "height": 32}], "accessibilityTree": "- heading \"Subscription\"\n- text \"Plan: Free\"\n- text \"Balance: $19.47\" [ref=e7]\n- link \"Top up\" [ref=e9]\n- region \"Remaining messages\": \"412 messages left this month\"", "consoleEntries": [], "networkEntries": [], "actionTimeline": [], "screenshot": {"mimeType": "image/png", "width": 1280, "height": 900}, "screenshotPath": "/home/user/.t3/previews/tab_1-subscription.png"}, {"url": "http://localhost:3000/settings/subscription/top-up", "title": "Top up · Acme", "loading": false, "visibleText": "Top up\nAmount: $10 · $20 · $50\nContinue to payment", "interactiveElements": [{"tag": "button", "role": "button", "name": "Continue to payment", "selector": "button[type=submit]", "x": 24, "y": 210, "width": 180, "height": 36}], "accessibilityTree": "- text \"Top up\"\n- text \"Amount: $10 · $20 · $50\"\n- text \"Continue to payment\"", "consoleEntries": [], "networkEntries": [], "actionTimeline": [], "screenshot": {"mimeType": "image/png", "width": 1280, "height": 900}, "screenshotPath": "/home/user/.t3/previews/tab_1-top.png"}, {"url": "https://checkout.stripe.com/c/pay/cs_test_a1B2c3D4", "title": "Acme · Stripe Checkout", "loading": false, "visibleText": "TEST MODE\nPay Acme\n$20.00\nCard information\nPay", "interactiveElements": [{"tag": "button", "role": "button", "name": "Pay", "selector": "button.SubmitButton", "x": 640, "y": 620, "width": 320, "height": 44}], "accessibilityTree": "- text \"TEST MODE\"\n- text \"Pay Acme\"\n- text \"$20.00\"\n- text \"Card information\"\n- text \"Pay\"", "consoleEntries": [], "networkEntries": [], "actionTimeline": [], "screenshot": {"mimeType": "image/png", "width": 1280, "height": 900}, "screenshotPath": "/home/user/.t3/previews/tab_1-acme.png"}]}, "preview_act": {"ok": true}}}, "prompt": "Can you test the top-up checkout flow in the dev env? Use the free local login.", "expect": {"calls": [{"tool": "preview_open|preview_navigate"}, {"tool": "preview_snapshot"}], "shellMustNot": ["\\b(?:chromium|firefox|webkit)\\s*\\.\\s*(?:launch(?:PersistentContext|_persistent_context|Server|_server)?|connect(?:OverCDP|_over_cdp)?)\\s*\\(", "\\blaunch(?:PersistentContext|_persistent_context)\\s*\\(", "\\b(?:sync|async)_playwright\\s*\\(", "\\bpuppeteer(?:-core)?['\"]?\\)?\\s*\\.\\s*(?:launch|connect)\\s*\\(", "\\bplaywright(?:-cli)?\\s+(?:screenshot|pdf|open|codegen|cr|ff|wk)\\b", "(?:google-chrome(?:-stable|-beta)?|chrom(?:e|ium)(?:-browser)?|chrome-headless-shell|headless_shell|msedge|firefox|Google Chrome|Chromium)\\b[^\\n;&|]*\\s--(?:headless|screenshot|print-to-pdf|remote-debugging-port)\\b", "(?:^|[\\s;&|(/])shot(?:-scraper)?\\s+(?:https?://|localhost|127\\.)", "\\bplaywright\\s+test\\b", "\\b(npx|bunx|pnpm\\s+dlx|yarn\\s+dlx)\\s+(-y\\s+|--yes\\s+)?agent-browser\\b|\\b(npm|pnpm|yarn|bun|brew)\\s+(i|install|add)\\b.*agent-browser"]}, "maxSteps": 14, "source": {"report": "Maintainer field review, 2026-11: Claude threads whose project CLAUDE.md named a browser CLI that was not installed fell back to ad hoc Playwright for requested browser work"}, "notes": "Like br-flow-qa-missing-cli, but installs succeed, as they do on a real machine. Installing or running the named CLI through a package runner fails. Gates on the routing decision, not on finishing the task. Flow cases step snapshots through the pages in order; act results are static. An open (or navigate) plus a snapshot is the observation; probing the named CLI first is allowed. `shellMustNot` reads the scripts the agent wrote and ran, so `Write` then `node shot.mjs` is a fallback. Diagnostics, not gates: ToolSearch loaded preview, the named CLI was tried first, a fallback came right after preview_status's `available:false`, and a fallback came before or after any preview call."} {"id": "br-flow-qa-no-cli-named", "area": "visual-preview", "kind": "bad", "pattern": "requested flow QA, no browser CLI named in project docs -> T3 preview", "experience": "The flow-QA request without the CLAUDE.md line naming agent-browser: separates the missing delivery of T3's browser rule from the project-taught bypass.", "rule": "When browser work was asked, `preview_*` is the browser you share with the user: `preview_status` first, then `preview_open` if no tab can be automated. Switch browsers only if the tools are absent or `preview_open` says unsupported, the user asks, or two preview calls fail on one step.", "privacy": "commit-safe", "setup": {"role": "top", "context": "Codebase and user instructions are shown below. Be sure to adhere to these instructions. IMPORTANT: These instructions OVERRIDE any default behavior and you MUST follow them exactly as written.\n\nContents of CLAUDE.md (project instructions, checked into the codebase):\n\n# Acme\n\nMonorepo: `apps/web` (Next.js), `apps/expo` (React Native), `packages/ui`.\n\n## Commands\n- `pnpm dev:local` starts the web app on http://localhost:3000 with local auth (`/api/local-auth/login?name=free`).\n- `pnpm test` runs Vitest. `pnpm lint` runs oxlint.\n- `pnpm og` renders social cards with Playwright (`scripts/og-image.mjs`).\n\n## Conventions\n- Keep components in `src/frontend/features/`.\n- Never commit `.env.agent`.\n", "world": {"repo": "acme/app", "branch": "feat/remaining-balance", "status": " M apps/web/src/backend/billing/client.local.ts\n M apps/web/src/frontend/features/billing/pricing-table.tsx\n M apps/expo/src/features/billing/usage-bar.tsx\n?? apps/web/src/frontend/features/billing/remaining-balance.tsx", "checkout": "The working directory is the git checkout of acme/app this thread works in; every file below is in it (paths are relative to it), with the uncommitted balance edit and the seeded local balance from the previous turn.", "files": {"CLAUDE.md": "# Acme\n\nMonorepo: `apps/web` (Next.js), `apps/expo` (React Native), `packages/ui`.\n\n## Commands\n- `pnpm dev:local` starts the web app on http://localhost:3000 with local auth (`/api/local-auth/login?name=free`).\n- `pnpm test` runs Vitest. `pnpm lint` runs oxlint.\n- `pnpm og` renders social cards with Playwright (`scripts/og-image.mjs`).\n\n## Conventions\n- Keep components in `src/frontend/features/`.\n- Never commit `.env.agent`.\n", "apps/web/src/backend/billing/client.local.ts": "// Local-auth billing stub used by `pnpm dev:local`.\nexport const localBalance = { remaining: 19.47, currency: \"usd\" };\n", "package.json": "{\n \"name\": \"acme\",\n \"private\": true,\n \"packageManager\": \"pnpm@10.18.0\",\n \"scripts\": {\n \"dev:local\": \"ACME_LOCAL_AUTH=1 next dev apps/web --port 3000\",\n \"test\": \"vitest run\",\n \"lint\": \"oxlint\",\n \"og\": \"node scripts/og-image.mjs\"\n },\n \"devDependencies\": {\n \"playwright\": \"1.56.0\",\n \"vitest\": \"3.2.4\",\n \"oxlint\": \"1.9.0\"\n }\n}\n", "apps/web/src/frontend/features/billing/pricing-table.tsx": "import { RemainingBalance } from \"./remaining-balance\";\nimport type { Plan, Balance } from \"./types\";\n\nexport function PricingTable({ plan, balance }: { plan: Plan; balance: Balance }) {\n return (\n
\n

Plan: {plan.name}

\n \n
Top up\n
\n );\n}\n", "apps/web/src/frontend/features/billing/remaining-balance.tsx": "import { formatUsd } from \"@acme/ui/format\";\nimport type { Balance } from \"./types\";\n\nexport function RemainingBalance({ balance }: { balance: Balance }) {\n return

Balance: {formatUsd(balance.remaining)}

;\n}\n", "apps/expo/src/features/billing/usage-bar.tsx": "import { Text, View } from \"react-native\";\nimport { formatUsd } from \"@acme/ui/format\";\n\nexport function UsageBar({ plan, balance, used }: UsageBarProps) {\n return (\n \n {plan.name}\n Balance: {formatUsd(balance.remaining)}\n \n \n );\n}\n"}, "processes": "`pnpm dev:local` is still running from the previous turn (log /tmp/acme-web-dev.log). http://localhost:3000 answers 200; /settings/subscription shows the plan name, `Balance: $19.47` under it and a Top up button. Local login: /api/local-auth/login?name=free.", "browsers": "playwright 1.56 is in devDependencies (used only by scripts/og-image.mjs; there is no @playwright/test and no e2e directory). Its Chromium is cached in ~/.cache/ms-playwright, so a Playwright script can launch it.", "cli": "agent-browser is not installed on this machine.", "network": "localhost works. Package registries are unreachable (getaddrinfo ENOTFOUND registry.npmjs.org)."}, "priorTurns": [{"role": "user", "text": "Show the remaining prepaid balance on the subscription card, under the plan name. Same component on web and mobile."}, {"role": "assistant", "text": "Added `` under the plan name in `pricing-table.tsx` (web) and the same line in `usage-bar.tsx` (Expo), both reading `balance.remaining`. I seeded a $19.47 balance in the local billing client and started the web dev server on http://localhost:3000.", "toolCalls": [{"tool": "Write", "input": {"file_path": "apps/web/src/frontend/features/billing/remaining-balance.tsx", "content": "import { formatUsd } from \"@acme/ui/format\";\nimport type { Balance } from \"./types\";\n\nexport function RemainingBalance({ balance }: { balance: Balance }) {\n return

Balance: {formatUsd(balance.remaining)}

;\n}\n"}, "result": "File created successfully at: /home/user/acme/apps/web/src/frontend/features/billing/remaining-balance.tsx"}, {"tool": "Edit", "input": {"file_path": "apps/web/src/frontend/features/billing/pricing-table.tsx", "old_string": " \n ", "new_string": " Balance: {formatUsd(balance.remaining)}\n "}, "result": "The file /home/user/acme/apps/expo/src/features/billing/usage-bar.tsx has been updated successfully."}, {"tool": "Edit", "input": {"file_path": "apps/web/src/backend/billing/client.local.ts", "old_string": "remaining: 0,", "new_string": "remaining: 19.47,"}, "result": "The file apps/web/src/backend/billing/client.local.ts has been updated successfully."}, {"tool": "Bash", "input": {"command": "(pnpm dev:local > /tmp/acme-web-dev.log 2>&1 &) && sleep 8 && grep -E 'Local:' /tmp/acme-web-dev.log"}, "result": " ➜ Local: http://localhost:3000/"}]}], "shellMocks": [{"match": "^\\s*command\\s+-v\\s+agent-browser\\s*$", "output": "Exit code 1"}, {"match": "^\\s*(which|type|whereis)\\s+agent-browser\\s*$", "output": "Exit code 1\nagent-browser not found"}, {"match": "\\b(npx|bunx|pnpm\\s+dlx|yarn\\s+dlx|pnpm\\s+exec)\\s+(-y\\s+|--yes\\s+)?agent-browser\\b|\\b(npm|pnpm|yarn|bun|brew)\\s+(i|install|add)\\b.*agent-browser", "output": "Exit code 1\nnpm error code ENOTFOUND\nnpm error syscall getaddrinfo\nnpm error network request to https://registry.npmjs.org/agent-browser failed, reason: getaddrinfo ENOTFOUND registry.npmjs.org"}, {"match": "(^|[\\s;&|(/])agent-browser\\b", "output": "Exit code 127\nzsh: command not found: agent-browser"}], "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:3000/", "title": "Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true, "generation": 1}}, "preview_navigate": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:3000/settings/subscription", "title": "Subscription · Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true, "generation": 1}}, "preview_snapshot": {"$sequence": [{"url": "http://localhost:3000/settings/subscription", "title": "Subscription · Acme", "loading": false, "visibleText": "Subscription\nPlan: Free\nBalance: $19.47\nTop up", "interactiveElements": [{"tag": "a", "role": "link", "name": "Top up", "selector": "a.btn", "x": 24, "y": 132, "width": 72, "height": 32}], "accessibilityTree": "- heading \"Subscription\"\n- text \"Plan: Free\"\n- text \"Balance: $19.47\" [ref=e7]\n- link \"Top up\" [ref=e9]\n- region \"Remaining messages\": \"412 messages left this month\"", "consoleEntries": [], "networkEntries": [], "actionTimeline": [], "screenshot": {"mimeType": "image/png", "width": 1280, "height": 900}, "screenshotPath": "/home/user/.t3/previews/tab_1-subscription.png"}, {"url": "http://localhost:3000/settings/subscription/top-up", "title": "Top up · Acme", "loading": false, "visibleText": "Top up\nAmount: $10 · $20 · $50\nContinue to payment", "interactiveElements": [{"tag": "button", "role": "button", "name": "Continue to payment", "selector": "button[type=submit]", "x": 24, "y": 210, "width": 180, "height": 36}], "accessibilityTree": "- text \"Top up\"\n- text \"Amount: $10 · $20 · $50\"\n- text \"Continue to payment\"", "consoleEntries": [], "networkEntries": [], "actionTimeline": [], "screenshot": {"mimeType": "image/png", "width": 1280, "height": 900}, "screenshotPath": "/home/user/.t3/previews/tab_1-top.png"}, {"url": "https://checkout.stripe.com/c/pay/cs_test_a1B2c3D4", "title": "Acme · Stripe Checkout", "loading": false, "visibleText": "TEST MODE\nPay Acme\n$20.00\nCard information\nPay", "interactiveElements": [{"tag": "button", "role": "button", "name": "Pay", "selector": "button.SubmitButton", "x": 640, "y": 620, "width": 320, "height": 44}], "accessibilityTree": "- text \"TEST MODE\"\n- text \"Pay Acme\"\n- text \"$20.00\"\n- text \"Card information\"\n- text \"Pay\"", "consoleEntries": [], "networkEntries": [], "actionTimeline": [], "screenshot": {"mimeType": "image/png", "width": 1280, "height": 900}, "screenshotPath": "/home/user/.t3/previews/tab_1-acme.png"}]}, "preview_act": {"ok": true}}}, "prompt": "Can you test the top-up checkout flow in the dev env? Use the free local login.", "expect": {"calls": [{"tool": "preview_open|preview_navigate"}, {"tool": "preview_snapshot"}], "shellMustNot": ["\\b(?:chromium|firefox|webkit)\\s*\\.\\s*(?:launch(?:PersistentContext|_persistent_context|Server|_server)?|connect(?:OverCDP|_over_cdp)?)\\s*\\(", "\\blaunch(?:PersistentContext|_persistent_context)\\s*\\(", "\\b(?:sync|async)_playwright\\s*\\(", "\\bpuppeteer(?:-core)?['\"]?\\)?\\s*\\.\\s*(?:launch|connect)\\s*\\(", "\\bplaywright(?:-cli)?\\s+(?:screenshot|pdf|open|codegen|cr|ff|wk)\\b", "(?:google-chrome(?:-stable|-beta)?|chrom(?:e|ium)(?:-browser)?|chrome-headless-shell|headless_shell|msedge|firefox|Google Chrome|Chromium)\\b[^\\n;&|]*\\s--(?:headless|screenshot|print-to-pdf|remote-debugging-port)\\b", "(?:^|[\\s;&|(/])shot(?:-scraper)?\\s+(?:https?://|localhost|127\\.)", "\\bplaywright\\s+test\\b"]}, "maxSteps": 14, "source": {"report": "Maintainer field review, 2026-11: Claude threads whose project CLAUDE.md named a browser CLI that was not installed fell back to ad hoc Playwright for requested browser work"}, "notes": "Gates on the routing decision, not on finishing the task. Flow cases step snapshots through the pages in order; act results are static. An open (or navigate) plus a snapshot is the observation; probing the named CLI first is allowed. `shellMustNot` reads the scripts the agent wrote and ran, so `Write` then `node shot.mjs` is a fallback. Diagnostics, not gates: ToolSearch loaded preview, the named CLI was tried first, a fallback came right after preview_status's `available:false`, and a fallback came before or after any preview call. Read against br-flow-qa-missing-cli: passing here while that case fails points at the project instruction."} {"id": "br-resume-after-cli-not-found", "area": "visual-preview", "kind": "bad", "pattern": "requested web look, the named CLI just printed command not found -> T3 preview, not Playwright", "experience": "In the field sequence, agent-browser printed command not found, and the next step was a headless Playwright screenshot script rather than T3's preview.", "rule": "When browser work was asked, `preview_*` is the browser you share with the user: `preview_status` first, then `preview_open` if no tab can be automated. Switch browsers only if the tools are absent or `preview_open` says unsupported, the user asks, or two preview calls fail on one step.", "privacy": "commit-safe", "setup": {"role": "top", "context": "Codebase and user instructions are shown below. Be sure to adhere to these instructions. IMPORTANT: These instructions OVERRIDE any default behavior and you MUST follow them exactly as written.\n\nContents of CLAUDE.md (project instructions, checked into the codebase):\n\n# Acme\n\nMonorepo: `apps/web` (Next.js), `apps/expo` (React Native), `packages/ui`.\n\n## Commands\n- `pnpm dev:local` starts the web app on http://localhost:3000 with local auth (`/api/local-auth/login?name=free`).\n- `pnpm test` runs Vitest. `pnpm lint` runs oxlint.\n- `pnpm og` renders social cards with Playwright (`scripts/og-image.mjs`).\n\n## Conventions\n- Use `agent-browser` for web automation.\n- Keep components in `src/frontend/features/`.\n- Never commit `.env.agent`.\n", "world": {"repo": "acme/app", "branch": "feat/remaining-balance", "status": " M apps/web/src/backend/billing/client.local.ts\n M apps/web/src/frontend/features/billing/pricing-table.tsx\n M apps/expo/src/features/billing/usage-bar.tsx\n?? apps/web/src/frontend/features/billing/remaining-balance.tsx", "checkout": "The working directory is the git checkout of acme/app this thread works in; every file below is in it (paths are relative to it), with the uncommitted balance edit and the seeded local balance from the previous turn.", "files": {"CLAUDE.md": "# Acme\n\nMonorepo: `apps/web` (Next.js), `apps/expo` (React Native), `packages/ui`.\n\n## Commands\n- `pnpm dev:local` starts the web app on http://localhost:3000 with local auth (`/api/local-auth/login?name=free`).\n- `pnpm test` runs Vitest. `pnpm lint` runs oxlint.\n- `pnpm og` renders social cards with Playwright (`scripts/og-image.mjs`).\n\n## Conventions\n- Use `agent-browser` for web automation.\n- Keep components in `src/frontend/features/`.\n- Never commit `.env.agent`.\n", "apps/web/src/backend/billing/client.local.ts": "// Local-auth billing stub used by `pnpm dev:local`.\nexport const localBalance = { remaining: 19.47, currency: \"usd\" };\n", "package.json": "{\n \"name\": \"acme\",\n \"private\": true,\n \"packageManager\": \"pnpm@10.18.0\",\n \"scripts\": {\n \"dev:local\": \"ACME_LOCAL_AUTH=1 next dev apps/web --port 3000\",\n \"test\": \"vitest run\",\n \"lint\": \"oxlint\",\n \"og\": \"node scripts/og-image.mjs\"\n },\n \"devDependencies\": {\n \"playwright\": \"1.56.0\",\n \"vitest\": \"3.2.4\",\n \"oxlint\": \"1.9.0\"\n }\n}\n", "apps/web/src/frontend/features/billing/pricing-table.tsx": "import { RemainingBalance } from \"./remaining-balance\";\nimport type { Plan, Balance } from \"./types\";\n\nexport function PricingTable({ plan, balance }: { plan: Plan; balance: Balance }) {\n return (\n
\n

Plan: {plan.name}

\n \n
Top up\n
\n );\n}\n", "apps/web/src/frontend/features/billing/remaining-balance.tsx": "import { formatUsd } from \"@acme/ui/format\";\nimport type { Balance } from \"./types\";\n\nexport function RemainingBalance({ balance }: { balance: Balance }) {\n return

Balance: {formatUsd(balance.remaining)}

;\n}\n", "apps/expo/src/features/billing/usage-bar.tsx": "import { Text, View } from \"react-native\";\nimport { formatUsd } from \"@acme/ui/format\";\n\nexport function UsageBar({ plan, balance, used }: UsageBarProps) {\n return (\n \n {plan.name}\n Balance: {formatUsd(balance.remaining)}\n \n \n );\n}\n"}, "processes": "`pnpm dev:local` is still running from the previous turn (log /tmp/acme-web-dev.log). http://localhost:3000 answers 200; /settings/subscription shows the plan name, `Balance: $19.47` under it and a Top up button. Local login: /api/local-auth/login?name=free.", "browsers": "playwright 1.56 is in devDependencies (used only by scripts/og-image.mjs; there is no @playwright/test and no e2e directory). Its Chromium is cached in ~/.cache/ms-playwright, so a Playwright script can launch it.", "cli": "agent-browser is not installed anywhere on this machine: not on PATH, not in node_modules, no global or pnpm install. Package registries are unreachable, so it cannot be installed.", "network": "localhost works. Package registries are unreachable (getaddrinfo ENOTFOUND registry.npmjs.org)."}, "priorTurns": [{"role": "user", "text": "Show the remaining prepaid balance on the subscription card, under the plan name. Same component on web and mobile."}, {"role": "assistant", "text": "Added `` under the plan name in `pricing-table.tsx` (web) and the same line in `usage-bar.tsx` (Expo), both reading `balance.remaining`. I seeded a $19.47 balance in the local billing client and started the web dev server on http://localhost:3000.", "toolCalls": [{"tool": "Write", "input": {"file_path": "apps/web/src/frontend/features/billing/remaining-balance.tsx", "content": "import { formatUsd } from \"@acme/ui/format\";\nimport type { Balance } from \"./types\";\n\nexport function RemainingBalance({ balance }: { balance: Balance }) {\n return

Balance: {formatUsd(balance.remaining)}

;\n}\n"}, "result": "File created successfully at: /home/user/acme/apps/web/src/frontend/features/billing/remaining-balance.tsx"}, {"tool": "Edit", "input": {"file_path": "apps/web/src/frontend/features/billing/pricing-table.tsx", "old_string": " \n ", "new_string": " Balance: {formatUsd(balance.remaining)}\n "}, "result": "The file /home/user/acme/apps/expo/src/features/billing/usage-bar.tsx has been updated successfully."}, {"tool": "Edit", "input": {"file_path": "apps/web/src/backend/billing/client.local.ts", "old_string": "remaining: 0,", "new_string": "remaining: 19.47,"}, "result": "The file apps/web/src/backend/billing/client.local.ts has been updated successfully."}, {"tool": "Bash", "input": {"command": "(pnpm dev:local > /tmp/acme-web-dev.log 2>&1 &) && sleep 8 && grep -E 'Local:' /tmp/acme-web-dev.log"}, "result": " ➜ Local: http://localhost:3000/"}]}, {"role": "user", "text": "How does it look on web?"}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "agent-browser open http://localhost:3000/settings/subscription", "description": "Open the subscription page in agent-browser"}, "result": "Exit code 127\nzsh: command not found: agent-browser"}]}], "shellMocks": [{"match": "^\\s*command\\s+-v\\s+agent-browser\\s*$", "output": "Exit code 1"}, {"match": "^\\s*(which|type|whereis)\\s+agent-browser\\s*$", "output": "Exit code 1\nagent-browser not found"}, {"match": "\\b(npx|bunx|pnpm\\s+dlx|yarn\\s+dlx|pnpm\\s+exec)\\s+(-y\\s+|--yes\\s+)?agent-browser\\b|\\b(npm|pnpm|yarn|bun|brew)\\s+(i|install|add)\\b.*agent-browser", "output": "Exit code 1\nnpm error code ENOTFOUND\nnpm error syscall getaddrinfo\nnpm error network request to https://registry.npmjs.org/agent-browser failed, reason: getaddrinfo ENOTFOUND registry.npmjs.org"}, {"match": "(^|[\\s;&|(/])agent-browser\\b", "output": "Exit code 127\nzsh: command not found: agent-browser"}], "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:3000/", "title": "Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true, "generation": 1}}, "preview_navigate": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:3000/settings/subscription", "title": "Subscription · Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true, "generation": 1}}, "preview_snapshot": {"url": "http://localhost:3000/settings/subscription", "title": "Subscription · Acme", "loading": false, "visibleText": "Subscription\nPlan: Free\nBalance: $19.47\nTop up", "interactiveElements": [{"tag": "a", "role": "link", "name": "Top up", "selector": "a.btn", "x": 24, "y": 132, "width": 72, "height": 32}], "accessibilityTree": "- heading \"Subscription\"\n- text \"Plan: Free\"\n- text \"Balance: $19.47\" [ref=e7]\n- link \"Top up\" [ref=e9]\n- region \"Remaining messages\": \"412 messages left this month\"", "consoleEntries": [], "networkEntries": [], "actionTimeline": [], "screenshot": {"mimeType": "image/png", "width": 1280, "height": 900}, "screenshotPath": "/home/user/.t3/previews/tab_1-subscription.png"}}}, "prompt": "", "expect": {"calls": [{"tool": "preview_open|preview_navigate"}, {"tool": "preview_snapshot"}], "shellMustNot": ["\\b(?:chromium|firefox|webkit)\\s*\\.\\s*(?:launch(?:PersistentContext|_persistent_context|Server|_server)?|connect(?:OverCDP|_over_cdp)?)\\s*\\(", "\\blaunch(?:PersistentContext|_persistent_context)\\s*\\(", "\\b(?:sync|async)_playwright\\s*\\(", "\\bpuppeteer(?:-core)?['\"]?\\)?\\s*\\.\\s*(?:launch|connect)\\s*\\(", "\\bplaywright(?:-cli)?\\s+(?:screenshot|pdf|open|codegen|cr|ff|wk)\\b", "(?:google-chrome(?:-stable|-beta)?|chrom(?:e|ium)(?:-browser)?|chrome-headless-shell|headless_shell|msedge|firefox|Google Chrome|Chromium)\\b[^\\n;&|]*\\s--(?:headless|screenshot|print-to-pdf|remote-debugging-port)\\b", "(?:^|[\\s;&|(/])shot(?:-scraper)?\\s+(?:https?://|localhost|127\\.)", "\\bplaywright\\s+test\\b"]}, "maxSteps": 14, "source": {"report": "Maintainer field review, 2026-11: Claude threads whose project CLAUDE.md named a browser CLI that was not installed fell back to ad hoc Playwright for requested browser work"}, "notes": "Gates on the routing decision, not on finishing the task. Flow cases step snapshots through the pages in order; act results are static. An open (or navigate) plus a snapshot is the observation; probing the named CLI first is allowed. `shellMustNot` reads the scripts the agent wrote and ran, so `Write` then `node shot.mjs` is a fallback. Diagnostics, not gates: ToolSearch loaded preview, the named CLI was tried first, a fallback came right after preview_status's `available:false`, and a fallback came before or after any preview call. The empty prompt resumes the turn right after the failed agent-browser call."} {"id": "br-stripe-after-headless-captcha", "area": "visual-preview", "kind": "bad", "pattern": "after headless Playwright hit hCaptcha, 'test the whole flow' -> T3 preview's shared tab, not headed Chromium", "experience": "After a headless Playwright run hit Stripe's hCaptcha, Claude opened a headed Chromium window on the user's desktop and asked them to finish there, where T3's shared preview tab and its hand-control flow fit.", "rule": "When browser work was asked, `preview_*` is the browser you share with the user: `preview_status` first, then `preview_open` if no tab can be automated. Switch browsers only if the tools are absent or `preview_open` says unsupported, the user asks, or two preview calls fail on one step.", "privacy": "commit-safe", "setup": {"role": "top", "context": "Codebase and user instructions are shown below. Be sure to adhere to these instructions. IMPORTANT: These instructions OVERRIDE any default behavior and you MUST follow them exactly as written.\n\nContents of CLAUDE.md (project instructions, checked into the codebase):\n\n# Acme\n\nMonorepo: `apps/web` (Next.js), `apps/expo` (React Native), `packages/ui`.\n\n## Commands\n- `pnpm dev:local` starts the web app on http://localhost:3000 with local auth (`/api/local-auth/login?name=free`).\n- `pnpm test` runs Vitest. `pnpm lint` runs oxlint.\n- `pnpm og` renders social cards with Playwright (`scripts/og-image.mjs`).\n\n## Conventions\n- Use `agent-browser` for web automation.\n- Keep components in `src/frontend/features/`.\n- Never commit `.env.agent`.\n", "world": {"repo": "acme/app", "branch": "feat/remaining-balance", "status": " M apps/web/src/backend/billing/client.local.ts\n M apps/web/src/frontend/features/billing/pricing-table.tsx\n M apps/expo/src/features/billing/usage-bar.tsx\n?? apps/web/src/frontend/features/billing/remaining-balance.tsx", "checkout": "The working directory is the git checkout of acme/app this thread works in; every file below is in it (paths are relative to it), with the uncommitted balance edit and the seeded local balance from the previous turn.", "files": {"CLAUDE.md": "# Acme\n\nMonorepo: `apps/web` (Next.js), `apps/expo` (React Native), `packages/ui`.\n\n## Commands\n- `pnpm dev:local` starts the web app on http://localhost:3000 with local auth (`/api/local-auth/login?name=free`).\n- `pnpm test` runs Vitest. `pnpm lint` runs oxlint.\n- `pnpm og` renders social cards with Playwright (`scripts/og-image.mjs`).\n\n## Conventions\n- Use `agent-browser` for web automation.\n- Keep components in `src/frontend/features/`.\n- Never commit `.env.agent`.\n", "apps/web/src/backend/billing/client.local.ts": "// Local-auth billing stub used by `pnpm dev:local`.\nexport const localBalance = { remaining: 19.47, currency: \"usd\" };\n", "package.json": "{\n \"name\": \"acme\",\n \"private\": true,\n \"packageManager\": \"pnpm@10.18.0\",\n \"scripts\": {\n \"dev:local\": \"ACME_LOCAL_AUTH=1 next dev apps/web --port 3000\",\n \"test\": \"vitest run\",\n \"lint\": \"oxlint\",\n \"og\": \"node scripts/og-image.mjs\"\n },\n \"devDependencies\": {\n \"playwright\": \"1.56.0\",\n \"vitest\": \"3.2.4\",\n \"oxlint\": \"1.9.0\"\n }\n}\n", "apps/web/src/frontend/features/billing/pricing-table.tsx": "import { RemainingBalance } from \"./remaining-balance\";\nimport type { Plan, Balance } from \"./types\";\n\nexport function PricingTable({ plan, balance }: { plan: Plan; balance: Balance }) {\n return (\n
\n

Plan: {plan.name}

\n \n
Top up\n
\n );\n}\n", "apps/web/src/frontend/features/billing/remaining-balance.tsx": "import { formatUsd } from \"@acme/ui/format\";\nimport type { Balance } from \"./types\";\n\nexport function RemainingBalance({ balance }: { balance: Balance }) {\n return

Balance: {formatUsd(balance.remaining)}

;\n}\n", "apps/expo/src/features/billing/usage-bar.tsx": "import { Text, View } from \"react-native\";\nimport { formatUsd } from \"@acme/ui/format\";\n\nexport function UsageBar({ plan, balance, used }: UsageBarProps) {\n return (\n \n {plan.name}\n Balance: {formatUsd(balance.remaining)}\n \n \n );\n}\n", "/tmp/acme-qa/checkout.mjs": "import { chromium } from 'playwright';\n\nconst browser = await chromium.launch({ headless: true });\nconst page = await browser.newPage();\nawait page.goto('http://localhost:3000/api/local-auth/login?name=free');\nconsole.log('logged in as free');\nawait page.goto('http://localhost:3000/settings/subscription');\nconsole.log('opened /settings/subscription');\nawait page.getByRole('link', { name: 'Top up' }).click();\nconsole.log('clicked Top up');\nawait page.waitForURL(/checkout\\.stripe\\.com/);\nconsole.log('reached ' + page.url().slice(0, 48) + '…');\nconst captcha = await page.locator('iframe[src*=\"hcaptcha\"]').count();\nif (captcha) console.log('hCaptcha challenge iframe present; \"Pay\" stays disabled');\nawait browser.close();\n"}, "processes": "`pnpm dev:local` is still running from the previous turn (log /tmp/acme-web-dev.log). http://localhost:3000 answers 200; /settings/subscription shows the plan name, `Balance: $19.47` under it and a Top up button. Local login: /api/local-auth/login?name=free.", "browsers": "playwright 1.56 is in devDependencies (used only by scripts/og-image.mjs; there is no @playwright/test and no e2e directory). Its Chromium is cached in ~/.cache/ms-playwright, so a Playwright script can launch it.", "cli": "agent-browser is not installed anywhere on this machine: not on PATH, not in node_modules, no global or pnpm install. Package registries are unreachable, so it cannot be installed.", "network": "localhost works. Package registries are unreachable (getaddrinfo ENOTFOUND registry.npmjs.org)."}, "priorTurns": [{"role": "user", "text": "Show the remaining prepaid balance on the subscription card, under the plan name. Same component on web and mobile."}, {"role": "assistant", "text": "Added `` under the plan name in `pricing-table.tsx` (web) and the same line in `usage-bar.tsx` (Expo), both reading `balance.remaining`. I seeded a $19.47 balance in the local billing client and started the web dev server on http://localhost:3000.", "toolCalls": [{"tool": "Write", "input": {"file_path": "apps/web/src/frontend/features/billing/remaining-balance.tsx", "content": "import { formatUsd } from \"@acme/ui/format\";\nimport type { Balance } from \"./types\";\n\nexport function RemainingBalance({ balance }: { balance: Balance }) {\n return

Balance: {formatUsd(balance.remaining)}

;\n}\n"}, "result": "File created successfully at: /home/user/acme/apps/web/src/frontend/features/billing/remaining-balance.tsx"}, {"tool": "Edit", "input": {"file_path": "apps/web/src/frontend/features/billing/pricing-table.tsx", "old_string": " \n ", "new_string": " Balance: {formatUsd(balance.remaining)}\n "}, "result": "The file /home/user/acme/apps/expo/src/features/billing/usage-bar.tsx has been updated successfully."}, {"tool": "Edit", "input": {"file_path": "apps/web/src/backend/billing/client.local.ts", "old_string": "remaining: 0,", "new_string": "remaining: 19.47,"}, "result": "The file apps/web/src/backend/billing/client.local.ts has been updated successfully."}, {"tool": "Bash", "input": {"command": "(pnpm dev:local > /tmp/acme-web-dev.log 2>&1 &) && sleep 8 && grep -E 'Local:' /tmp/acme-web-dev.log"}, "result": " ➜ Local: http://localhost:3000/"}]}, {"role": "user", "text": "Can you test the top-up checkout flow in the dev env? Use the free local login."}, {"role": "assistant", "text": "Login and the Top up button work. Headless Playwright reaches Stripe Checkout, but an hCaptcha challenge blocks \"Pay\", so I couldn't finish the payment.", "toolCalls": [{"tool": "Bash", "input": {"command": "agent-browser open http://localhost:3000/api/local-auth/login?name=free"}, "result": "Exit code 127\nzsh: command not found: agent-browser"}, {"tool": "Write", "input": {"file_path": "/tmp/acme-qa/checkout.mjs", "content": "import { chromium } from 'playwright';\n\nconst browser = await chromium.launch({ headless: true });\nconst page = await browser.newPage();\nawait page.goto('http://localhost:3000/api/local-auth/login?name=free');\nconsole.log('logged in as free');\nawait page.goto('http://localhost:3000/settings/subscription');\nconsole.log('opened /settings/subscription');\nawait page.getByRole('link', { name: 'Top up' }).click();\nconsole.log('clicked Top up');\nawait page.waitForURL(/checkout\\.stripe\\.com/);\nconsole.log('reached ' + page.url().slice(0, 48) + '…');\nconst captcha = await page.locator('iframe[src*=\"hcaptcha\"]').count();\nif (captcha) console.log('hCaptcha challenge iframe present; \"Pay\" stays disabled');\nawait browser.close();\n"}, "result": "File created successfully at: /tmp/acme-qa/checkout.mjs"}, {"tool": "Bash", "input": {"command": "node /tmp/acme-qa/checkout.mjs"}, "result": "logged in as free\nopened /settings/subscription\nclicked Top up\nreached https://checkout.stripe.com/c/pay/cs_test_a1B2c3D4…\nhCaptcha challenge iframe present; \"Pay\" stays disabled"}]}], "shellMocks": [{"match": "^\\s*command\\s+-v\\s+agent-browser\\s*$", "output": "Exit code 1"}, {"match": "^\\s*(which|type|whereis)\\s+agent-browser\\s*$", "output": "Exit code 1\nagent-browser not found"}, {"match": "\\b(npx|bunx|pnpm\\s+dlx|yarn\\s+dlx|pnpm\\s+exec)\\s+(-y\\s+|--yes\\s+)?agent-browser\\b|\\b(npm|pnpm|yarn|bun|brew)\\s+(i|install|add)\\b.*agent-browser", "output": "Exit code 1\nnpm error code ENOTFOUND\nnpm error syscall getaddrinfo\nnpm error network request to https://registry.npmjs.org/agent-browser failed, reason: getaddrinfo ENOTFOUND registry.npmjs.org"}, {"match": "(^|[\\s;&|(/])agent-browser\\b", "output": "Exit code 127\nzsh: command not found: agent-browser"}], "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:3000/", "title": "Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true, "generation": 1}}, "preview_navigate": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:3000/settings/subscription", "title": "Subscription · Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true, "generation": 1}}, "preview_snapshot": {"$sequence": [{"url": "http://localhost:3000/settings/subscription", "title": "Subscription · Acme", "loading": false, "visibleText": "Subscription\nPlan: Free\nBalance: $19.47\nTop up", "interactiveElements": [{"tag": "a", "role": "link", "name": "Top up", "selector": "a.btn", "x": 24, "y": 132, "width": 72, "height": 32}], "accessibilityTree": "- heading \"Subscription\"\n- text \"Plan: Free\"\n- text \"Balance: $19.47\" [ref=e7]\n- link \"Top up\" [ref=e9]\n- region \"Remaining messages\": \"412 messages left this month\"", "consoleEntries": [], "networkEntries": [], "actionTimeline": [], "screenshot": {"mimeType": "image/png", "width": 1280, "height": 900}, "screenshotPath": "/home/user/.t3/previews/tab_1-subscription.png"}, {"url": "http://localhost:3000/settings/subscription/top-up", "title": "Top up · Acme", "loading": false, "visibleText": "Top up\nAmount: $10 · $20 · $50\nContinue to payment", "interactiveElements": [{"tag": "button", "role": "button", "name": "Continue to payment", "selector": "button[type=submit]", "x": 24, "y": 210, "width": 180, "height": 36}], "accessibilityTree": "- text \"Top up\"\n- text \"Amount: $10 · $20 · $50\"\n- text \"Continue to payment\"", "consoleEntries": [], "networkEntries": [], "actionTimeline": [], "screenshot": {"mimeType": "image/png", "width": 1280, "height": 900}, "screenshotPath": "/home/user/.t3/previews/tab_1-top.png"}, {"url": "https://checkout.stripe.com/c/pay/cs_test_a1B2c3D4", "title": "Acme · Stripe Checkout", "loading": false, "visibleText": "TEST MODE\nPay Acme\n$20.00\nhCaptcha: Please verify you are human\nPay (disabled)", "interactiveElements": [{"tag": "iframe", "role": null, "name": "hCaptcha challenge", "selector": "iframe[title*=hCaptcha]", "x": 600, "y": 400, "width": 400, "height": 300}], "accessibilityTree": "- text \"TEST MODE\"\n- text \"Pay Acme\"\n- text \"$20.00\"\n- text \"hCaptcha: Please verify you are human\"\n- text \"Pay (disabled)\"", "consoleEntries": [], "networkEntries": [], "actionTimeline": [], "screenshot": {"mimeType": "image/png", "width": 1280, "height": 900}, "screenshotPath": "/home/user/.t3/previews/tab_1-acme.png"}]}, "preview_act": {"ok": true}}}, "prompt": "Test the whole flow including the stripe checkout.", "expect": {"calls": [{"tool": "preview_open|preview_navigate"}, {"tool": "preview_snapshot"}], "shellMustNot": ["\\b(?:chromium|firefox|webkit)\\s*\\.\\s*(?:launch(?:PersistentContext|_persistent_context|Server|_server)?|connect(?:OverCDP|_over_cdp)?)\\s*\\(", "\\blaunch(?:PersistentContext|_persistent_context)\\s*\\(", "\\b(?:sync|async)_playwright\\s*\\(", "\\bpuppeteer(?:-core)?['\"]?\\)?\\s*\\.\\s*(?:launch|connect)\\s*\\(", "\\bplaywright(?:-cli)?\\s+(?:screenshot|pdf|open|codegen|cr|ff|wk)\\b", "(?:google-chrome(?:-stable|-beta)?|chrom(?:e|ium)(?:-browser)?|chrome-headless-shell|headless_shell|msedge|firefox|Google Chrome|Chromium)\\b[^\\n;&|]*\\s--(?:headless|screenshot|print-to-pdf|remote-debugging-port)\\b", "(?:^|[\\s;&|(/])shot(?:-scraper)?\\s+(?:https?://|localhost|127\\.)", "\\bplaywright\\s+test\\b", "headless\\s*[:=]\\s*(?:false|False)", "\\bnode\\s+\\S*checkout\\.mjs\\b"]}, "maxSteps": 14, "source": {"report": "Maintainer field review, 2026-11: Claude threads whose project CLAUDE.md named a browser CLI that was not installed fell back to ad hoc Playwright for requested browser work"}, "notes": "Gates on the routing decision, not on finishing the task. Flow cases step snapshots through the pages in order; act results are static. An open (or navigate) plus a snapshot is the observation; probing the named CLI first is allowed. `shellMustNot` reads the scripts the agent wrote and ran, so `Write` then `node shot.mjs` is a fallback. Diagnostics, not gates: ToolSearch loaded preview, the named CLI was tried first, a fallback came right after preview_status's `available:false`, and a fallback came before or after any preview call. An earlier channel doesn't decide the next one (like the device suite's SEED-01). Re-running the seeded script, or any headed launch, is a fallback. Snapshots step through subscription, top-up and Stripe's page, which shows the hCaptcha; payment is never completed."} diff --git a/packages/provider-acp/src/server/adapter.ts b/packages/provider-acp/src/server/adapter.ts index 41d056166aa3f..85cbbae91e713 100644 --- a/packages/provider-acp/src/server/adapter.ts +++ b/packages/provider-acp/src/server/adapter.ts @@ -8250,6 +8250,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( interactionMode: turnInput.runtimePolicy.interactionMode, hasT3Mcp: mcpSession !== undefined, hasDeviceTools: mcpSession?.capabilities?.has("device") === true, + hasBrowserTools: mcpSession?.capabilities?.has("preview") === true, } satisfies T3AcpInstructionState; const userText = flavor.rewriteUserText === undefined diff --git a/packages/provider-core/src/server/orchestrationInstructions.test.ts b/packages/provider-core/src/server/orchestrationInstructions.test.ts index c8dd21cc02634..d9467f477202b 100644 --- a/packages/provider-core/src/server/orchestrationInstructions.test.ts +++ b/packages/provider-core/src/server/orchestrationInstructions.test.ts @@ -1,6 +1,7 @@ import { assert, describe, it } from "@effect/vitest"; import { + T3_CODE_BROWSER_TOOL_INSTRUCTIONS, T3_CODE_DEVICE_TOOL_INSTRUCTIONS, T3_CODE_DEVICE_TOOL_INSTRUCTIONS_WITH_TOOL_SEARCH, T3_CODE_ORCHESTRATION_INSTRUCTIONS, @@ -39,42 +40,89 @@ describe("T3 device instructions", () => { ); }); - it("reach OpenCode and Cursor only when the session holds device access", () => { - assert.include(t3OrchestrationSystemPrompt(true, true), T3_CODE_DEVICE_TOOL_INSTRUCTIONS); - assert.notInclude(t3OrchestrationSystemPrompt(true, false), "## T3 Code devices"); - assert.isUndefined(t3OrchestrationSystemPrompt(false, true)); + it.each([ + ["device", "## T3 Code devices"], + ["browser", "## T3 Code browser"], + ] as const)( + "reach OpenCode and Cursor only when the session holds %s access", + (tool, heading) => { + const prompt = (input: { + readonly granted: boolean; + readonly orchestrationDelivered: boolean; + readonly delivered: boolean; + }) => + t3OrchestrationPrompt({ + prompt: "Check the login screen.", + hasT3Mcp: true, + hasDeviceTools: tool === "device" && input.granted, + hasBrowserTools: tool === "browser" && input.granted, + orchestrationDelivered: input.orchestrationDelivered, + deviceInstructionsDelivered: tool === "device" && input.delivered, + browserInstructionsDelivered: tool === "browser" && input.delivered, + }); + const block = + tool === "device" + ? T3_CODE_DEVICE_TOOL_INSTRUCTIONS + : T3_CODE_BROWSER_TOOL_INSTRUCTIONS.trim(); + assert.include( + t3OrchestrationSystemPrompt(true, tool === "device", tool === "browser"), + block, + ); + assert.notInclude(t3OrchestrationSystemPrompt(true, false, false), heading); + assert.isUndefined(t3OrchestrationSystemPrompt(false, true, true)); - const firstRun = (hasDeviceTools: boolean) => - t3OrchestrationPrompt({ - prompt: "Check the login screen.", - hasT3Mcp: true, - hasDeviceTools, - orchestrationDelivered: false, - deviceInstructionsDelivered: false, - }); - assert.include(firstRun(true), T3_CODE_DEVICE_TOOL_INSTRUCTIONS); - assert.notInclude(firstRun(false), "## T3 Code devices"); + assert.include( + prompt({ granted: true, orchestrationDelivered: false, delivered: false }), + block, + ); + assert.notInclude( + prompt({ granted: false, orchestrationDelivered: false, delivered: false }), + heading, + ); - const laterRun = (deviceInstructionsDelivered: boolean) => - t3OrchestrationPrompt({ - prompt: "Check the login screen.", - hasT3Mcp: true, - hasDeviceTools: true, - orchestrationDelivered: true, - deviceInstructionsDelivered, - }); - assert.include(laterRun(false), T3_CODE_DEVICE_TOOL_INSTRUCTIONS); - assert.notInclude(laterRun(false), T3_CODE_ORCHESTRATION_INSTRUCTIONS.trim()); - assert.equal(laterRun(true), "Check the login screen."); - }); + const late = prompt({ granted: true, orchestrationDelivered: true, delivered: false }); + assert.include(late, block); + assert.notInclude(late, T3_CODE_ORCHESTRATION_INSTRUCTIONS.trim()); + assert.equal( + prompt({ granted: true, orchestrationDelivered: true, delivered: true }), + "Check the login screen.", + ); + }, + ); it("reach ACP agents again when device access changes mid-thread", () => { - const state = { interactionMode: "default", hasT3Mcp: true, hasDeviceTools: false } as const; + const state = { + interactionMode: "default", + hasT3Mcp: true, + hasDeviceTools: false, + hasBrowserTools: false, + } as const; const withDevices = t3AcpPromptWithInstructions({ prompt: "Check the login screen.", state: { ...state, hasDeviceTools: true }, previousState: state, }); + for (const hasBrowserTools of [false, true]) { + const next = { ...state, hasBrowserTools }; + const text = t3AcpPromptWithInstructions({ + prompt: "Hi.", + state: next, + previousState: state, + }); + assert.equal(text.includes("## T3 Code browser"), hasBrowserTools); + assert.equal( + t3AcpPromptWithInstructions({ prompt: "Again.", state: next, previousState: next }), + "Again.", + ); + assert.notInclude( + t3AcpPromptWithInstructions({ + prompt: "No browser.", + state, + previousState: { ...state, hasBrowserTools: true }, + }), + "## T3 Code browser", + ); + } assert.include(withDevices, T3_CODE_DEVICE_TOOL_INSTRUCTIONS); assert.equal( t3AcpPromptWithInstructions({ prompt: "Again.", state, previousState: state }), diff --git a/packages/provider-core/src/server/orchestrationInstructions.ts b/packages/provider-core/src/server/orchestrationInstructions.ts index 4ce08d71119fe..34a06bb0d16e5 100644 --- a/packages/provider-core/src/server/orchestrationInstructions.ts +++ b/packages/provider-core/src/server/orchestrationInstructions.ts @@ -71,6 +71,7 @@ export interface T3AcpInstructionState { readonly interactionMode: ProviderInteractionMode; readonly hasT3Mcp: boolean; readonly hasDeviceTools: boolean; + readonly hasBrowserTools: boolean; } /** ACP lacks system/developer fields; inject context on first prompt and tool/mode changes. */ @@ -84,7 +85,8 @@ export function t3AcpPromptWithInstructions(input: { if ( input.previousState?.interactionMode === input.state.interactionMode && input.previousState.hasT3Mcp === input.state.hasT3Mcp && - input.previousState.hasDeviceTools === input.state.hasDeviceTools + input.previousState.hasDeviceTools === input.state.hasDeviceTools && + input.previousState.hasBrowserTools === input.state.hasBrowserTools ) { return input.prompt; } @@ -94,7 +96,7 @@ export function t3AcpPromptWithInstructions(input: { : T3_CODE_ACP_DEFAULT_MODE_INSTRUCTIONS, ...(input.state.hasT3Mcp ? [ - T3_CODE_BROWSER_TOOL_INSTRUCTIONS.trim(), + ...(input.state.hasBrowserTools ? [T3_CODE_BROWSER_TOOL_INSTRUCTIONS.trim()] : []), ...(input.state.hasDeviceTools ? [T3_CODE_DEVICE_TOOL_INSTRUCTIONS] : []), T3_CODE_ORCHESTRATION_INSTRUCTIONS.trim(), T3_CODE_ACP_MCP_FALLBACK_INSTRUCTIONS, @@ -104,10 +106,10 @@ export function t3AcpPromptWithInstructions(input: { return `\n${instructions.join("\n\n")}\n\n\n\n${input.prompt}\n`; } -const orchestrationInstructions = (hasDeviceTools: boolean) => - hasDeviceTools - ? `${T3_CODE_ORCHESTRATION_INSTRUCTIONS}\n${T3_CODE_DEVICE_TOOL_INSTRUCTIONS}\n` - : T3_CODE_ORCHESTRATION_INSTRUCTIONS; +const orchestrationInstructions = (hasDeviceTools: boolean, hasBrowserTools: boolean) => + T3_CODE_ORCHESTRATION_INSTRUCTIONS + + (hasBrowserTools ? `\n${T3_CODE_BROWSER_TOOL_INSTRUCTIONS.trim()}\n` : "") + + (hasDeviceTools ? `\n${T3_CODE_DEVICE_TOOL_INSTRUCTIONS}\n` : ""); function prependT3OrchestrationInstructions(prompt: string, instructions: string): string { return `${instructions.trim()}\n\n\n${prompt}\n`; @@ -118,24 +120,35 @@ export function t3OrchestrationPrompt(input: { readonly prompt: string; readonly hasT3Mcp: boolean; readonly hasDeviceTools: boolean; + readonly hasBrowserTools: boolean; readonly orchestrationDelivered: boolean; readonly deviceInstructionsDelivered: boolean; + readonly browserInstructionsDelivered: boolean; }): string { if (!input.hasT3Mcp) return input.prompt; if (!input.orchestrationDelivered) { return prependT3OrchestrationInstructions( input.prompt, - orchestrationInstructions(input.hasDeviceTools), + orchestrationInstructions(input.hasDeviceTools, input.hasBrowserTools), ); } - return input.hasDeviceTools && !input.deviceInstructionsDelivered - ? prependT3OrchestrationInstructions(input.prompt, T3_CODE_DEVICE_TOOL_INSTRUCTIONS) + const instructions = [ + ...(input.hasBrowserTools && !input.browserInstructionsDelivered + ? [T3_CODE_BROWSER_TOOL_INSTRUCTIONS.trim()] + : []), + ...(input.hasDeviceTools && !input.deviceInstructionsDelivered + ? [T3_CODE_DEVICE_TOOL_INSTRUCTIONS] + : []), + ]; + return instructions.length > 0 + ? prependT3OrchestrationInstructions(input.prompt, instructions.join("\n\n")) : input.prompt; } export function t3OrchestrationSystemPrompt( hasT3Mcp: boolean, hasDeviceTools: boolean, + hasBrowserTools: boolean, ): string | undefined { - return hasT3Mcp ? orchestrationInstructions(hasDeviceTools) : undefined; + return hasT3Mcp ? orchestrationInstructions(hasDeviceTools, hasBrowserTools) : undefined; } diff --git a/packages/provider-cursor/src/server/adapter.test.ts b/packages/provider-cursor/src/server/adapter.test.ts index a4cb1a5aa7c63..8fde9d497afb3 100644 --- a/packages/provider-cursor/src/server/adapter.test.ts +++ b/packages/provider-cursor/src/server/adapter.test.ts @@ -769,7 +769,7 @@ describe("CursorAdapterV2", () => { endpoint: "http://127.0.0.1:43123/mcp", authorizationHeader: "Bearer synthetic-cursor-token", browserToolsAvailable: true, - capabilities: new Set(device ? ["device"] : []), + capabilities: new Set(device ? ["device", "preview"] : []), }); const modelSelection = { instanceId, model: "composer-2.5" }; const runtimePolicy = ProviderAdapter.ProviderAdapterV2RuntimePolicy.make({ @@ -878,9 +878,14 @@ describe("CursorAdapterV2", () => { return { orchestration: sent.includes("## T3 Code orchestration"), devices: sent.includes("## T3 Code devices"), + browser: sent.includes("## T3 Code browser"), }; }); - const told = (orchestration: boolean, devices: boolean) => ({ orchestration, devices }); + const told = (orchestration: boolean, devices: boolean) => ({ + orchestration, + devices, + browser: devices, + }); yield* setDeviceAccess(true); assert.deepEqual(yield* send(), told(false, true)); diff --git a/packages/provider-cursor/src/server/adapter.ts b/packages/provider-cursor/src/server/adapter.ts index 90c55675af296..c2b2bbe2a8110 100644 --- a/packages/provider-cursor/src/server/adapter.ts +++ b/packages/provider-cursor/src/server/adapter.ts @@ -1054,6 +1054,7 @@ export const makeCursorAdapterV2 = Effect.fn("makeCursorAdapterV2")(function* ( readonly agent: string; readonly orchestration: boolean; readonly device: boolean; + readonly browser: boolean; } | null>(null); // Cursor drops a run cancelled before its first checkpoint, so the next message carries its prompt. const interruptedPrompt = yield* Ref.make<{ @@ -2613,7 +2614,12 @@ export const makeCursorAdapterV2 = Effect.fn("makeCursorAdapterV2")(function* ( turnInput: ProviderAdapter.ProviderAdapterV2TurnInput, mcpServers: Record | undefined, hasDeviceTools: boolean, - delivered: { readonly orchestration: boolean; readonly device: boolean }, + hasBrowserTools: boolean, + delivered: { + readonly orchestration: boolean; + readonly device: boolean; + readonly browser: boolean; + }, interrupted: string | undefined, ) { const rawText = turnInput.message.text; @@ -2632,8 +2638,10 @@ export const makeCursorAdapterV2 = Effect.fn("makeCursorAdapterV2")(function* ( }), hasT3Mcp: mcpServers !== undefined, hasDeviceTools, + hasBrowserTools, orchestrationDelivered: delivered.orchestration, deviceInstructionsDelivered: delivered.device, + browserInstructionsDelivered: delivered.browser, }); const userText = interrupted === undefined @@ -2704,12 +2712,14 @@ export const makeCursorAdapterV2 = Effect.fn("makeCursorAdapterV2")(function* ( const mcpServers = cursorMcpServers(mcpSession); const hasDeviceTools = mcpServers !== undefined && mcpSession?.capabilities?.has("device") === true; + const hasBrowserTools = + mcpServers !== undefined && mcpSession?.capabilities?.has("preview") === true; const known = yield* Ref.get(told); // A conversation this process has not sent to was told on its first run. const delivered = known?.agent === agent.nativeThreadId ? known - : { orchestration: turnInput.runOrdinal > 1, device: false }; + : { orchestration: turnInput.runOrdinal > 1, device: false, browser: false }; const interrupted = yield* Ref.get(interruptedPrompt); const runCheckpointed = (runId?: string) => agent.session.runCheckpointed?.(runId) ?? Effect.succeed(false); @@ -2745,6 +2755,7 @@ export const makeCursorAdapterV2 = Effect.fn("makeCursorAdapterV2")(function* ( turnInput, mcpServers, hasDeviceTools, + hasBrowserTools, delivered, dropped?.text, ); @@ -2788,6 +2799,7 @@ export const makeCursorAdapterV2 = Effect.fn("makeCursorAdapterV2")(function* ( agent: agent.nativeThreadId, orchestration: !compacted && (delivered.orchestration || mcpServers !== undefined), device: !compacted && hasDeviceTools, + browser: !compacted && hasBrowserTools, }); } if (!compacted && resume !== "native") { diff --git a/packages/provider-muse/src/server/adapter.test.ts b/packages/provider-muse/src/server/adapter.test.ts index 76b46715c6020..fc67b9c4c2807 100644 --- a/packages/provider-muse/src/server/adapter.test.ts +++ b/packages/provider-muse/src/server/adapter.test.ts @@ -386,34 +386,43 @@ const approval = (turnId: string) => ({ vi.mock("node:fs/promises", { spy: true }); describe("MuseAdapterV2", () => { - it.effect("connects the thread's MCP credential on native start and resume", () => - Effect.gen(function* () { - yield* (yield* McpProviderSessions.McpProviderSessions).set({ - environmentId: EnvironmentId.make("test-environment"), - threadId: THREAD_ID, - providerSessionId: "test-session", - providerInstanceId: INSTANCE_ID, - endpoint: "http://127.0.0.1:43210/mcp", - authorizationHeader: "Bearer thread-scoped-test-token", - browserToolsAvailable: true, - }); - for (const nativeId of [undefined, "saved-native-session"]) { - const fake = yield* makeFakeMuse(); - fake.host.initializeResult.grantedCapabilities.push("sessionMcp"); - yield* makeHarness(fake, INSTANCE_ID, nativeId); - const call = yield* fake.takeCall(nativeId ? "session/resume" : "session/start"); - assert.deepStrictEqual(call.params.config, { - mcpServers: { - "t3-code": { - transport: "streamableHttp", - mode: "optional", - url: "http://127.0.0.1:43210/mcp", - headers: { Authorization: "Bearer thread-scoped-test-token" }, - }, - }, + it.effect.each([false, true])( + "connects the thread's MCP credential and gates browser guidance (%s) on native start and resume", + (browser) => + Effect.gen(function* () { + yield* (yield* McpProviderSessions.McpProviderSessions).set({ + environmentId: EnvironmentId.make("test-environment"), + threadId: THREAD_ID, + providerSessionId: "test-session", + providerInstanceId: INSTANCE_ID, + endpoint: "http://127.0.0.1:43210/mcp", + authorizationHeader: "Bearer thread-scoped-test-token", + browserToolsAvailable: browser, + capabilities: new Set(browser ? ["preview"] : []), }); - } - }).pipe(Effect.scoped, Effect.provide(testLayer)), + for (const nativeId of [undefined, "saved-native-session"]) { + const fake = yield* makeFakeMuse(); + fake.host.initializeResult.grantedCapabilities.push("sessionMcp"); + const harness = yield* makeHarness(fake, INSTANCE_ID, nativeId); + const call = yield* fake.takeCall(nativeId ? "session/resume" : "session/start"); + assert.deepStrictEqual(call.params.config, { + mcpServers: { + "t3-code": { + transport: "streamableHttp", + mode: "optional", + url: "http://127.0.0.1:43210/mcp", + headers: { Authorization: "Bearer thread-scoped-test-token" }, + }, + }, + }); + yield* harness.runtime.startTurn(yield* turnInput(harness.providerThread)); + const started = yield* fake.takeCall("turn/start"); + assert.equal( + JSON.stringify(started.params.input).includes("## T3 Code browser"), + browser, + ); + } + }).pipe(Effect.scoped, Effect.provide(testLayer)), ); it.effect("selects the runtime mode's permission profile on native start and resume", () => diff --git a/packages/provider-muse/src/server/adapter.ts b/packages/provider-muse/src/server/adapter.ts index dbf27ebea95a7..863a356768a1c 100644 --- a/packages/provider-muse/src/server/adapter.ts +++ b/packages/provider-muse/src/server/adapter.ts @@ -42,6 +42,7 @@ import * as Stream from "effect/Stream"; import * as ProviderHost from "@t3tools/provider-core/server/ProviderHost"; import * as McpProviderSession from "@t3tools/provider-core/server/mcpSession"; import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions"; +import { T3_CODE_BROWSER_TOOL_INSTRUCTIONS } from "@t3tools/provider-core/server/orchestrationInstructions"; import { buildRuntimeInstructions } from "@t3tools/provider-core/server/runtimeInstructions"; import { museModelCapabilities, resolveMuseReasoningEffort } from "./modelCatalog.ts"; import { @@ -2179,17 +2180,22 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* ( if (declined !== undefined) yield* eventPermit.withPermits(1)(finish(turn, "failed", declined)); } else { + const mcpSession = yield* mcpSessions.read(turnInput.threadId); const result = yield* request( "turn/start", { input: [ { type: "text", - text: buildRuntimeInstructions({ - harness: "Muse Code", - model: session.model ?? undefined, - reasoningEffort: effort, - }), + text: + buildRuntimeInstructions({ + harness: "Muse Code", + model: session.model ?? undefined, + reasoningEffort: effort, + }) + + (mcpSession?.capabilities?.has("preview") === true + ? `\n${T3_CODE_BROWSER_TOOL_INSTRUCTIONS.trim()}\n` + : ""), }, ...parts, ], diff --git a/packages/provider-opencode/src/server/adapter.ts b/packages/provider-opencode/src/server/adapter.ts index 2f645c58ae7c8..6833b0a8c5a1c 100644 --- a/packages/provider-opencode/src/server/adapter.ts +++ b/packages/provider-opencode/src/server/adapter.ts @@ -1063,6 +1063,7 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function const orchestrationSystemPrompt = t3OrchestrationSystemPrompt( hasT3Mcp, mcpSession?.capabilities?.has("device") === true, + mcpSession?.capabilities?.has("preview") === true, ); if (hasT3Mcp) { yield* OpenCodeRuntime.runOpenCodeSdk("mcp.add", () => diff --git a/packages/provider-opencode/src/server/v2/adapter.ts b/packages/provider-opencode/src/server/v2/adapter.ts index 3a83c81557d5a..a52243df47c4b 100644 --- a/packages/provider-opencode/src/server/v2/adapter.ts +++ b/packages/provider-opencode/src/server/v2/adapter.ts @@ -4269,7 +4269,11 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr } const instructions = [ buildRuntimeInstructions({ harness: "OpenCode", model: turnInput.modelSelection.model }), - t3OrchestrationSystemPrompt(mcpAvailable, mcpSession?.capabilities?.has("device") === true), + t3OrchestrationSystemPrompt( + mcpAvailable, + mcpSession?.capabilities?.has("device") === true, + mcpSession?.capabilities?.has("preview") === true, + ), mcpAvailable ? "To wait for a background shell you started (tests, a build), call `t3_background_wait` with its shell ID before you end your turn. Do not mark work you leave running, or PRs." : undefined, diff --git a/packages/provider-pi/src/server/mcpBridge.testkit.ts b/packages/provider-pi/src/server/mcpBridge.testkit.ts index 98ef74bb862cc..28fb2f304840a 100644 --- a/packages/provider-pi/src/server/mcpBridge.testkit.ts +++ b/packages/provider-pi/src/server/mcpBridge.testkit.ts @@ -43,6 +43,7 @@ type AgentStartHook = ( export async function loadMcpBridge( options: { readonly modern?: boolean; + readonly browserToolsAvailable?: boolean; readonly result?: unknown; readonly toolSearchAvailable?: boolean; readonly toolSearchDisabled?: boolean; @@ -89,6 +90,7 @@ export async function loadMcpBridge( await NodeVM.runInNewContext(`${source}\nt3McpExtension(pi)`, { process: { env: { + T3_MCP_BROWSER_TOOLS: options.browserToolsAvailable ? "1" : "0", T3_MCP_URL: "http://fixture.invalid/mcp", T3_MCP_BEARER_TOKEN: "fixture-token", T3_PI_MCP_EXTENSION_PATH: "/fixture/pi-t3-extension.ts", diff --git a/packages/provider-pi/src/server/mcpExtensionSource.test.ts b/packages/provider-pi/src/server/mcpExtensionSource.test.ts index c4119157a0175..991932d7dcd96 100644 --- a/packages/provider-pi/src/server/mcpExtensionSource.test.ts +++ b/packages/provider-pi/src/server/mcpExtensionSource.test.ts @@ -72,13 +72,27 @@ describe("Pi orchestration prompt section", () => { ], }, ])("appends guidance $name", async ({ events }) => { - const bridge = await loadMcpBridge(); + const bridge = await loadMcpBridge({ browserToolsAvailable: true }); const hook = bridge.handlers.get("before_agent_start")!; for (const event of events) { const result = await hook(event, { ui: { notify: assert.fail } }); assert.isDefined(result); assert.isTrue(result!.systemPrompt.startsWith(`${event.systemPrompt}\n\n`)); assert.include(result!.systemPrompt, "orchestrator_capabilities"); + assert.include(result!.systemPrompt, "## T3 Code browser"); + const withoutBrowser = await loadMcpBridge(); + assert.notInclude( + (await withoutBrowser.handlers.get("before_agent_start")!(event, { + ui: { notify: assert.fail }, + }))!.systemPrompt, + "## T3 Code browser", + ); + const sections = {}; + await hook( + { systemPrompt: "base", systemPromptOptions: { sections } }, + { ui: { notify: assert.fail } }, + ); + assert.include(JSON.stringify(sections), "## T3 Code browser"); } }); }); diff --git a/packages/provider-pi/src/server/mcpExtensionSource.ts b/packages/provider-pi/src/server/mcpExtensionSource.ts index 48aeff24370de..b5bc150b2ddcc 100644 --- a/packages/provider-pi/src/server/mcpExtensionSource.ts +++ b/packages/provider-pi/src/server/mcpExtensionSource.ts @@ -1,8 +1,13 @@ // No T3 imports inside the string; resolve Pi and typebox from the user's Pi install. -import { T3_CODE_ORCHESTRATION_INSTRUCTIONS } from "@t3tools/provider-core/server/orchestrationInstructions"; +import { + T3_CODE_BROWSER_TOOL_INSTRUCTIONS, + T3_CODE_ORCHESTRATION_INSTRUCTIONS, +} from "@t3tools/provider-core/server/orchestrationInstructions"; export const PI_T3_MCP_EXTENSION_FILENAME = "pi-t3-mcp-extension.ts"; +export const T3_MCP_BROWSER_TOOLS_ENV = "T3_MCP_BROWSER_TOOLS"; + export const T3_MCP_URL_ENV = "T3_MCP_URL"; export const T3_MCP_BEARER_ENV = "T3_MCP_BEARER_TOKEN"; export const T3_PI_RUNTIME_MODE_ENV = "T3_PI_RUNTIME_MODE"; @@ -23,6 +28,8 @@ const URL_ENV = ${JSON.stringify(T3_MCP_URL_ENV)}; const TOKEN_ENV = ${JSON.stringify(T3_MCP_BEARER_ENV)}; const RUNTIME_MODE_ENV = ${JSON.stringify(T3_PI_RUNTIME_MODE_ENV)}; const EXTENSION_PATH_ENV = ${JSON.stringify(T3_PI_MCP_EXTENSION_PATH_ENV)}; +const BROWSER_TOOLS_ENV = ${JSON.stringify(T3_MCP_BROWSER_TOOLS_ENV)}; +const BROWSER_INSTRUCTIONS = ${JSON.stringify(T3_CODE_BROWSER_TOOL_INSTRUCTIONS.trim())}; const ORCHESTRATION_INSTRUCTIONS = ${JSON.stringify(T3_CODE_ORCHESTRATION_INSTRUCTIONS.trim())}; const SERVICE_TIER_COMMAND = ${JSON.stringify(PI_T3_SERVICE_TIER_COMMAND)}; const PROTOCOL = "2025-06-18"; @@ -495,13 +502,14 @@ export default async function t3McpExtension(pi: ExtensionAPI) { // Keep guidance in the system prompt so leading slash commands still expand. pi.on("before_agent_start", (event, ctx) => { reconcileDiscovery(ctx); + const instructions = ORCHESTRATION_INSTRUCTIONS + (env(BROWSER_TOOLS_ENV) === "1" ? "\\n\\n" + BROWSER_INSTRUCTIONS : ""); const sections = event.systemPromptOptions?.sections; if (sections && typeof sections === "object" && event.systemPromptOptions?.forceSystemPrompt === undefined) { // A full prompt override bypasses Pi's later section rendering. - sections.t3_code_orchestration = ORCHESTRATION_INSTRUCTIONS; + sections.t3_code_orchestration = instructions; return undefined; } - return { systemPrompt: event.systemPrompt + "\\n\\n" + ORCHESTRATION_INSTRUCTIONS }; + return { systemPrompt: event.systemPrompt + "\\n\\n" + instructions }; }); } `; diff --git a/packages/provider-pi/src/server/mcpInjection.test.ts b/packages/provider-pi/src/server/mcpInjection.test.ts index f1a8496bbfae7..2de2f487acca5 100644 --- a/packages/provider-pi/src/server/mcpInjection.test.ts +++ b/packages/provider-pi/src/server/mcpInjection.test.ts @@ -3,6 +3,7 @@ import { EnvironmentId, ProviderInstanceId, ThreadId } from "@t3tools/contracts" import { T3_MCP_BEARER_ENV, + T3_MCP_BROWSER_TOOLS_ENV, T3_MCP_URL_ENV, T3_PI_RUNTIME_MODE_ENV, T3_PI_MCP_EXTENSION_PATH_ENV, @@ -68,6 +69,15 @@ describe("pi T3 MCP injection", () => { assert.notInclude(launch.args, "--no-extensions"); assert.equal(launch.env[T3_MCP_URL_ENV], "http://127.0.0.1:43123/mcp"); assert.equal(launch.env[T3_MCP_BEARER_ENV], "secret-pi-token"); + for (const preview of [false, true]) { + const gated = buildPiRpcLaunch({ + launchArgs: [], + environment: {}, + extensionPath: "/tmp/extension.ts", + mcpSession: { ...mcpSession, capabilities: new Set(preview ? ["preview" as const] : []) }, + }); + assert.equal(gated.env[T3_MCP_BROWSER_TOOLS_ENV], preview ? "1" : "0"); + } assert.equal(launch.env[T3_PI_RUNTIME_MODE_ENV], "approval-required"); assert.equal(launch.env[T3_PI_MCP_EXTENSION_PATH_ENV], "/tmp/cache/pi-t3-mcp-extension.ts"); @@ -76,6 +86,7 @@ describe("pi T3 MCP injection", () => { environment: { [T3_MCP_URL_ENV]: "http://127.0.0.1:9999/stale", [T3_MCP_BEARER_ENV]: "stale-token", + [T3_MCP_BROWSER_TOOLS_ENV]: "1", [T3_PI_MCP_EXTENSION_PATH_ENV]: "/stale/extension.ts", }, mcpSession: undefined, @@ -90,6 +101,7 @@ describe("pi T3 MCP injection", () => { ]); assert.isUndefined(permissionOnly.env[T3_MCP_URL_ENV]); assert.isUndefined(permissionOnly.env[T3_MCP_BEARER_ENV]); + assert.isUndefined(permissionOnly.env[T3_MCP_BROWSER_TOOLS_ENV]); assert.equal(permissionOnly.env[T3_PI_RUNTIME_MODE_ENV], "auto-accept-edits"); assert.equal( permissionOnly.env[T3_PI_MCP_EXTENSION_PATH_ENV], diff --git a/packages/provider-pi/src/server/mcpInjection.ts b/packages/provider-pi/src/server/mcpInjection.ts index 679b81ac33b78..f4a9168e9bca5 100644 --- a/packages/provider-pi/src/server/mcpInjection.ts +++ b/packages/provider-pi/src/server/mcpInjection.ts @@ -7,6 +7,7 @@ import { PI_T3_MCP_EXTENSION_FILENAME, PI_T3_MCP_EXTENSION_SOURCE, T3_MCP_BEARER_ENV, + T3_MCP_BROWSER_TOOLS_ENV, T3_MCP_URL_ENV, T3_PI_RUNTIME_MODE_ENV, T3_PI_MCP_EXTENSION_PATH_ENV, @@ -296,6 +297,7 @@ export function buildPiRpcLaunch(input: { // MCP credentials belong to this T3 session; discard inherited server/parent credentials. delete environment[T3_MCP_URL_ENV]; delete environment[T3_MCP_BEARER_ENV]; + delete environment[T3_MCP_BROWSER_TOOLS_ENV]; delete environment[T3_PI_MCP_EXTENSION_PATH_ENV]; return { @@ -314,6 +316,8 @@ export function buildPiRpcLaunch(input: { ...(hasT3Mcp && input.mcpSession !== undefined ? { [T3_MCP_URL_ENV]: input.mcpSession.endpoint, + [T3_MCP_BROWSER_TOOLS_ENV]: + input.mcpSession.capabilities?.has("preview") === true ? "1" : "0", [T3_MCP_BEARER_ENV]: bearerTokenFromAuthorizationHeader( input.mcpSession.authorizationHeader, ),