From 6ac2c149bf9d30d818fce02f42405f4bef8570d6 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 21 Aug 2026 12:08:55 +0000 Subject: [PATCH 1/7] Adopt Anthropic GA computer and browser tools --- docs/architecture.md | 11 +- packages/browser-loop/CHANGELOG.md | 6 + packages/browser-loop/README.md | 27 +- .../browser-loop/docs/supported-models.md | 9 +- .../examples/anthropic-native-smoke.ts | 4 +- .../browser-loop/examples/shared/tools.ts | 2 +- .../browser-loop/src/core/actions/browser.ts | 70 +++- .../browser-loop/src/core/anthropic-native.ts | 32 +- packages/browser-loop/src/core/resources.ts | 16 +- .../browser-loop/src/core/tool-catalog.ts | 50 +-- packages/browser-loop/src/core/tools.ts | 131 ++------ .../src/core/translator/browser.ts | 117 ++++++- .../browser-loop/src/core/translator/types.ts | 13 + packages/browser-loop/src/index.ts | 2 + .../browser-loop/src/pi-extension/index.ts | 4 +- .../src/pi-extension/selection.ts | 4 +- packages/browser-loop/src/pi/providers.ts | 8 +- .../providers/anthropic/browser-fallback.ts | 223 ------------- .../pi/providers/anthropic/capabilities.ts | 11 +- .../src/pi/providers/anthropic/toolsets.ts | 305 ++++++++++++++++++ .../test/anthropic-browser-fallback.test.ts | 125 ------- .../test/anthropic-native.integration.test.ts | 6 +- .../test/anthropic-toolsets.test.ts | 139 ++++++++ packages/browser-loop/test/e2e.live.test.ts | 2 +- packages/browser-loop/test/extension.test.ts | 6 +- packages/browser-loop/test/models.test.ts | 2 + packages/browser-loop/test/selection.test.ts | 24 +- .../browser-loop/test/tool-catalog.test.ts | 35 +- .../test/translator-browser.test.ts | 22 +- 29 files changed, 807 insertions(+), 599 deletions(-) delete mode 100644 packages/browser-loop/src/pi/providers/anthropic/browser-fallback.ts create mode 100644 packages/browser-loop/src/pi/providers/anthropic/toolsets.ts delete mode 100644 packages/browser-loop/test/anthropic-browser-fallback.test.ts create mode 100644 packages/browser-loop/test/anthropic-toolsets.test.ts diff --git a/docs/architecture.md b/docs/architecture.md index 5edc882f..8b25842c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -178,14 +178,15 @@ Catalog compilation composes provider behavior rather than replacing the whole catalog: - Ordinary function tools stay ordinary. -- Anthropic native browser/computer declarations replace only their own - placeholders and merge required beta headers with caller headers. +- Anthropic's GA browser/computer client-toolset declarations replace only + their own placeholders; their member calls and results retain `toolset_name` + through pi's function-tool transcript without beta headers. - OpenAI streams through pi's builtin Responses transport and its automatic prompt caching by default; a Browser Loop-owned adapter handles OpenAI's native computer tool and tool-search namespace round-trips. -- Anthropic's native browser tool falls back to an equivalent function-tool - declaration when the active credential cannot access `browser_20260701`; - the selected tool identity, name, schema, and executor remain unchanged. +- Anthropic's `computer_toolset_20260801` and `browser_toolset_20260801` may be + selected together; calls remain separated by their computer and viewport + coordinate frames. - Google's current predefined browser toolset serializes one `computer_use` declaration plus exact exclusions through the Browser Loop-owned Interactions API adapter. Excluded calls fail with a named catalog error instead of reaching diff --git a/packages/browser-loop/CHANGELOG.md b/packages/browser-loop/CHANGELOG.md index 2ca3eb9b..4e2ed668 100644 --- a/packages/browser-loop/CHANGELOG.md +++ b/packages/browser-loop/CHANGELOG.md @@ -2,6 +2,12 @@ ## Unreleased +- Replace Anthropic's early-access `computer_20260701` and `browser_20260701` + integrations with the generally available `computer_toolset_20260801` and + `browser_toolset_20260801` client toolsets. Requests no longer send beta + headers or use the browser access fallback; member calls/results preserve + `toolset_name`, sequential batches stop at the first failure, and both + toolsets may be selected together. - Rename the product and unpublished package to Browser Loop: the repository is prepared for `kernel/browser-loop`, the package is `@onkernel/browser-loop`, and release tags use `browser-loop/v*`. The public `loop` namespace, `Loop*` diff --git a/packages/browser-loop/README.md b/packages/browser-loop/README.md index d063b39b..7c483bd5 100644 --- a/packages/browser-loop/README.md +++ b/packages/browser-loop/README.md @@ -318,8 +318,8 @@ function tools. ```ts const tools = [ loop.providers.anthropic.tools.computer({ - version: "20260701", - enableZoom: true, + version: "20260801", + zoom: true, }), loop.tools.browser.snapshot(), ]; @@ -330,9 +330,9 @@ Available groups: ```ts loop.providers.openai.tools.computer(); -loop.providers.anthropic.source; -loop.providers.anthropic.tools.computer({ version: "20260701" }); -loop.providers.anthropic.tools.browser({ version: "20260701" }); +loop.providers.anthropic.sources; +loop.providers.anthropic.tools.computer({ version: "20260801" }); +loop.providers.anthropic.tools.browser({ version: "20260801" }); loop.providers.google.source; loop.providers.google.toolsets.browser({ exclude: ["right_click"] }); @@ -357,11 +357,12 @@ than on the wire. [Models and native surfaces](docs/supported-models.md) lists which models carry which surface. Provider-native caller-visible names are fixed by protocol. Version/tool/model -mismatches fail during catalog compilation. If an Anthropic credential cannot -access `browser_20260701`, Browser Loop retries with an equivalent `browser` function -tool and remembers that choice for the credential and process. Every -`loop.providers.*` tool surface exposes its first-party `source` (or versioned -`sources`), and every returned provider spec carries the applicable URL. +mismatches fail during catalog compilation. Anthropic's GA +`computer_toolset_20260801` and `browser_toolset_20260801` declarations need no +beta header and may be selected together; Browser Loop preserves member +`toolset_name` fields and sequential batch semantics across pi's transcript. +Every `loop.providers.*` tool surface exposes its first-party `source` or +`sources`, and every returned provider spec carries the applicable URL. ## Catalog compilation @@ -434,9 +435,9 @@ require different transports fails to compile. `openai-computer-use` api instead, which a Browser Loop adapter handles; that same adapter also covers tool-search namespace round-trips regardless of api, since pi's builtin transport does not replay them. -- **Anthropic**: exact native declarations, beta-header composition, and - adaptive model preparation. No api fork — every Anthropic model streams - through pi's builtin transport. +- **Anthropic**: GA client-toolset declarations, member/result transcript + adaptation, and adaptive model preparation. No beta header or api fork — + every Anthropic model streams through pi's builtin transport. - **Google**: a model selected without Google's native browser toolset streams through pi's builtin transport. Selecting `loop.providers.google.toolsets.browser()` derives the Browser Loop-owned diff --git a/packages/browser-loop/docs/supported-models.md b/packages/browser-loop/docs/supported-models.md index 3bdbffc0..e34189c9 100644 --- a/packages/browser-loop/docs/supported-models.md +++ b/packages/browser-loop/docs/supported-models.md @@ -22,8 +22,7 @@ models and need their own entry. Each cites first-party documentation. | provider | models | surfaces | | --- | --- | --- | -| `anthropic` | `claude-opus-4-8`, `claude-opus-5`, `claude-sonnet-5` families | computer, browser | -| `anthropic` | `claude-fable-5` family | computer | +| `anthropic` | `claude-fable-5`, `claude-mythos-5`, `claude-opus-4-8`, `claude-opus-5`, `claude-sonnet-5` families | computer, browser | | `openai` | `gpt-5.6-sol`, `gpt-5.4`, `gpt-5.4-mini`, `gpt-5.5` | computer | | `google` | `gemini-3.6-flash`, `gemini-3.5-flash`, `gemini-3.5-flash-lite`, `gemini-2.5-computer-use-preview-10-2025` | browser | @@ -93,9 +92,9 @@ npx tsx packages/browser-loop/scripts/native-action-probe.ts --provider openai - Update that provider's adapter under `src/pi/providers/` to execute the actions the probe returns, then add or adjust the `COMPUTER_USE_NATIVE_SURFACES` entry, citing the -provider's documentation. Anthropic's computer tool version and its -`computer-use-*` beta header are chosen by pi-ai per model, so a new dated -version there usually means bumping pi-ai rather than editing this package. +provider's documentation. Anthropic's GA client toolsets are adapted in +`src/pi/providers/anthropic/toolsets.ts`; a new dated toolset version requires +updating its declarations and member transcript adapter together. **A model rejects a tool Browser Loop sends.** Add a `LOOP_MODEL_QUIRKS` entry with the observed error as its `reason`, scoped as narrowly as the evidence supports: a diff --git a/packages/browser-loop/examples/anthropic-native-smoke.ts b/packages/browser-loop/examples/anthropic-native-smoke.ts index eec71955..0042ea0b 100644 --- a/packages/browser-loop/examples/anthropic-native-smoke.ts +++ b/packages/browser-loop/examples/anthropic-native-smoke.ts @@ -16,8 +16,8 @@ const CONFIGS: Record = { computer: loop.toolsets.computer(), browser: loop.toolsets.browser(), mixed: loop.toolsets.mixed(), - "native-computer": [loop.providers.anthropic.tools.computer({ version: "20260701", enableZoom: true })], - "native-browser": [loop.providers.anthropic.tools.browser({ version: "20260701" })], + "native-computer": [loop.providers.anthropic.tools.computer({ version: "20260801", zoom: true })], + "native-browser": [loop.providers.anthropic.tools.browser({ version: "20260801" })], }; const PROMPT = [ diff --git a/packages/browser-loop/examples/shared/tools.ts b/packages/browser-loop/examples/shared/tools.ts index 68bf06eb..ee32555d 100644 --- a/packages/browser-loop/examples/shared/tools.ts +++ b/packages/browser-loop/examples/shared/tools.ts @@ -24,7 +24,7 @@ export function toolsForModel(model: LoopModelRef): LoopAgentTool[] { // Claude 5 can use Anthropic's native browser tool; older models use portable // Loop tools plus the explicit semantic action-plan surface. return loop.providers.anthropic.supports.browser(modelId) - ? [loop.providers.anthropic.tools.browser({ version: "20260701", javascript: true })] + ? [loop.providers.anthropic.tools.browser({ version: "20260801", javascript: true })] : structuredBrowserTools(); case "google": // Current Gemini computer-use models expect Google's predefined browser actions. diff --git a/packages/browser-loop/src/core/actions/browser.ts b/packages/browser-loop/src/core/actions/browser.ts index c55d5cc3..e19168f9 100644 --- a/packages/browser-loop/src/core/actions/browser.ts +++ b/packages/browser-loop/src/core/actions/browser.ts @@ -23,15 +23,20 @@ export const BROWSER_ACTION_TYPES = [ "browser_find", "browser_click", "browser_hover", + "browser_mouse_down", + "browser_mouse_up", "browser_drag", "browser_fill", "browser_scroll_to", "browser_scroll", "browser_type", "browser_key", + "browser_hold_key", "browser_navigate", "browser_list_tabs", "browser_new_tab", + "browser_switch_tab", + "browser_close_tab", "browser_screenshot", "browser_evaluate", ] as const; @@ -138,6 +143,20 @@ export interface BrowserActionHover { tab_id?: string; } +export interface BrowserActionMouseDown { + type: "browser_mouse_down"; + x: number; + y: number; + tab_id?: string; +} + +export interface BrowserActionMouseUp { + type: "browser_mouse_up"; + x: number; + y: number; + tab_id?: string; +} + export interface BrowserActionDrag { type: "browser_drag"; from: { x: number; y: number }; @@ -180,9 +199,16 @@ export interface BrowserActionKey { tab_id?: string; } +export interface BrowserActionHoldKey { + type: "browser_hold_key"; + text: string; + duration: number; + tab_id?: string; +} + export interface BrowserActionNavigate { type: "browser_navigate"; - /** A URL, or the sentinels "back" / "forward" for history navigation. */ + /** A URL, or the sentinels "back", "forward", or "reload". */ url: string; tab_id?: string; } @@ -195,6 +221,16 @@ export interface BrowserActionNewTab { type: "browser_new_tab"; } +export interface BrowserActionSwitchTab { + type: "browser_switch_tab"; + tab_id: string; +} + +export interface BrowserActionCloseTab { + type: "browser_close_tab"; + tab_id: string; +} + export interface BrowserActionScreenshot { type: "browser_screenshot"; /** Optional crop region, [x0, y0, x1, y1] in viewport pixels. */ @@ -216,15 +252,20 @@ export type BrowserAction = | BrowserActionFind | BrowserActionClick | BrowserActionHover + | BrowserActionMouseDown + | BrowserActionMouseUp | BrowserActionDrag | BrowserActionFill | BrowserActionScrollTo | BrowserActionScroll | BrowserActionTypeText | BrowserActionKey + | BrowserActionHoldKey | BrowserActionNavigate | BrowserActionListTabs | BrowserActionNewTab + | BrowserActionSwitchTab + | BrowserActionCloseTab | BrowserActionScreenshot | BrowserActionEvaluate; @@ -403,6 +444,14 @@ export function createBrowserActionSchemaByType(options: BrowserActionSchemaOpti }, { additionalProperties: false }, ), + browser_mouse_down: Type.Object( + { type: Type.Literal("browser_mouse_down"), x: Type.Number(), y: Type.Number(), tab_id: TabId() }, + { additionalProperties: false }, + ), + browser_mouse_up: Type.Object( + { type: Type.Literal("browser_mouse_up"), x: Type.Number(), y: Type.Number(), tab_id: TabId() }, + { additionalProperties: false }, + ), browser_drag: Type.Object( { type: Type.Literal("browser_drag"), @@ -459,16 +508,33 @@ export function createBrowserActionSchemaByType(options: BrowserActionSchemaOpti }, { additionalProperties: false }, ), + browser_hold_key: Type.Object( + { + type: Type.Literal("browser_hold_key"), + text: Type.String({ description: "Key or chord to hold." }), + duration: Type.Number({ minimum: 0, maximum: 30 }), + tab_id: TabId(), + }, + { additionalProperties: false }, + ), browser_navigate: Type.Object( { type: Type.Literal("browser_navigate"), - url: Type.String({ description: "URL to navigate to, or \"back\" / \"forward\" for history navigation." }), + url: Type.String({ description: "URL to navigate to, or \"back\", \"forward\", or \"reload\"." }), tab_id: TabId(), }, { additionalProperties: false }, ), browser_list_tabs: Type.Object({ type: Type.Literal("browser_list_tabs") }, { additionalProperties: false }), browser_new_tab: Type.Object({ type: Type.Literal("browser_new_tab") }, { additionalProperties: false }), + browser_switch_tab: Type.Object( + { type: Type.Literal("browser_switch_tab"), tab_id: Type.String() }, + { additionalProperties: false }, + ), + browser_close_tab: Type.Object( + { type: Type.Literal("browser_close_tab"), tab_id: Type.String() }, + { additionalProperties: false }, + ), browser_screenshot: Type.Object( { type: Type.Literal("browser_screenshot"), diff --git a/packages/browser-loop/src/core/anthropic-native.ts b/packages/browser-loop/src/core/anthropic-native.ts index 0fda94ae..1fda1bb9 100644 --- a/packages/browser-loop/src/core/anthropic-native.ts +++ b/packages/browser-loop/src/core/anthropic-native.ts @@ -14,7 +14,7 @@ function asNativeInput(args: unknown): NativeInput { const MAX_KEY_REPEAT = 100; -/** Map one `computer_20260701` tool input onto canonical computer-plane actions. */ +/** Map one GA computer toolset member onto canonical computer-plane actions. */ export function mapNativeComputerInput(input: NativeInput): ComputerUseAction[] { switch (input.action) { case "screenshot": @@ -52,19 +52,19 @@ export function mapNativeComputerInput(input: NativeInput): ComputerUseAction[] } case "hold_key": // Canonical keypress duration is milliseconds; the native tool speaks seconds. - return [{ type: "keypress", keys: [text(input)], duration: durationSeconds(input) * 1000 }]; + return [{ type: "keypress", keys: [text(input)], duration: durationSeconds(input, 300) * 1000 }]; case "wait": - return [{ type: "wait", ms: durationSeconds(input) * 1000 }]; + return [{ type: "wait", ms: durationSeconds(input, 300) * 1000 }]; case "cursor_position": return [{ type: "cursor_position" }]; case "zoom": return [{ type: "zoom", region: region(input.region) }]; default: - throw new Error(`unsupported computer_20260701 action "${input.action}"`); + throw new Error(`unsupported computer toolset member "${input.action}"`); } } -/** Map one `browser_20260701` tool input onto canonical browser-plane actions. */ +/** Map one GA browser toolset member onto canonical browser-plane actions. */ export function mapNativeBrowserInput(input: NativeInput): ComputerUseAction[] { const tab = tabId(input); switch (input.action) { @@ -100,12 +100,20 @@ export function mapNativeBrowserInput(input: NativeInput): ComputerUseAction[] { return [{ type: "browser_click", ...pageTarget(input.target), ...modifiers(input.modifiers), ...tab }]; case "right_click": return [{ type: "browser_click", ...pageTarget(input.target), button: "right", ...modifiers(input.modifiers), ...tab }]; + case "middle_click": + return [{ type: "browser_click", ...pageTarget(input.target), button: "middle", ...modifiers(input.modifiers), ...tab }]; case "double_click": return [{ type: "browser_click", ...pageTarget(input.target), num_clicks: 2, ...modifiers(input.modifiers), ...tab }]; case "triple_click": return [{ type: "browser_click", ...pageTarget(input.target), num_clicks: 3, ...modifiers(input.modifiers), ...tab }]; case "hover": return [{ type: "browser_hover", ...pageTarget(input.target), ...tab }]; + case "mouse_move": + return [{ type: "browser_hover", ...coordinateTarget(input.target, "target"), ...tab }]; + case "left_mouse_down": + return [{ type: "browser_mouse_down", ...coordinateTarget(input.target, "target"), ...tab }]; + case "left_mouse_up": + return [{ type: "browser_mouse_up", ...coordinateTarget(input.target, "target"), ...tab }]; case "left_click_drag": return [{ type: "browser_drag", from: coordinateTarget(input.from, "from"), to: coordinateTarget(input.target, "target"), ...tab }]; case "scroll": @@ -124,12 +132,18 @@ export function mapNativeBrowserInput(input: NativeInput): ComputerUseAction[] { const repeat = clampRepeat(input.repeat); return Array.from({ length: repeat }, () => ({ type: "browser_key" as const, text: text(input), ...tab })); } + case "hold_key": + return [{ type: "browser_hold_key", text: text(input), duration: durationSeconds(input, 30), ...tab }]; case "wait": - return [{ type: "wait", ms: durationSeconds(input) * 1000 }]; + return [{ type: "wait", ms: durationSeconds(input, 30) * 1000 }]; + case "switch_tab": + return [{ type: "browser_switch_tab", tab_id: requireString(input.tab_id, "tab_id") }]; + case "close_tab": + return [{ type: "browser_close_tab", tab_id: requireString(input.tab_id, "tab_id") }]; case "javascript_exec": return [{ type: "browser_evaluate", code: text(input), ...tab }]; default: - throw new Error(`unsupported browser_20260701 action "${input.action}"`); + throw new Error(`unsupported browser toolset member "${input.action}"`); } } @@ -191,10 +205,10 @@ function requireString(value: unknown, field: string): string { return value; } -function durationSeconds(input: NativeInput): number { +function durationSeconds(input: NativeInput, maximum: number): number { const value = input.duration; if (typeof value !== "number" || !Number.isFinite(value) || value < 0) throw new Error("invalid duration: expected a non-negative number"); - return Math.min(value, 100); + return Math.min(value, maximum); } function clampRepeat(value: unknown): number { diff --git a/packages/browser-loop/src/core/resources.ts b/packages/browser-loop/src/core/resources.ts index b103297b..b8425372 100644 --- a/packages/browser-loop/src/core/resources.ts +++ b/packages/browser-loop/src/core/resources.ts @@ -4,7 +4,7 @@ import type { LoopCoordinateContract, LoopToolSpec } from "./tool-catalog"; import { formatBrowserActResult } from "./browser-result-format"; import { BatchExecutionError, InternalComputerTranslator, type KernelBrowser, type PlaywrightExecutionResult } from "./translator/translator"; import type { BrowserExecutor } from "./translator/browser"; -import type { BatchExecutionResult, BatchReadResult, BrowserWaitForResult } from "./translator/types"; +import type { BatchExecutionResult, BatchReadResult, BrowserState, BrowserWaitForResult } from "./translator/types"; /** Structured execution metadata returned by materialized Loop tools. */ export interface LoopExecutionDetails { @@ -187,6 +187,12 @@ function formatReadResults( content.push({ type: "text", text: read.text }); details.push({ type: "browser_text", label: read.label, bytes: read.text.length }); break; + case "browser_state": { + const text = formatBrowserState(read.state); + content.push({ type: "text", text }); + details.push({ type: "browser_state", state: read.state }); + break; + } case "browser_wait_for": content.push({ type: "text", text: formatBrowserWaitResult(read.result) }); details.push({ type: "browser_wait_for", result: read.result }); @@ -205,6 +211,14 @@ function formatReadResults( return { content, details }; } +function formatBrowserState(state: BrowserState): string { + if (state.tabs.length === 0) return "No tabs available"; + return [ + "Available tabs:", + ...state.tabs.map((tab) => ` • tab_id ${tab.tab_id}: ${JSON.stringify(tab.title)} (${tab.url})${tab.active ? " (current)" : ""}`), + ].join("\n"); +} + function hasUnsatisfiedSemanticRead(reads: readonly BatchReadResult[]): boolean { return reads.some((read) => read.type === "browser_wait_for" diff --git a/packages/browser-loop/src/core/tool-catalog.ts b/packages/browser-loop/src/core/tool-catalog.ts index fdbd0d30..f420f0a4 100644 --- a/packages/browser-loop/src/core/tool-catalog.ts +++ b/packages/browser-loop/src/core/tool-catalog.ts @@ -49,8 +49,8 @@ export type LoopProviderBinding = | { readonly kind: "anthropic-native"; readonly declaration: Record; - readonly beta: string; - readonly accessFallback?: LoopAnthropicBrowserFallback; + /** Client-toolset name echoed on each member tool call and result. */ + readonly toolsetName: "computer" | "browser"; } | { readonly kind: "openai-native"; @@ -150,16 +150,10 @@ export interface LoopPayloadPlan { apply(payload: unknown, model: LoopCatalogModel): Promise; } -/** Function-tool fallback for an Anthropic native browser tool unavailable to the active credential. */ -export interface LoopAnthropicBrowserFallback { - readonly beta: string; - readonly nativeType: string; - readonly declaration: Record; -} - /** Identity-addressed native call dispatch passed to Loop custom provider streams. */ export interface LoopIncomingToolPlan { - readonly anthropicBrowserFallback?: LoopAnthropicBrowserFallback; + /** GA Anthropic client toolsets selected for this request. */ + readonly anthropicToolsets: readonly ("computer" | "browser")[]; readonly openaiComputerName?: string; readonly googleNames: Readonly>; /** Google predefined functions disabled by the exact selected native subset. */ @@ -246,7 +240,7 @@ export function compileLoopToolCatalog(options: Comp const drafts = resolveProviderFacingDeclarations(normalizedEntries); const names = new Map(drafts.map((entry) => [entry.identity, entry.name])); - const requirements = compileHeaderRequirements(drafts); + const requirements: LoopHeaderRequirement[] = []; const transforms = compilePayloadTransforms(model, options.facts, drafts, validatePreparation(options.preparation)); validateTransformClaims(transforms); const incoming = compileIncomingPlan(drafts); @@ -447,25 +441,6 @@ function validateToolsetCompatibility(model: LoopCatalogModel, entries: readonly throw new Error(`selected tools contribute incompatible native provider transports: ${[...nativeProviderKinds].join(", ")}`); } - // Anthropic rejects its native browser and native computer tools in one - // request, because the browser tool addresses a viewport coordinate frame and - // the computer tool a display frame. Verified against the live API, which - // answers 400 "browser_20260701 cannot be declared alongside a computer_* - // tool". Catch it at compile time rather than on the wire. - const anthropicNativeTypes = entries.flatMap((entry) => - entry.providerBinding?.kind === "anthropic-native" && isRecord(entry.providerBinding.declaration) - ? [String(entry.providerBinding.declaration.type ?? "")] - : [], - ); - const anthropicBrowser = anthropicNativeTypes.find((type) => type.startsWith("browser_")); - const anthropicComputer = anthropicNativeTypes.find((type) => type.startsWith("computer_")); - if (anthropicBrowser && anthropicComputer) { - throw new Error( - `Anthropic's native browser tool (${anthropicBrowser}) cannot be selected alongside its native computer tool (${anthropicComputer}): ` + - "the browser tool's viewport coordinate frame is incompatible with the computer tool's display frame", - ); - } - const requiresApis = new Set(entries.flatMap((entry) => bindingRequiresApi(entry.providerBinding))); if (requiresApis.size > 1) { throw new Error(`selected tools require incompatible provider transports: ${[...requiresApis].join(", ")}`); @@ -501,15 +476,6 @@ function withApi(model: M | LoopCompiledModel, ap return { ...model, api }; } -function compileHeaderRequirements(entries: readonly LoopCatalogEntryDraft[]): LoopHeaderRequirement[] { - return entries.flatMap((entry) => { - const binding = entry.providerBinding; - return binding?.kind === "anthropic-native" - ? [{ identity: entry.identity, name: "anthropic-beta", value: binding.beta, merge: "comma-set" as const }] - : []; - }); -} - function createHeaderPlan(requirements: readonly LoopHeaderRequirement[]): LoopHeaderPlan { const frozenRequirements = Object.freeze(requirements.map((requirement) => Object.freeze({ ...requirement }))); return Object.freeze({ @@ -716,7 +682,7 @@ function createPayloadPlan( } function compileIncomingPlan(entries: readonly LoopCatalogEntryDraft[]): LoopIncomingToolPlan { - let anthropicBrowserFallback: LoopAnthropicBrowserFallback | undefined; + const anthropicToolsets: Array<"computer" | "browser"> = []; let openaiComputerName: string | undefined; let googleAllNativeNames: readonly string[] = []; const googleNames: Record = {}; @@ -725,7 +691,7 @@ function compileIncomingPlan(entries: readonly LoopCatalogEntryDraft[]): LoopInc const binding = entry.providerBinding; if (!binding) continue; nativeToolNames.push(entry.name); - if (binding.kind === "anthropic-native" && binding.accessFallback) anthropicBrowserFallback = binding.accessFallback; + if (binding.kind === "anthropic-native") anthropicToolsets.push(binding.toolsetName); else if (binding.kind === "openai-native") openaiComputerName = entry.name; else if (binding.kind === "google-native") { googleNames[binding.nativeName] = entry.name; @@ -735,7 +701,7 @@ function compileIncomingPlan(entries: readonly LoopCatalogEntryDraft[]): LoopInc const selectedGoogleNames = new Set(Object.keys(googleNames)); const googleExcludedNames = googleAllNativeNames.filter((name) => !selectedGoogleNames.has(name)); return Object.freeze({ - ...(anthropicBrowserFallback ? { anthropicBrowserFallback: Object.freeze(anthropicBrowserFallback) } : {}), + anthropicToolsets: Object.freeze(anthropicToolsets), ...(openaiComputerName ? { openaiComputerName } : {}), googleNames: Object.freeze(googleNames), googleExcludedNames: Object.freeze(googleExcludedNames), diff --git a/packages/browser-loop/src/core/tools.ts b/packages/browser-loop/src/core/tools.ts index 88a46549..74273eb9 100644 --- a/packages/browser-loop/src/core/tools.ts +++ b/packages/browser-loop/src/core/tools.ts @@ -71,7 +71,8 @@ const pixels = Object.freeze({ type: "pixel" as const }); const providerSources = Object.freeze({ openai: "https://developers.openai.com/api/docs/guides/tools-computer-use", - anthropic: "https://platform.claude.com/docs/en/agents-and-tools/tool-use/computer-use-tool", + anthropicComputer: "https://platform.claude.com/docs/en/agents-and-tools/tool-use/computer-use-tool", + anthropicBrowser: "https://platform.claude.com/docs/en/agents-and-tools/tool-use/browser-use-tool", google: "https://ai.google.dev/gemini-api/docs/computer-use", }); @@ -142,15 +143,20 @@ const browserDescriptions: Record = { browser_find: "Find matching elements and return snapshot-scoped refs.", browser_click: "Click an element, preferably by a ref from a current snapshot.", browser_hover: "Hover an element.", + browser_mouse_down: "Press the left mouse button at viewport coordinates.", + browser_mouse_up: "Release the left mouse button at viewport coordinates.", browser_drag: "Drag between viewport coordinates.", browser_fill: "Set a form element value by ref.", browser_scroll_to: "Scroll an element ref into view.", browser_scroll: "Scroll the page at a viewport position.", browser_type: "Type literal text at the current focus.", browser_key: "Press a key or chord.", + browser_hold_key: "Hold a key or chord for a bounded duration.", browser_navigate: "Navigate to a URL or move back/forward in history.", browser_list_tabs: "List open tabs.", browser_new_tab: "Open a new tab.", + browser_switch_tab: "Switch the active tab.", + browser_close_tab: "Close a tab.", browser_screenshot: "Capture the current viewport.", browser_evaluate: "Execute JavaScript in the page context.", browser_wait_for: "Wait for page text, element, URL, title, value, or state evidence without delivering input.", @@ -292,134 +298,42 @@ function playwright(options: LoopToolNameOptions = {}): LoopToolSpec { }); } -function anthropicNativeComputer(options: { version: "20260701"; enableZoom?: boolean; displayNumber?: number } = { version: "20260701" }): LoopToolSpec { - if (options.version !== "20260701") throw new Error(`unsupported Anthropic native computer version "${String(options.version)}"`); +function anthropicNativeComputer(options: { version: "20260801"; zoom?: boolean } = { version: "20260801" }): LoopToolSpec { + if (options.version !== "20260801") throw new Error(`unsupported Anthropic native computer version "${String(options.version)}"`); const declaration = { - type: "computer_20260701", - name: "computer", - ...(options.enableZoom !== undefined ? { enable_zoom: options.enableZoom } : {}), - ...(options.displayNumber !== undefined ? { display_number: options.displayNumber } : {}), + type: "computer_toolset_20260801", + ...(options.zoom !== undefined ? { configs: { zoom: { enabled: options.zoom } } } : {}), }; return providerNativeSpec({ - identity: "provider.anthropic.native.computer.20260701", + identity: "provider.anthropic.native.computer.20260801", name: "computer", - source: providerSources.anthropic, + source: providerSources.anthropicComputer, declaration, - binding: { kind: "anthropic-native", declaration, beta: "computer-use-2026-07-01" }, + binding: { kind: "anthropic-native", declaration, toolsetName: "computer" }, toActions: (input) => mapNativeComputerInput(asNativeInput(input)), coordinates: pixels, stopTurnOnFailureMessage: "Not executed: an earlier computer action in this turn failed.", }); } -function anthropicNativeBrowser(options: { version: "20260701"; javascript?: boolean } = { version: "20260701" }): LoopToolSpec { - if (options.version !== "20260701") throw new Error(`unsupported Anthropic native browser version "${String(options.version)}"`); +function anthropicNativeBrowser(options: { version: "20260801"; javascript?: boolean } = { version: "20260801" }): LoopToolSpec { + if (options.version !== "20260801") throw new Error(`unsupported Anthropic native browser version "${String(options.version)}"`); const declaration = { - type: "browser_20260701", - name: "browser", - ...(options.javascript !== undefined ? { enable_javascript_exec: options.javascript } : {}), + type: "browser_toolset_20260801", + ...(options.javascript !== undefined ? { configs: { javascript_exec: { enabled: options.javascript } } } : {}), }; return providerNativeSpec({ - identity: "provider.anthropic.native.browser.20260701", + identity: "provider.anthropic.native.browser.20260801", name: "browser", - source: providerSources.anthropic, + source: providerSources.anthropicBrowser, declaration, - binding: { - kind: "anthropic-native", - declaration, - beta: "browser-use-2026-07-01", - accessFallback: { - beta: "browser-use-2026-07-01", - nativeType: "browser_20260701", - declaration: anthropicBrowserFunctionFallback(options.javascript !== false), - }, - }, + binding: { kind: "anthropic-native", declaration, toolsetName: "browser" }, toActions: (input) => mapNativeBrowserInput(asNativeInput(input)), coordinates: pixels, stopTurnOnFailureMessage: "Not executed: an earlier action in this turn failed.", }); } -const anthropicNativeBrowserActions = [ - "navigate", "list_tabs", "new_tab", "read_page", "get_page_text", "find", "form_input", "scroll_to", "screenshot", "zoom", - "left_click", "right_click", "double_click", "triple_click", "hover", "left_click_drag", "scroll", "type", "key", "wait", "javascript_exec", -] as const; - -type AnthropicNativeBrowserAction = (typeof anthropicNativeBrowserActions)[number]; - -function anthropicBrowserFunctionFallback(javascript: boolean): Record { - const actions = javascript - ? anthropicNativeBrowserActions - : anthropicNativeBrowserActions.filter((action) => action !== "javascript_exec"); - const union = Type.Union(actions.map(anthropicNativeBrowserActionSchema)); - return { - name: "browser", - description: "Use a browser through structured navigation, observation, and interaction actions.", - input_schema: { ...union, type: "object" }, - }; -} - -function anthropicNativeBrowserActionSchema(action: AnthropicNativeBrowserAction): TSchema { - const tab = () => Type.Optional(Type.String()); - const region = () => Type.Array(Type.Integer(), { minItems: 4, maxItems: 4 }); - const refTarget = () => Type.Object({ type: Type.Literal("ref"), ref: Type.String() }, { additionalProperties: false }); - const coordinateTarget = () => Type.Object({ type: Type.Literal("coordinate"), x: Type.Integer(), y: Type.Integer() }, { additionalProperties: false }); - const pageTarget = () => Type.Union([refTarget(), coordinateTarget()]); - const clickable = () => Type.Object({ - action: Type.Literal(action), - target: pageTarget(), - modifiers: Type.Optional(Type.String()), - tab_id: tab(), - }, { additionalProperties: false }); - switch (action) { - case "navigate": - return Type.Object({ action: Type.Literal(action), url: Type.String(), tab_id: tab() }, { additionalProperties: false }); - case "list_tabs": case "new_tab": - return Type.Object({ action: Type.Literal(action) }, { additionalProperties: false }); - case "read_page": - return Type.Object({ - action: Type.Literal(action), - filter: Type.Optional(Type.Union([Type.Literal("interactive"), Type.Literal("all")])), - depth: Type.Optional(Type.Integer({ minimum: 0 })), - ref: Type.Optional(Type.String()), - tab_id: tab(), - }, { additionalProperties: false }); - case "get_page_text": case "screenshot": - return Type.Object({ action: Type.Literal(action), tab_id: tab() }, { additionalProperties: false }); - case "find": - return Type.Object({ action: Type.Literal(action), query: Type.String(), tab_id: tab() }, { additionalProperties: false }); - case "form_input": - return Type.Object({ - action: Type.Literal(action), - target: refTarget(), - value: Type.Union([Type.String(), Type.Number(), Type.Boolean()]), - tab_id: tab(), - }, { additionalProperties: false }); - case "scroll_to": - return Type.Object({ action: Type.Literal(action), target: refTarget(), tab_id: tab() }, { additionalProperties: false }); - case "zoom": - return Type.Object({ action: Type.Literal(action), region: region(), tab_id: tab() }, { additionalProperties: false }); - case "left_click": case "right_click": case "double_click": case "triple_click": case "hover": - return clickable(); - case "left_click_drag": - return Type.Object({ action: Type.Literal(action), from: coordinateTarget(), target: coordinateTarget(), tab_id: tab() }, { additionalProperties: false }); - case "scroll": - return Type.Object({ - action: Type.Literal(action), - target: coordinateTarget(), - scroll_direction: Type.Union([Type.Literal("up"), Type.Literal("down"), Type.Literal("left"), Type.Literal("right")]), - scroll_amount: Type.Optional(Type.Integer({ minimum: 1 })), - tab_id: tab(), - }, { additionalProperties: false }); - case "type": case "javascript_exec": - return Type.Object({ action: Type.Literal(action), text: Type.String(), tab_id: tab() }, { additionalProperties: false }); - case "key": - return Type.Object({ action: Type.Literal(action), text: Type.String(), repeat: Type.Optional(Type.Integer({ minimum: 1, maximum: 100 })), tab_id: tab() }, { additionalProperties: false }); - case "wait": - return Type.Object({ action: Type.Literal(action), duration: Type.Number({ minimum: 0, maximum: 100 }) }, { additionalProperties: false }); - } -} - function openaiNativeComputer(): LoopToolSpec { const declaration = { type: "computer" }; return providerNativeSpec({ @@ -842,7 +756,8 @@ function computerToolset(options: LoopComputerToolsetOptions = {}): LoopToolSpec const providers = Object.freeze({ openai: Object.freeze({ source: providerSources.openai, tools: Object.freeze({ computer: openaiNativeComputer }) }), anthropic: Object.freeze({ - source: providerSources.anthropic, + source: providerSources.anthropicComputer, + sources: Object.freeze({ computer: providerSources.anthropicComputer, browser: providerSources.anthropicBrowser }), tools: Object.freeze({ computer: anthropicNativeComputer, browser: anthropicNativeBrowser }), }), google: Object.freeze({ diff --git a/packages/browser-loop/src/core/translator/browser.ts b/packages/browser-loop/src/core/translator/browser.ts index cb2685c1..c90a882c 100644 --- a/packages/browser-loop/src/core/translator/browser.ts +++ b/packages/browser-loop/src/core/translator/browser.ts @@ -6,7 +6,10 @@ import type { BrowserActionFill, BrowserActionFind, BrowserActionHover, + BrowserActionHoldKey, BrowserActionKey, + BrowserActionMouseDown, + BrowserActionMouseUp, BrowserActionNavigate, BrowserActionScroll, BrowserActionScrollTo, @@ -52,7 +55,7 @@ import { type RefEntry, } from "./browser-ref-lifecycle"; import { evaluateBrowserExpectation, waitForBrowserExpectation, type BrowserExpectationEvaluation } from "./browser-wait"; -import type { BatchReadResult, BrowserActResult, BrowserWaitForResult } from "./types"; +import type { BatchReadResult, BrowserActResult, BrowserState, BrowserWaitForResult } from "./types"; const SNAPSHOT_CHAR_LIMIT = 50_000; const DEFAULT_SNAPSHOT_DEPTH = 15; @@ -409,6 +412,12 @@ export class BrowserExecutor { case "browser_hover": await this.hover(action, signal); return []; + case "browser_mouse_down": + await this.mouseButton(action, "mousePressed"); + return []; + case "browser_mouse_up": + await this.mouseButton(action, "mouseReleased"); + return []; case "browser_drag": await this.drag(action); return []; @@ -429,12 +438,19 @@ export class BrowserExecutor { case "browser_key": await this.key(action, signal); return []; + case "browser_hold_key": + await this.holdKey(action, signal); + return []; case "browser_navigate": return [{ type: "browser_text", label: "navigate", text: await this.navigate(action, signal) }]; case "browser_list_tabs": - return [{ type: "browser_text", label: "tabs", text: await this.listTabs() }]; + return [{ type: "browser_state", state: await this.listTabs() }]; case "browser_new_tab": - return [{ type: "browser_text", label: "new_tab", text: await this.newTab() }]; + return [{ type: "browser_state", state: await this.newTab() }]; + case "browser_switch_tab": + return [{ type: "browser_state", state: await this.switchTab(action.tab_id) }]; + case "browser_close_tab": + return [{ type: "browser_state", state: await this.closeTab(action.tab_id) }]; case "browser_screenshot": return [{ type: "screenshot", ...(await this.screenshot(action.region, action.tab_id)) }]; case "browser_evaluate": @@ -921,6 +937,11 @@ export class BrowserExecutor { await this.cdp.send("Input.dispatchMouseEvent", { type: "mouseMoved", x: point.x, y: point.y }, point.session); } + private async mouseButton(action: BrowserActionMouseDown | BrowserActionMouseUp, type: "mousePressed" | "mouseReleased"): Promise { + const session = await this.session(tabOf(action)); + await this.cdp.send("Input.dispatchMouseEvent", { type, x: action.x, y: action.y, button: "left", clickCount: 1 }, session); + } + private async drag(action: BrowserActionDrag): Promise { const session = await this.session(tabOf(action)); await this.cdp.send("Input.dispatchMouseEvent", { type: "mousePressed", x: action.from.x, y: action.from.y, button: "left", clickCount: 1 }, session); @@ -992,6 +1013,22 @@ export class BrowserExecutor { } } + private async holdKey(action: BrowserActionHoldKey, signal?: AbortSignal): Promise { + throwIfAborted(signal); + const session = await this.session(tabOf(action)); + const parts = action.text.split("+").filter(Boolean); + const keyPart = parts[parts.length - 1] ?? ""; + const modifiers = modifierBits(parts.slice(0, -1)); + const key = resolveKey(keyPart); + const base = { key: key.key, code: key.code, windowsVirtualKeyCode: key.keyCode, modifiers }; + await this.cdp.send("Input.dispatchKeyEvent", { type: key.text ? "keyDown" : "rawKeyDown", ...base, ...(key.text ? { text: key.text } : {}) }, session); + try { + await abortableDelay(Math.min(Math.max(action.duration, 0), 30) * 1000, signal); + } finally { + await this.cdp.send("Input.dispatchKeyEvent", { type: "keyUp", ...base }, session); + } + } + private async dispatchChord(chord: string, session: string): Promise { const parts = chord.split("+").filter(Boolean); const keyPart = parts[parts.length - 1] ?? ""; @@ -1009,6 +1046,23 @@ export class BrowserExecutor { const session = await this.attach(targetId); throwIfAborted(signal); const direction = action.url.trim().toLowerCase(); + if (direction === "reload") { + const frameId = await this.mainFrameId(targetId, session); + const pending = this.beginNavigation(targetId, session); + try { + await this.selfNavigate(targetId, () => this.cdp.send("Page.reload", {}, session)); + } catch (error) { + this.cancelNavigation(pending); + throw error; + } + this.lifecycle.invalidateTarget(targetId); + try { + await this.waitForNavigation(pending, { frameId }, signal); + } finally { + this.selfNavigations.delete(targetId); + } + return `Reloaded page.\n${await this.tabContext(targetId)}`; + } if (direction === "back" || direction === "forward") { const history = await this.cdp.send<{ currentIndex: number; entries: Array<{ id: number; url: string }> }>( "Page.getNavigationHistory", @@ -1035,7 +1089,7 @@ export class BrowserExecutor { return `Navigated ${direction}.\n${await this.tabContext(targetId)}`; } const url = normalizeGotoUrl(action.url); - if (!url) throw new Error("invalid url"); + if (!url || !/^https?:\/\//i.test(url)) throw new Error("browser navigation requires an http or https URL"); const pending = this.beginNavigation(targetId, session); let response: { frameId?: string; loaderId?: string; errorText?: string; isDownload?: boolean }; try { @@ -1099,16 +1153,51 @@ export class BrowserExecutor { return targets.find((target) => target.targetId === targetId)?.url ?? ""; } - private async listTabs(): Promise { - const targets = await this.cdp.pageTargets(); - if (targets.length === 0) return "No open tabs."; - return targets.map((target) => `tab_id ${shortTabId(target.targetId)}: ${JSON.stringify(target.title)} (${target.url})`).join("\n"); + private async listTabs(): Promise { + return this.browserState(); } - private async newTab(): Promise { + private async newTab(): Promise { const targetId = await this.cdp.createTarget("about:blank"); this.activeTargetId = targetId; - return `Opened tab_id ${shortTabId(targetId)}.\n${await this.tabContext(targetId)}`; + await this.cdp.send("Target.activateTarget", { targetId }); + return this.browserState([{ type: "tab_opened", tab_id: shortTabId(targetId) }]); + } + + private async switchTab(tabId: string): Promise { + const targetId = await this.resolveTarget(tabId); + await this.cdp.send("Target.activateTarget", { targetId }); + this.activeTargetId = targetId; + return this.browserState(); + } + + private async closeTab(tabId: string): Promise { + const targetId = await this.resolveTarget(tabId); + await this.cdp.send("Target.closeTarget", { targetId }); + this.dropTarget(targetId); + const targets = await this.cdp.pageTargets(); + this.activeTargetId = targets.find((target) => target.targetId !== targetId)?.targetId; + if (this.activeTargetId) await this.cdp.send("Target.activateTarget", { targetId: this.activeTargetId }); + return this.browserState(undefined, targetId); + } + + private async browserState( + stateChanges?: BrowserState["state_changes"], + excludedTargetId?: string, + ): Promise { + const targets = (await this.cdp.pageTargets()).filter((target) => target.targetId !== excludedTargetId); + if (targets.length > 0 && !targets.some((target) => target.targetId === this.activeTargetId)) { + this.activeTargetId = targets[0]!.targetId; + } + return { + tabs: targets.map((target) => ({ + tab_id: shortTabId(target.targetId), + title: target.title, + url: target.url, + ...(target.targetId === this.activeTargetId ? { active: true as const } : {}), + })), + ...(stateChanges?.length ? { state_changes: stateChanges } : {}), + }; } private async evaluate(code: string, tabId?: string): Promise { @@ -1376,18 +1465,18 @@ function throwIfAborted(signal?: AbortSignal): void { throw signal.reason instanceof Error ? signal.reason : new Error("browser action aborted"); } -function abortableDelay(ms: number, signal: AbortSignal): Promise { +function abortableDelay(ms: number, signal?: AbortSignal): Promise { throwIfAborted(signal); return new Promise((resolve, reject) => { const onAbort = () => { clearTimeout(timer); - reject(signal.reason instanceof Error ? signal.reason : new Error("browser action aborted")); + reject(signal?.reason instanceof Error ? signal.reason : new Error("browser action aborted")); }; const timer = setTimeout(() => { - signal.removeEventListener("abort", onAbort); + signal?.removeEventListener("abort", onAbort); resolve(); }, ms); - signal.addEventListener("abort", onAbort, { once: true }); + signal?.addEventListener("abort", onAbort, { once: true }); }); } diff --git a/packages/browser-loop/src/core/translator/types.ts b/packages/browser-loop/src/core/translator/types.ts index 8eecba82..184a6175 100644 --- a/packages/browser-loop/src/core/translator/types.ts +++ b/packages/browser-loop/src/core/translator/types.ts @@ -150,6 +150,18 @@ export interface BrowserActResult { successor: BrowserActSuccessor; } +export interface BrowserTabState { + tab_id: string; + title: string; + url: string; + active?: true; +} + +export interface BrowserState { + tabs: BrowserTabState[]; + state_changes?: Array<{ type: "tab_opened"; tab_id: string }>; +} + /** Structured result returned by the standalone `browser_wait_for` action. */ export interface BrowserWaitForResult { status: "satisfied" | "timed_out" | "unverifiable" | "interrupted"; @@ -166,6 +178,7 @@ export type BatchReadResult = | { type: "url"; url: string } | { type: "cursor_position"; x: number; y: number } | { type: "browser_text"; label: string; text: string } + | { type: "browser_state"; state: BrowserState } | { type: "browser_wait_for"; result: BrowserWaitForResult } | { type: "browser_act"; result: BrowserActResult }; diff --git a/packages/browser-loop/src/index.ts b/packages/browser-loop/src/index.ts index 651ae253..26dcf158 100644 --- a/packages/browser-loop/src/index.ts +++ b/packages/browser-loop/src/index.ts @@ -43,6 +43,8 @@ export type { BrowserExpectationState, BrowserObservationDiff, BrowserObservationDiffEntry, + BrowserState, + BrowserTabState, BrowserWaitForResult, BrowserWaitReason, } from "./core/translator/types"; diff --git a/packages/browser-loop/src/pi-extension/index.ts b/packages/browser-loop/src/pi-extension/index.ts index b857e09f..70465da2 100644 --- a/packages/browser-loop/src/pi-extension/index.ts +++ b/packages/browser-loop/src/pi-extension/index.ts @@ -148,8 +148,8 @@ export default function loopPiExtension(pi: ExtensionAPI): void { * streams its own registry model, but the transport a native surface needs is * derived onto the *compiled* model — so the wrapper swaps in `catalog.model`, * which is the resolved model with only `api` replaced, and adds the incoming - * native-call plan that normalizes `computer_call`-style items and drives - * Anthropic's browser-beta fallback. pi's own resolved credential rides along + * native-call plan that normalizes provider-native items and preserves + * Anthropic client-toolset member calls. pi's own resolved credential rides along * in `options.apiKey`. */ function registerLoopProviders(): void { diff --git a/packages/browser-loop/src/pi-extension/selection.ts b/packages/browser-loop/src/pi-extension/selection.ts index 66bcbf21..0fcce0ad 100644 --- a/packages/browser-loop/src/pi-extension/selection.ts +++ b/packages/browser-loop/src/pi-extension/selection.ts @@ -40,8 +40,8 @@ const MENU: Readonly LoopToolS ], "browser-act": () => [loop.tools.browser.act()], playwright: () => [loop.tools.playwright()], - "anthropic-computer": () => [loop.providers.anthropic.tools.computer({ version: "20260701", enableZoom: true })], - "anthropic-browser": () => [loop.providers.anthropic.tools.browser({ version: "20260701", javascript: true })], + "anthropic-computer": () => [loop.providers.anthropic.tools.computer({ version: "20260801", zoom: true })], + "anthropic-browser": () => [loop.providers.anthropic.tools.browser({ version: "20260801", javascript: true })], "openai-computer": () => [loop.providers.openai.tools.computer()], "google-browser": () => loop.providers.google.toolsets.browser(), }); diff --git a/packages/browser-loop/src/pi/providers.ts b/packages/browser-loop/src/pi/providers.ts index 15187b9f..55619496 100644 --- a/packages/browser-loop/src/pi/providers.ts +++ b/packages/browser-loop/src/pi/providers.ts @@ -12,7 +12,7 @@ import { } from "@earendil-works/pi-ai"; import { builtinModels } from "@earendil-works/pi-ai/providers/all"; import { loopApiKeyEnvVarsForProvider } from "./api-keys"; -import { withAnthropicBrowserFallback } from "./providers/anthropic/browser-fallback"; +import { withAnthropicToolsets } from "./providers/anthropic/toolsets"; import { GOOGLE_INTERACTIONS_API, streamGoogleInteractions, streamSimpleGoogleInteractions } from "./providers/google/provider"; import { OPENAI_COMPUTER_USE_API, requiresOpenAINamespaceAdapter, streamOpenAIComputerUse, streamOpenAIResponses, streamSimpleOpenAIResponses } from "./providers/openai/provider"; @@ -20,8 +20,8 @@ import { OPENAI_COMPUTER_USE_API, requiresOpenAINamespaceAdapter, streamOpenAICo * Build the pi `Models` collection Loop streams through: pi's builtin * providers, adjusted for Loop: * - * - `anthropic` retries an inaccessible native browser beta through the - * selected tool's equivalent function declaration. + * - `anthropic` adapts pi's function-tool transcript to the GA computer and + * browser client-toolset protocol, including member names and toolset results. * - `openai` streams through pi's builtin `openai-responses` transport and its * automatic prompt caching by default; a model compiled with OpenAI's native * computer tool carries `openai-computer-use` instead, which this wrapper @@ -43,7 +43,7 @@ import { OPENAI_COMPUTER_USE_API, requiresOpenAINamespaceAdapter, streamOpenAICo export function createLoopModels(options?: CreateModelsOptions): MutableModels { const models = builtinModels(options); const anthropic = models.getProvider("anthropic"); - if (anthropic) models.setProvider(withAnthropicBrowserFallback(anthropic)); + if (anthropic) models.setProvider(withAnthropicToolsets(anthropic)); const openai = models.getProvider("openai"); if (openai) models.setProvider(withOpenAIComputerUseAdapter(openai)); const google = models.getProvider("google"); diff --git a/packages/browser-loop/src/pi/providers/anthropic/browser-fallback.ts b/packages/browser-loop/src/pi/providers/anthropic/browser-fallback.ts deleted file mode 100644 index 8bb051d7..00000000 --- a/packages/browser-loop/src/pi/providers/anthropic/browser-fallback.ts +++ /dev/null @@ -1,223 +0,0 @@ -import { createHash } from "node:crypto"; -import { - createAssistantMessageEventStream, - type Api, - type AssistantMessage, - type AssistantMessageEvent, - type AssistantMessageEventStream, - type Context, - type Model, - type Provider, - type SimpleStreamOptions, - type StreamOptions, -} from "@earendil-works/pi-ai"; -import type { LoopAnthropicBrowserFallback } from "../../../core/tool-catalog"; -import type { LoopSimpleStreamOptions } from "../common"; - -const supportedCredentials = new Set(); -const unsupportedCredentials = new Set(); - -/** Add transparent function-tool fallback when a credential cannot use Anthropic's native browser beta. */ -export function withAnthropicBrowserFallback(base: Provider): Provider { - return { - ...base, - stream: (model, context, options) => streamWithFallback( - base.stream.bind(base), - model, - context, - options as LoopSimpleStreamOptions | undefined, - ), - streamSimple: (model, context, options) => streamWithFallback( - base.streamSimple.bind(base), - model, - context, - options as LoopSimpleStreamOptions | undefined, - ), - }; -} - -type StartStream = ( - model: Model, - context: Context, - options?: StreamOptions | SimpleStreamOptions, -) => AssistantMessageEventStream; - -function streamWithFallback( - start: StartStream, - model: Model, - context: Context, - options: LoopSimpleStreamOptions | undefined, -): AssistantMessageEventStream { - const fallback = options?.loopIncomingToolPlan?.anthropicBrowserFallback; - if (!fallback) return start(model, context, options); - const credential = credentialKey(model, options); - if (supportedCredentials.has(credential)) return start(model, context, options); - if (unsupportedCredentials.has(credential)) return start(model, context, fallbackOptions(options, fallback)); - - const output = createAssistantMessageEventStream(); - void detectAccessAndRelay(output, start, model, context, options, fallback, credential); - return output; -} - -async function detectAccessAndRelay( - output: AssistantMessageEventStream, - start: StartStream, - model: Model, - context: Context, - options: LoopSimpleStreamOptions, - fallback: LoopAnthropicBrowserFallback, - credential: string, -): Promise { - let committed = false; - let terminal: AssistantMessage | undefined; - const pending: AssistantMessageEvent[] = []; - try { - const source = start(model, context, options); - for await (const event of source) { - if (!committed && event.type === "error" && isAnthropicBrowserAccessError(event.error.errorMessage, fallback)) { - unsupportedCredentials.add(credential); - await relay(output, start(model, context, fallbackOptions(options, fallback))); - return; - } - if (!committed) { - pending.push(event); - if (event.type === "start") continue; - committed = true; - if (event.type !== "error") supportedCredentials.add(credential); - for (const buffered of pending) output.push(buffered); - } else { - output.push(event); - } - terminal = terminalMessage(event) ?? terminal; - } - if (!committed) { - for (const event of pending) output.push(event); - } - output.end(terminal ?? await source.result()); - } catch (error) { - if (!committed && isAnthropicBrowserAccessError(errorMessage(error), fallback)) { - unsupportedCredentials.add(credential); - try { - await relay(output, start(model, context, fallbackOptions(options, fallback))); - return; - } catch (fallbackError) { - error = fallbackError; - } - } - const message = failureMessage(model, error); - output.push({ type: "error", reason: "error", error: message }); - output.end(message); - } -} - -async function relay(output: AssistantMessageEventStream, source: AssistantMessageEventStream): Promise { - let terminal: AssistantMessage | undefined; - for await (const event of source) { - output.push(event); - terminal = terminalMessage(event) ?? terminal; - } - output.end(terminal ?? await source.result()); -} - -function fallbackOptions( - options: LoopSimpleStreamOptions, - fallback: LoopAnthropicBrowserFallback, -): LoopSimpleStreamOptions { - const originalOnPayload = options.onPayload; - return { - ...options, - headers: removeHeaderToken(options.headers, "anthropic-beta", fallback.beta), - onPayload: async (payload, model) => { - const generated = (await originalOnPayload?.(payload, model)) ?? payload; - return replaceNativeDeclaration(generated, fallback); - }, - }; -} - -function replaceNativeDeclaration(payload: unknown, fallback: LoopAnthropicBrowserFallback): unknown { - if (!isRecord(payload) || !Array.isArray(payload.tools)) return payload; - let replaced = false; - const tools = payload.tools.map((tool) => { - if (!isRecord(tool) || tool.type !== fallback.nativeType) return tool; - replaced = true; - return fallback.declaration; - }); - return replaced ? { ...payload, tools } : payload; -} - -function removeHeaderToken( - headers: StreamOptions["headers"], - name: string, - token: string, -): StreamOptions["headers"] { - if (!headers) return undefined; - const next = { ...headers }; - for (const [headerName, value] of Object.entries(next)) { - if (headerName.toLowerCase() !== name || typeof value !== "string") continue; - const tokens = value.split(",").map((entry) => entry.trim()).filter((entry) => entry && entry !== token); - if (tokens.length > 0) next[headerName] = tokens.join(","); - else delete next[headerName]; - } - return Object.keys(next).length > 0 ? next : undefined; -} - -/** Return whether an Anthropic provider error specifically denies the selected native browser beta. */ -export function isAnthropicBrowserAccessError( - message: string | undefined, - fallback: Pick, -): boolean { - if (!message) return false; - const value = message.toLowerCase(); - const namesBrowserFeature = value.includes(fallback.nativeType.toLowerCase()) - || value.includes(fallback.beta.toLowerCase()) - || value.includes("browser tool") - || value.includes("browser use"); - return namesBrowserFeature && [ - "access", "permission", "not available", "not enabled", "not supported", "unsupported", - "unknown beta", "invalid beta", "unrecognized beta", - ].some((marker) => value.includes(marker)); -} - -function terminalMessage(event: AssistantMessageEvent): AssistantMessage | undefined { - if (event.type === "done") return event.message; - if (event.type === "error") return event.error; - return undefined; -} - -function credentialKey(model: Model, options: StreamOptions): string { - const authHeaders = Object.entries(options.headers ?? {}) - .filter(([name, value]) => ["authorization", "x-api-key"].includes(name.toLowerCase()) && typeof value === "string") - .sort(([left], [right]) => left.localeCompare(right)); - const credential = options.apiKey ?? JSON.stringify(authHeaders); - const digest = createHash("sha256").update(credential).digest("hex"); - return `${model.id}\u0000${digest}`; -} - -function failureMessage(model: Model, error: unknown): AssistantMessage { - return { - role: "assistant", - content: [], - api: model.api, - provider: model.provider, - model: model.id, - usage: { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - totalTokens: 0, - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, - }, - stopReason: "error", - errorMessage: errorMessage(error), - timestamp: Date.now(), - }; -} - -function errorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error); -} - -function isRecord(value: unknown): value is Record { - return Boolean(value && typeof value === "object" && !Array.isArray(value)); -} diff --git a/packages/browser-loop/src/pi/providers/anthropic/capabilities.ts b/packages/browser-loop/src/pi/providers/anthropic/capabilities.ts index 81e987de..303c3e29 100644 --- a/packages/browser-loop/src/pi/providers/anthropic/capabilities.ts +++ b/packages/browser-loop/src/pi/providers/anthropic/capabilities.ts @@ -1,14 +1,13 @@ -const NATIVE_BROWSER_FAMILIES = ["claude-opus-4-8", "claude-opus-5", "claude-sonnet-5"] as const; -const NATIVE_COMPUTER_FAMILIES = ["claude-fable-5", ...NATIVE_BROWSER_FAMILIES] as const; +const NATIVE_TOOLSET_FAMILIES = ["claude-fable-5", "claude-mythos-5", "claude-opus-4-8", "claude-opus-5", "claude-sonnet-5"] as const; -/** Return whether an Anthropic model ID supports the July 2026 native browser tool. */ +/** Return whether an Anthropic model ID supports the GA browser toolset. */ export function supportsAnthropicNativeBrowser(modelId: string): boolean { - return NATIVE_BROWSER_FAMILIES.some((family) => modelFamily(modelId, family)); + return NATIVE_TOOLSET_FAMILIES.some((family) => modelFamily(modelId, family)); } -/** Return whether an Anthropic model ID supports the July 2026 native computer tool. */ +/** Return whether an Anthropic model ID supports the GA computer toolset. */ export function supportsAnthropicNativeComputer(modelId: string): boolean { - return NATIVE_COMPUTER_FAMILIES.some((family) => modelFamily(modelId, family)); + return NATIVE_TOOLSET_FAMILIES.some((family) => modelFamily(modelId, family)); } function modelFamily(modelId: string, family: string): boolean { diff --git a/packages/browser-loop/src/pi/providers/anthropic/toolsets.ts b/packages/browser-loop/src/pi/providers/anthropic/toolsets.ts new file mode 100644 index 00000000..cc29fd63 --- /dev/null +++ b/packages/browser-loop/src/pi/providers/anthropic/toolsets.ts @@ -0,0 +1,305 @@ +import { + createAssistantMessageEventStream, + type Api, + type AssistantMessage, + type AssistantMessageEvent, + type AssistantMessageEventStream, + type Context, + type FetchFunction, + type Model, + type Provider, + type SimpleStreamOptions, + type StreamOptions, + type ToolCall, +} from "@earendil-works/pi-ai"; +import type { BrowserState } from "../../../core/translator/types"; +import type { LoopSimpleStreamOptions } from "../common"; + +type AnthropicToolsetName = "computer" | "browser"; + +// pi-ai currently drops `toolset_name`. Encode it into the streamed tool name +// before pi-ai parses the event, then restore the local tool name and action. +const ENCODED_MEMBER_PREFIX = "loop_anthropic_toolset_"; +const TAB_MEMBERS = new Set(["new_tab", "list_tabs", "switch_tab", "close_tab"]); + +/** Adapt pi-ai's function-tool transcript to Anthropic's GA client-toolset protocol. */ +export function withAnthropicToolsets(base: Provider): Provider { + return { + ...base, + stream: (model, context, options) => streamWithToolsets( + base.stream.bind(base), + model, + context, + options as LoopSimpleStreamOptions | undefined, + ), + streamSimple: (model, context, options) => streamWithToolsets( + base.streamSimple.bind(base), + model, + context, + options as LoopSimpleStreamOptions | undefined, + ), + }; +} + +type StartStream = ( + model: Model, + context: Context, + options?: StreamOptions | SimpleStreamOptions, +) => AssistantMessageEventStream; + +function streamWithToolsets( + start: StartStream, + model: Model, + context: Context, + options: LoopSimpleStreamOptions | undefined, +): AssistantMessageEventStream { + const toolsets = options?.loopIncomingToolPlan?.anthropicToolsets ?? []; + if (toolsets.length === 0) return start(model, context, options); + if ((options as StreamOptions & { client?: unknown } | undefined)?.client) { + throw new Error("Anthropic client toolsets require the provider fetch transport; a custom Anthropic client is not supported"); + } + const selected = new Set(toolsets); + const fetch = createToolsetFetch(options?.fetch ?? globalThis.fetch, selected, browserStates(context)); + const source = start(model, context, { ...options, fetch }); + const output = createAssistantMessageEventStream(); + void relayToolsetEvents(output, source, model); + return output; +} + +async function relayToolsetEvents( + output: AssistantMessageEventStream, + source: AssistantMessageEventStream, + model: Model, +): Promise { + let terminal: AssistantMessage | undefined; + const members = new Map(); + try { + for await (const event of source) { + normalizeEvent(event, members); + output.push(event); + terminal = terminalMessage(event) ?? terminal; + } + const result = terminal ?? await source.result(); + normalizeMessage(result, members); + output.end(result); + } catch (error) { + const message = failureMessage(model, error); + output.push({ type: "error", reason: "error", error: message }); + output.end(message); + } +} + +function normalizeEvent( + event: AssistantMessageEvent, + members: Map, +): void { + if ("partial" in event) normalizeMessage(event.partial, members); + if (event.type === "toolcall_end") normalizeToolCall(event.toolCall, members); + if (event.type === "done") normalizeMessage(event.message, members); + if (event.type === "error") normalizeMessage(event.error, members); +} + +function normalizeMessage( + message: AssistantMessage, + members: Map, +): void { + for (const block of message.content) if (block.type === "toolCall") normalizeToolCall(block, members); +} + +function normalizeToolCall( + call: ToolCall, + members: Map, +): void { + const decoded = decodeMemberName(call.name) ?? members.get(call.id); + if (!decoded) return; + members.set(call.id, decoded); + call.name = decoded.toolset; + call.arguments = { action: decoded.member, ...call.arguments }; +} + +function terminalMessage(event: AssistantMessageEvent): AssistantMessage | undefined { + if (event.type === "done") return event.message; + if (event.type === "error") return event.error; + return undefined; +} + +function createToolsetFetch( + base: FetchFunction, + selected: ReadonlySet, + states: ReadonlyMap, +): FetchFunction { + return async (input, init) => { + let nextInit = init; + if (typeof init?.body === "string") { + const payload: unknown = JSON.parse(init.body); + const rewritten = rewriteAnthropicToolsetPayload(payload, selected, states); + nextInit = { ...init, body: JSON.stringify(rewritten) }; + } + const response = await base(input, nextInit); + if (!response.body || !response.headers.get("content-type")?.includes("text/event-stream")) return response; + return rewriteSseResponse(response, selected); + }; +} + +/** @internal Exported for protocol-focused tests. */ +export function rewriteAnthropicToolsetPayload( + payload: unknown, + selected: ReadonlySet, + states: ReadonlyMap = new Map(), +): unknown { + if (!isRecord(payload) || !Array.isArray(payload.messages)) return payload; + const calls = new Map(); + const messages = payload.messages.map((message) => { + if (!isRecord(message) || !Array.isArray(message.content)) return message; + if (message.role === "assistant") { + return { + ...message, + content: message.content.map((block) => { + if (!isRecord(block) || block.type !== "tool_use" || typeof block.id !== "string") return block; + if (!isToolsetName(block.name) || !selected.has(block.name) || !isRecord(block.input)) return block; + const member = block.input.action; + if (typeof member !== "string" || !member) return block; + const { action: _action, ...input } = block.input; + calls.set(block.id, { toolset: block.name, member }); + return { ...block, name: member, toolset_name: block.name, input }; + }), + }; + } + if (message.role === "user") { + return { + ...message, + content: message.content.map((block) => { + if (!isRecord(block) || block.type !== "tool_result" || typeof block.tool_use_id !== "string") return block; + const call = calls.get(block.tool_use_id); + if (!call) return block; + const needsState = call.toolset === "browser" && TAB_MEMBERS.has(call.member) && block.is_error !== true; + const state = needsState ? states.get(block.tool_use_id) : undefined; + if (needsState && !state) { + throw new Error(`successful Anthropic browser member "${call.member}" is missing browser_state execution details`); + } + return { + ...block, + toolset_name: call.toolset, + ...(state ? { content: [{ type: "browser_state", ...state }] } : {}), + }; + }), + }; + } + return message; + }); + return { ...payload, messages }; +} + +function rewriteSseResponse(response: Response, selected: ReadonlySet): Response { + const body = response.body; + if (!body) return response; + const decoder = new TextDecoder(); + const encoder = new TextEncoder(); + let buffered = ""; + const transform = new TransformStream({ + transform(chunk, controller) { + buffered += decoder.decode(chunk, { stream: true }); + let newline = buffered.indexOf("\n"); + while (newline >= 0) { + const line = buffered.slice(0, newline); + buffered = buffered.slice(newline + 1); + controller.enqueue(encoder.encode(`${rewriteAnthropicToolsetSseLine(line, selected)}\n`)); + newline = buffered.indexOf("\n"); + } + }, + flush(controller) { + buffered += decoder.decode(); + if (buffered) controller.enqueue(encoder.encode(rewriteAnthropicToolsetSseLine(buffered, selected))); + }, + }); + const headers = new Headers(response.headers); + headers.delete("content-length"); + return new Response(body.pipeThrough(transform), { + status: response.status, + statusText: response.statusText, + headers, + }); +} + +/** @internal Exported for protocol-focused tests. */ +export function rewriteAnthropicToolsetSseLine( + line: string, + selected: ReadonlySet, +): string { + const carriageReturn = line.endsWith("\r") ? "\r" : ""; + const source = carriageReturn ? line.slice(0, -1) : line; + if (!source.startsWith("data:")) return line; + const separator = source.startsWith("data: ") ? "data: " : "data:"; + try { + const event: unknown = JSON.parse(source.slice(separator.length)); + if (!isRecord(event) || event.type !== "content_block_start" || !isRecord(event.content_block)) return line; + const block = event.content_block; + if (block.type !== "tool_use" || !isToolsetName(block.toolset_name) || !selected.has(block.toolset_name)) return line; + if (typeof block.name !== "string") return line; + return `${separator}${JSON.stringify({ + ...event, + content_block: { ...block, name: encodeMemberName(block.toolset_name, block.name) }, + })}${carriageReturn}`; + } catch { + return line; + } +} + +function browserStates(context: Context): Map { + const result = new Map(); + for (const message of context.messages) { + if (message.role !== "toolResult" || !isRecord(message.details)) continue; + const reads = message.details.readResults; + if (!Array.isArray(reads)) continue; + const state = reads.find((read) => isRecord(read) && read.type === "browser_state" && isRecord(read.state)); + if (isRecord(state) && isBrowserState(state.state)) result.set(message.toolCallId, state.state); + } + return result; +} + +function isBrowserState(value: unknown): value is BrowserState { + return isRecord(value) && Array.isArray(value.tabs); +} + +function encodeMemberName(toolset: AnthropicToolsetName, member: string): string { + return `${ENCODED_MEMBER_PREFIX}${toolset}__${member}`; +} + +function decodeMemberName(name: string): { toolset: AnthropicToolsetName; member: string } | undefined { + if (!name.startsWith(ENCODED_MEMBER_PREFIX)) return undefined; + const value = name.slice(ENCODED_MEMBER_PREFIX.length); + const separator = value.indexOf("__"); + if (separator <= 0) return undefined; + const toolset = value.slice(0, separator); + const member = value.slice(separator + 2); + return isToolsetName(toolset) && member ? { toolset, member } : undefined; +} + +function isToolsetName(value: unknown): value is AnthropicToolsetName { + return value === "computer" || value === "browser"; +} + +function failureMessage(model: Model, error: unknown): AssistantMessage { + return { + role: "assistant", + content: [], + api: model.api, + provider: model.provider, + model: model.id, + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "error", + errorMessage: error instanceof Error ? error.message : String(error), + timestamp: Date.now(), + }; +} + +function isRecord(value: unknown): value is Record { + return Boolean(value && typeof value === "object" && !Array.isArray(value)); +} diff --git a/packages/browser-loop/test/anthropic-browser-fallback.test.ts b/packages/browser-loop/test/anthropic-browser-fallback.test.ts deleted file mode 100644 index e84ff796..00000000 --- a/packages/browser-loop/test/anthropic-browser-fallback.test.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { - createAssistantMessageEventStream, - type Api, - type AssistantMessage, - type Context, - type Model, - type Provider, - type SimpleStreamOptions, -} from "@earendil-works/pi-ai"; -import { describe, expect, it } from "vitest"; -import { getLoopModel, type LoopSimpleStreamOptions } from "../src/pi/index"; -import { compileLoopToolCatalog, loop } from "../src/index"; -import { withAnthropicBrowserFallback } from "../src/pi/providers/anthropic/browser-fallback"; - -const context: Context = { - systemPrompt: "", - messages: [{ role: "user", content: [{ type: "text", text: "Use the browser." }], timestamp: 1 }], - tools: [], -}; - -describe("Anthropic native browser access fallback", () => { - it("retries with the equivalent function tool and remembers the credential", async () => { - const model = getLoopModel("anthropic:claude-opus-5"); - const catalog = compileLoopToolCatalog({ - model, - requestedTools: [loop.providers.anthropic.tools.browser()], - }); - const payloads: Array<{ tools: unknown[]; headers: SimpleStreamOptions["headers"] }> = []; - let calls = 0; - const base = fakeProvider(async (selectedModel, options) => { - calls += 1; - const raw = { tools: [{ name: "browser", input_schema: {} }] }; - const payload = (await options?.onPayload?.(raw, selectedModel) ?? raw) as { tools: unknown[] }; - payloads.push({ tools: payload.tools, headers: options?.headers }); - return calls === 1 - ? message(selectedModel, "error", "API key does not have access to browser_20260701") - : message(selectedModel, "toolUse"); - }); - const provider = withAnthropicBrowserFallback(base); - const options: LoopSimpleStreamOptions = { - apiKey: "fallback-test-key", - headers: catalog.headers.merge({ "anthropic-beta": "other-beta" }), - loopIncomingToolPlan: catalog.incoming, - onPayload: (payload) => catalog.payload.apply(payload, model), - }; - - await expect(provider.streamSimple(model, { ...context, tools: [...catalog.toolDeclarations] }, options).result()).resolves.toMatchObject({ stopReason: "toolUse" }); - expect(calls).toBe(2); - expect(payloads[0]?.tools[0]).toMatchObject({ type: "browser_20260701" }); - expect(payloads[0]?.headers).toMatchObject({ "anthropic-beta": "other-beta,browser-use-2026-07-01" }); - expect(payloads[1]?.tools[0]).toMatchObject({ name: "browser", input_schema: { anyOf: expect.any(Array) } }); - expect(payloads[1]?.tools[0]).not.toHaveProperty("type"); - expect(payloads[1]?.headers).toEqual({ "anthropic-beta": "other-beta" }); - - await provider.streamSimple(model, { ...context, tools: [...catalog.toolDeclarations] }, options).result(); - expect(calls).toBe(3); - expect(payloads[2]?.tools[0]).not.toHaveProperty("type"); - }); - - it("does not hide unrelated provider errors", async () => { - const model = getLoopModel("anthropic:claude-opus-5"); - const catalog = compileLoopToolCatalog({ - model, - requestedTools: [loop.providers.anthropic.tools.browser()], - }); - let calls = 0; - const provider = withAnthropicBrowserFallback(fakeProvider(async (selectedModel) => { - calls += 1; - return message(selectedModel, "error", "rate limit exceeded"); - })); - - const result = await provider.streamSimple(model, { ...context, tools: [...catalog.toolDeclarations] }, { - apiKey: "non-access-error-test-key", - loopIncomingToolPlan: catalog.incoming, - }).result(); - - expect(calls).toBe(1); - expect(result).toMatchObject({ stopReason: "error", errorMessage: "rate limit exceeded" }); - }); -}); - -function fakeProvider( - respond: (model: Model, options?: SimpleStreamOptions) => Promise, -): Provider { - const streamSimple = (model: Model, _context: Context, options?: SimpleStreamOptions) => { - const stream = createAssistantMessageEventStream(); - void (async () => { - const response = await respond(model, options); - if (response.stopReason === "error") stream.push({ type: "error", reason: "error", error: response }); - else { - stream.push({ type: "start", partial: response }); - stream.push({ type: "done", reason: response.stopReason, message: response }); - } - stream.end(response); - })(); - return stream; - }; - return { - id: "anthropic", - name: "Anthropic", - stream: streamSimple, - streamSimple, - } as unknown as Provider; -} - -function message(model: Model, stopReason: AssistantMessage["stopReason"], errorMessage?: string): AssistantMessage { - return { - role: "assistant", - content: [], - api: model.api, - provider: model.provider, - model: model.id, - usage: { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - totalTokens: 0, - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, - }, - stopReason, - ...(errorMessage ? { errorMessage } : {}), - timestamp: 1, - }; -} diff --git a/packages/browser-loop/test/anthropic-native.integration.test.ts b/packages/browser-loop/test/anthropic-native.integration.test.ts index 3337db82..3c77ed24 100644 --- a/packages/browser-loop/test/anthropic-native.integration.test.ts +++ b/packages/browser-loop/test/anthropic-native.integration.test.ts @@ -8,19 +8,19 @@ const liveIt = apiKey ? it : it.skip; const cases = [ { name: "computer", - tool: loop.providers.anthropic.tools.computer({ version: "20260701", enableZoom: true }), + tool: loop.providers.anthropic.tools.computer({ version: "20260801", zoom: true }), prompt: "Use the computer tool to take one screenshot.", expectedAction: "screenshot", }, { name: "browser", - tool: loop.providers.anthropic.tools.browser({ version: "20260701", javascript: true }), + tool: loop.providers.anthropic.tools.browser({ version: "20260801", javascript: true }), prompt: "Use the browser tool to navigate to example.com.", expectedAction: "navigate", }, ] as const; -describe("Anthropic early-access native tools", () => { +describe("Anthropic GA client toolsets", () => { for (const current of cases) { liveIt(`${current.name} survives catalog and pi-ai serialization`, async () => { const catalog = compileLoopToolCatalog({ diff --git a/packages/browser-loop/test/anthropic-toolsets.test.ts b/packages/browser-loop/test/anthropic-toolsets.test.ts new file mode 100644 index 00000000..7a7d2b7a --- /dev/null +++ b/packages/browser-loop/test/anthropic-toolsets.test.ts @@ -0,0 +1,139 @@ +import { describe, expect, it } from "vitest"; +import { loop } from "../src/index"; +import { + rewriteAnthropicToolsetPayload, + rewriteAnthropicToolsetSseLine, +} from "../src/pi/providers/anthropic/toolsets"; + +const both = new Set(["computer", "browser"] as const); + +describe("Anthropic GA client toolsets", () => { + it("declares the released toolsets without preview fields or beta requirements", () => { + expect(loop.providers.anthropic.tools.computer({ version: "20260801", zoom: false }).providerBinding).toEqual({ + kind: "anthropic-native", + toolsetName: "computer", + declaration: { + type: "computer_toolset_20260801", + configs: { zoom: { enabled: false } }, + }, + }); + expect(loop.providers.anthropic.tools.browser({ version: "20260801", javascript: true }).providerBinding).toEqual({ + kind: "anthropic-native", + toolsetName: "browser", + declaration: { + type: "browser_toolset_20260801", + configs: { javascript_exec: { enabled: true } }, + }, + }); + }); + + it("maps the GA browser members added after the preview", () => { + const tool = loop.providers.anthropic.tools.browser(); + if (tool.execution.kind !== "actions") throw new Error("expected action tool"); + expect(tool.execution.toActions({ action: "middle_click", target: { type: "coordinate", x: 1, y: 2 } })).toEqual([ + { type: "browser_click", x: 1, y: 2, button: "middle" }, + ]); + expect(tool.execution.toActions({ action: "left_mouse_down", target: { type: "coordinate", x: 3, y: 4 } })).toEqual([ + { type: "browser_mouse_down", x: 3, y: 4 }, + ]); + expect(tool.execution.toActions({ action: "hold_key", text: "shift", duration: 2 })).toEqual([ + { type: "browser_hold_key", text: "shift", duration: 2 }, + ]); + expect(tool.execution.toActions({ action: "switch_tab", tab_id: "TAB1" })).toEqual([ + { type: "browser_switch_tab", tab_id: "TAB1" }, + ]); + expect(tool.execution.toActions({ action: "close_tab", tab_id: "TAB1" })).toEqual([ + { type: "browser_close_tab", tab_id: "TAB1" }, + ]); + }); + + it("rewrites pi function calls and results into member toolset blocks", () => { + const payload = { + messages: [ + { + role: "assistant", + content: [ + { type: "tool_use", id: "computer-1", name: "computer", input: { action: "left_click", coordinate: [10, 20] } }, + { type: "tool_use", id: "browser-1", name: "browser", input: { action: "navigate", url: "https://example.com" } }, + ], + }, + { + role: "user", + content: [ + { type: "tool_result", tool_use_id: "computer-1", content: "OK", is_error: false }, + { type: "tool_result", tool_use_id: "browser-1", content: "OK", is_error: false }, + ], + }, + ], + }; + + expect(rewriteAnthropicToolsetPayload(payload, both)).toEqual({ + messages: [ + { + role: "assistant", + content: [ + { type: "tool_use", id: "computer-1", name: "left_click", toolset_name: "computer", input: { coordinate: [10, 20] } }, + { type: "tool_use", id: "browser-1", name: "navigate", toolset_name: "browser", input: { url: "https://example.com" } }, + ], + }, + { + role: "user", + content: [ + { type: "tool_result", tool_use_id: "computer-1", toolset_name: "computer", content: "OK", is_error: false }, + { type: "tool_result", tool_use_id: "browser-1", toolset_name: "browser", content: "OK", is_error: false }, + ], + }, + ], + }); + }); + + it("emits browser_state for successful tab-management results", () => { + const state = { + tabs: [{ tab_id: "TAB1", title: "", url: "about:blank", active: true as const }], + state_changes: [{ type: "tab_opened" as const, tab_id: "TAB1" }], + }; + const payload = { + messages: [ + { role: "assistant", content: [{ type: "tool_use", id: "tab-1", name: "browser", input: { action: "new_tab" } }] }, + { role: "user", content: [{ type: "tool_result", tool_use_id: "tab-1", content: "Available tabs", is_error: false }] }, + ], + }; + const rewritten = rewriteAnthropicToolsetPayload(payload, both, new Map([["tab-1", state]])) as typeof payload; + expect(rewritten.messages[1]!.content[0]).toEqual({ + type: "tool_result", + tool_use_id: "tab-1", + toolset_name: "browser", + content: [{ type: "browser_state", ...state }], + is_error: false, + }); + }); + + it("rejects a successful tab result without structured browser state", () => { + const payload = { + messages: [ + { role: "assistant", content: [{ type: "tool_use", id: "tab-1", name: "browser", input: { action: "list_tabs" } }] }, + { role: "user", content: [{ type: "tool_result", tool_use_id: "tab-1", content: "Available tabs", is_error: false }] }, + ], + }; + expect(() => rewriteAnthropicToolsetPayload(payload, both)).toThrow(/missing browser_state execution details/); + }); + + it("marks streamed member calls without touching ordinary tools", () => { + const member = rewriteAnthropicToolsetSseLine( + `data: ${JSON.stringify({ + type: "content_block_start", + index: 0, + content_block: { type: "tool_use", id: "call-1", name: "screenshot", input: {}, toolset_name: "computer" }, + })}`, + both, + ); + expect(JSON.parse(member.slice("data: ".length)).content_block.name).toBe("loop_anthropic_toolset_computer__screenshot"); + + const ordinary = `data: ${JSON.stringify({ + type: "content_block_start", + index: 1, + content_block: { type: "tool_use", id: "call-2", name: "custom", input: {} }, + })}`; + expect(rewriteAnthropicToolsetSseLine(ordinary, both)).toBe(ordinary); + }); +}); diff --git a/packages/browser-loop/test/e2e.live.test.ts b/packages/browser-loop/test/e2e.live.test.ts index 6e7cccc2..5092decd 100644 --- a/packages/browser-loop/test/e2e.live.test.ts +++ b/packages/browser-loop/test/e2e.live.test.ts @@ -151,7 +151,7 @@ function toolsForCase(c: ProviderCase) { // Moonshot's API rejects `browser_act`'s schema; Kimi runs primitives only. return loop.toolsets.browser(); case "anthropic": - return [loop.providers.anthropic.tools.browser({ version: "20260701", javascript: true })]; + return [loop.providers.anthropic.tools.browser({ version: "20260801", javascript: true })]; case "gemini": return loop.providers.google.toolsets.browser(); } diff --git a/packages/browser-loop/test/extension.test.ts b/packages/browser-loop/test/extension.test.ts index c8636045..02f7806a 100644 --- a/packages/browser-loop/test/extension.test.ts +++ b/packages/browser-loop/test/extension.test.ts @@ -115,7 +115,7 @@ describe("pi extension activation", () => { expect(() => getHandler(pi, "session_start")({}, ctx)).toThrow('unknown browser tool selector "nope"'); }); - it("registers the Loop Anthropic provider and serializes native computer use", async () => { + it("registers the Loop Anthropic provider and serializes the GA computer toolset", async () => { const pi = makePi({ "browser-tools": "anthropic-computer", "browser-coordinates": "pixels", @@ -127,11 +127,11 @@ describe("pi extension activation", () => { const headers: Record = {}; await getHandler(pi, "before_provider_headers")({ headers }, anthropicCtx); - expect(headers["anthropic-beta"]).toContain("computer-use-2026-07-01"); + expect(headers).toEqual({}); const payload = { tools: [{ name: "computer", input_schema: { type: "object" } }] }; const transformed = await getHandler(pi, "before_provider_request")({ payload }, anthropicCtx); - expect(transformed).toEqual({ tools: [expect.objectContaining({ name: "computer", type: "computer_20260701" })] }); + expect(transformed).toEqual({ tools: [{ type: "computer_toolset_20260801", configs: { zoom: { enabled: true } } }] }); }); it("keeps the browser out of the request path, and blocks execution after shutdown", async () => { diff --git a/packages/browser-loop/test/models.test.ts b/packages/browser-loop/test/models.test.ts index 8f530b55..72f6bbc9 100644 --- a/packages/browser-loop/test/models.test.ts +++ b/packages/browser-loop/test/models.test.ts @@ -161,6 +161,8 @@ describe("native surfaces", () => { expect(computerUseNativeSurfaces(getLoopModel("openai:gpt-5.4-mini"))).toEqual(["computer"]); expect(computerUseNativeSurfaces(getLoopModel("anthropic:claude-opus-5"))).toEqual(["computer", "browser"]); expect(computerUseNativeSurfaces(getLoopModel("anthropic:claude-opus-5-20260724"))).toEqual(["computer", "browser"]); + expect(computerUseNativeSurfaces(getLoopModel("anthropic:claude-fable-5"))).toEqual(["computer", "browser"]); + expect(computerUseNativeSurfaces(getLoopModel("anthropic:claude-mythos-5"))).toEqual(["computer", "browser"]); }); it("does not match adjacent families or named sibling variants", () => { diff --git a/packages/browser-loop/test/selection.test.ts b/packages/browser-loop/test/selection.test.ts index 420a7ee7..ca701e6d 100644 --- a/packages/browser-loop/test/selection.test.ts +++ b/packages/browser-loop/test/selection.test.ts @@ -78,7 +78,8 @@ describe("Loop pi selectors", () => { const catalog = compileSpecs(getLoopModel("anthropic:claude-fable-5"), specs); expect(specs.map((tool) => tool.name)).toEqual(["computer"]); expect(catalog.entries.map((entry) => entry.transport)).toEqual(["native"]); - expect(catalog.headers.requirements).toContainEqual(expect.objectContaining({ value: "computer-use-2026-07-01" })); + expect(catalog.headers.requirements).toEqual([]); + expect(catalog.incoming.anthropicToolsets).toEqual(["computer"]); expect(() => compileSpecs(getLoopModel("openai:gpt-5.6-sol"), specs)).toThrow("requires a anthropic model"); }); @@ -101,7 +102,7 @@ describe("Loop pi selectors", () => { // And the incoming plan is what normalizes the calls that come back. expect(compileSpecs(getLoopModel("openai:gpt-5.6-sol"), openai).incoming.openaiComputerName).toBe("computer"); expect(compileSpecs(getLoopModel("anthropic:claude-opus-5"), expandSelection(parseSelection("anthropic-browser", "pixels"))).incoming - .anthropicBrowserFallback).toBeDefined(); + .anthropicToolsets).toEqual(["browser"]); }); it("reports selector availability per model with the compiler's own reason", () => { @@ -122,28 +123,25 @@ describe("Loop pi selectors", () => { expect(() => parseSelection("browser", "screen")).toThrow("coordinates"); }); - it("rejects Anthropic's two native surfaces together, before the API does", () => { - // Anthropic answers 400: the browser tool's viewport coordinate frame is - // incompatible with the computer tool's display frame. + it("combines Anthropic's GA computer and browser toolsets", () => { const both = expandSelection(parseSelection("anthropic-computer,anthropic-browser", "pixels")); - expect(() => compileSpecs(getLoopModel("anthropic:claude-opus-5"), both)).toThrow(/cannot be selected alongside/); + const catalog = compileSpecs(getLoopModel("anthropic:claude-opus-5"), both); + expect(catalog.incoming.anthropicToolsets).toEqual(["computer", "browser"]); }); - it("reports that pairwise conflict as a conflict, not as unavailability", () => { + it("reports no conflict between Anthropic's GA toolsets", () => { const byName = new Map( selectorAvailability(getLoopModel("anthropic:claude-opus-5"), parseSelection(undefined, "pixels")).map((e) => [e.selector, e]), ); expect(byName.get("anthropic-computer")?.available).toBe(true); - expect(byName.get("anthropic-computer")?.conflictsWith).toContain("anthropic-browser"); - expect(byName.get("anthropic-browser")?.conflictsWith).toContain("anthropic-computer"); + expect(byName.get("anthropic-computer")?.conflictsWith).not.toContain("anthropic-browser"); + expect(byName.get("anthropic-browser")?.conflictsWith).not.toContain("anthropic-computer"); expect(byName.get("playwright")?.conflictsWith).toEqual([]); }); - it("no longer marks every row unavailable when the current selection fails to compile", () => { - // The regression this replaces: a failing selection's error became the reason - // on every row, including rows that then activated fine. + it("does not mark every row unavailable when the current selection fails to compile", () => { const model = getLoopModel("anthropic:claude-opus-5"); - const failing = parseSelection("anthropic-computer,anthropic-browser", "pixels"); + const failing = parseSelection("anthropic-computer,openai-computer", "pixels"); expect(() => compileSpecs(model, expandSelection(failing))).toThrow(); const byName = new Map(selectorAvailability(model, failing).map((e) => [e.selector, e])); diff --git a/packages/browser-loop/test/tool-catalog.test.ts b/packages/browser-loop/test/tool-catalog.test.ts index 6b464679..0573e6df 100644 --- a/packages/browser-loop/test/tool-catalog.test.ts +++ b/packages/browser-loop/test/tool-catalog.test.ts @@ -67,10 +67,8 @@ describe("loop tool namespace", () => { expect("legacyBrowser" in loop.providers.google.toolsets).toBe(false); const surfaces: Array<[string, LoopToolSpec[]]> = [ [loop.providers.openai.source, [loop.providers.openai.tools.computer()]], - [loop.providers.anthropic.source, [ - loop.providers.anthropic.tools.browser(), - loop.providers.anthropic.tools.computer(), - ]], + [loop.providers.anthropic.sources.browser, [loop.providers.anthropic.tools.browser()]], + [loop.providers.anthropic.sources.computer, [loop.providers.anthropic.tools.computer()]], [loop.providers.google.source, loop.providers.google.toolsets.browser()], ]; for (const [source, tools] of surfaces) { @@ -220,28 +218,37 @@ describe("compileLoopToolCatalog", () => { expect(catalog.incoming.openaiComputerName).toBe("computer"); }); - it("composes Anthropic native browser declarations, access fallback, and ordinary functions", async () => { + it("composes Anthropic GA toolset declarations and ordinary functions", async () => { const catalog = compile("anthropic:claude-opus-5", [ loop.providers.anthropic.tools.browser(), loop.tools.browser.snapshot(), ]); - expect(catalog.headers.merge({ "anthropic-beta": "other-beta" })).toEqual({ - "anthropic-beta": "other-beta,browser-use-2026-07-01", - }); + expect(catalog.headers.merge({ "x-custom": "value" })).toEqual({ "x-custom": "value" }); const next = await catalog.payload.apply({ tools: [ { name: "browser", input_schema: {} }, { name: "browser_snapshot", input_schema: {} }, ] }, catalog.model) as { tools: Array> }; - expect(next.tools[0]).toMatchObject({ type: "browser_20260701", name: "browser" }); + expect(next.tools[0]).toEqual({ type: "browser_toolset_20260801" }); expect(next.tools[1]).toMatchObject({ name: "browser_snapshot" }); - expect(catalog.incoming.anthropicBrowserFallback).toMatchObject({ - beta: "browser-use-2026-07-01", - nativeType: "browser_20260701", - declaration: { name: "browser", input_schema: { anyOf: expect.any(Array) } }, - }); + expect(catalog.incoming.anthropicToolsets).toEqual(["browser"]); expect(catalog.entries[0]?.dynamicLoading).toBe("eager-only"); }); + it("allows Anthropic's GA computer and browser toolsets together", async () => { + const catalog = compile("anthropic:claude-opus-5", [ + loop.providers.anthropic.tools.computer(), + loop.providers.anthropic.tools.browser(), + ]); + expect(catalog.incoming.anthropicToolsets).toEqual(["computer", "browser"]); + expect(await catalog.payload.apply({ tools: [ + { name: "computer", input_schema: {} }, + { name: "browser", input_schema: {} }, + ] }, catalog.model)).toEqual({ tools: [ + { type: "computer_toolset_20260801" }, + { type: "browser_toolset_20260801" }, + ] }); + }); + it("serializes Google's current native declaration and keeps custom functions", async () => { const selected = loop.providers.google.toolsets.browser({ exclude: ["right_click", "triple_click"] }); const catalog = compile("google:gemini-3.6-flash", [...selected, callerTool("custom")]); diff --git a/packages/browser-loop/test/translator-browser.test.ts b/packages/browser-loop/test/translator-browser.test.ts index bff32117..50ff767c 100644 --- a/packages/browser-loop/test/translator-browser.test.ts +++ b/packages/browser-loop/test/translator-browser.test.ts @@ -695,7 +695,8 @@ function createFakeCdp(initialNodes: unknown[] = []) { case "Runtime.evaluate": if (params.returnByValue === false) return { result: { objectId: "cursor-scan" } }; return { result: { value: runtimeValue } }; - case "Page.navigate": { + case "Page.navigate": + case "Page.reload": { const nextLoaderId = `NAV-${++navigationSequence}`; if (autoNavigationLifecycle) { emit({ method: "Page.frameNavigated", params: { frame: { id: mainFrameId, loaderId: nextLoaderId } }, sessionId }); @@ -1094,6 +1095,25 @@ describe("BrowserExecutor navigation stabilization", () => { } }); + it("rejects non-http browser navigation schemes", async () => { + const fake = createFakeCdp(BUTTON_TREE); + const executor = new BrowserExecutor(fake.cdp); + + await expect(executor.execute({ type: "browser_navigate", url: "file:///tmp/secret" } as BrowserAction)) + .rejects.toThrow(/requires an http or https URL/); + expect(fake.sent.some((command) => command.method === "Page.navigate")).toBe(false); + }); + + it("reloads the active tab and waits for its fresh loader", async () => { + const fake = createFakeCdp(BUTTON_TREE); + const executor = new BrowserExecutor(fake.cdp); + + await expect(executor.execute({ type: "browser_navigate", url: "reload" } as BrowserAction)).resolves.toEqual([ + expect.objectContaining({ type: "browser_text", label: "navigate", text: expect.stringContaining("Reloaded page") }), + ]); + expect(fake.sent).toContainEqual(expect.objectContaining({ method: "Page.reload" })); + }); + it("completes a non-bfcache history navigation after its fresh loader emits load", async () => { const fake = createFakeCdp(BUTTON_TREE); fake.setHistoryNavigationMode("load"); From f04844a727494f2db206ed52ba5bdb9d2a316ec6 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 21 Aug 2026 12:17:31 +0000 Subject: [PATCH 2/7] Preserve active tab when closing background tabs --- .../src/core/translator/browser.ts | 10 ++++++--- .../test/translator-browser.test.ts | 22 +++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/packages/browser-loop/src/core/translator/browser.ts b/packages/browser-loop/src/core/translator/browser.ts index c90a882c..3cb044f3 100644 --- a/packages/browser-loop/src/core/translator/browser.ts +++ b/packages/browser-loop/src/core/translator/browser.ts @@ -1172,12 +1172,16 @@ export class BrowserExecutor { } private async closeTab(tabId: string): Promise { + const previousActiveTargetId = this.activeTargetId; const targetId = await this.resolveTarget(tabId); await this.cdp.send("Target.closeTarget", { targetId }); this.dropTarget(targetId); - const targets = await this.cdp.pageTargets(); - this.activeTargetId = targets.find((target) => target.targetId !== targetId)?.targetId; - if (this.activeTargetId) await this.cdp.send("Target.activateTarget", { targetId: this.activeTargetId }); + const targets = (await this.cdp.pageTargets()).filter((target) => target.targetId !== targetId); + const previousActiveStillOpen = targets.some((target) => target.targetId === previousActiveTargetId); + this.activeTargetId = previousActiveStillOpen ? previousActiveTargetId : targets[0]?.targetId; + if (!previousActiveStillOpen && this.activeTargetId) { + await this.cdp.send("Target.activateTarget", { targetId: this.activeTargetId }); + } return this.browserState(undefined, targetId); } diff --git a/packages/browser-loop/test/translator-browser.test.ts b/packages/browser-loop/test/translator-browser.test.ts index 50ff767c..ad08cc5b 100644 --- a/packages/browser-loop/test/translator-browser.test.ts +++ b/packages/browser-loop/test/translator-browser.test.ts @@ -1114,6 +1114,28 @@ describe("BrowserExecutor navigation stabilization", () => { expect(fake.sent).toContainEqual(expect.objectContaining({ method: "Page.reload" })); }); + it("keeps the active tab when closing a background tab", async () => { + const fake = createFakeCdp(BUTTON_TREE); + let closed = false; + fake.setTargetProvider(() => closed + ? [{ targetId: "TARGET-1", type: "page", title: "First", url: "https://a.test/" }] + : [ + { targetId: "TARGET-1", type: "page", title: "First", url: "https://a.test/" }, + { targetId: "TARGET-2", type: "page", title: "Second", url: "https://b.test/" }, + ]); + fake.setSendHook((method) => { if (method === "Target.closeTarget") closed = true; }); + const executor = new BrowserExecutor(fake.cdp); + await executor.execute({ type: "browser_list_tabs" }); + + await expect(executor.execute({ type: "browser_close_tab", tab_id: "TARGET-2" })).resolves.toEqual([ + { + type: "browser_state", + state: { tabs: [{ tab_id: "TARGET-1", title: "First", url: "https://a.test/", active: true }] }, + }, + ]); + expect(fake.sent.filter((command) => command.method === "Target.activateTarget")).toEqual([]); + }); + it("completes a non-bfcache history navigation after its fresh loader emits load", async () => { const fake = createFakeCdp(BUTTON_TREE); fake.setHistoryNavigationMode("load"); From 0a5918601e7054018bfa45eeca1ebb6c27558459 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 21 Aug 2026 13:16:28 +0000 Subject: [PATCH 3/7] Harden Anthropic toolset state handling --- docs/architecture.md | 6 +-- packages/browser-loop/CHANGELOG.md | 10 ++--- packages/browser-loop/README.md | 14 +++--- .../browser-loop/docs/supported-models.md | 2 +- .../browser-loop/src/core/anthropic-native.ts | 4 +- .../browser-loop/src/core/tool-catalog.ts | 2 +- .../src/core/translator/browser.ts | 33 +++++++++----- packages/browser-loop/src/pi/attach.ts | 14 ++++-- packages/browser-loop/src/pi/providers.ts | 2 +- .../pi/providers/anthropic/capabilities.ts | 4 +- .../src/pi/providers/anthropic/toolsets.ts | 12 ++++- .../test/anthropic-native.integration.test.ts | 2 +- .../test/anthropic-toolsets.test.ts | 17 ++++--- packages/browser-loop/test/attach.test.ts | 22 +++++++++- packages/browser-loop/test/extension.test.ts | 2 +- packages/browser-loop/test/selection.test.ts | 4 +- .../browser-loop/test/tool-catalog.test.ts | 4 +- .../test/translator-browser.test.ts | 44 +++++++++++++++++++ 18 files changed, 147 insertions(+), 51 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 8b25842c..95511305 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -178,9 +178,9 @@ Catalog compilation composes provider behavior rather than replacing the whole catalog: - Ordinary function tools stay ordinary. -- Anthropic's GA browser/computer client-toolset declarations replace only - their own placeholders; their member calls and results retain `toolset_name` - through pi's function-tool transcript without beta headers. +- Anthropic's browser/computer client-toolset declarations replace only their + own placeholders; their member calls and results retain `toolset_name` + through pi's function-tool transcript. - OpenAI streams through pi's builtin Responses transport and its automatic prompt caching by default; a Browser Loop-owned adapter handles OpenAI's native computer tool and tool-search namespace round-trips. diff --git a/packages/browser-loop/CHANGELOG.md b/packages/browser-loop/CHANGELOG.md index 4e2ed668..cc863e0a 100644 --- a/packages/browser-loop/CHANGELOG.md +++ b/packages/browser-loop/CHANGELOG.md @@ -2,12 +2,10 @@ ## Unreleased -- Replace Anthropic's early-access `computer_20260701` and `browser_20260701` - integrations with the generally available `computer_toolset_20260801` and - `browser_toolset_20260801` client toolsets. Requests no longer send beta - headers or use the browser access fallback; member calls/results preserve - `toolset_name`, sequential batches stop at the first failure, and both - toolsets may be selected together. +- Update Anthropic's computer and browser integrations to use the + `computer_toolset_20260801` and `browser_toolset_20260801` client toolsets. + Member calls/results preserve `toolset_name`, sequential batches stop at the + first failure, and both toolsets may be selected together. - Rename the product and unpublished package to Browser Loop: the repository is prepared for `kernel/browser-loop`, the package is `@onkernel/browser-loop`, and release tags use `browser-loop/v*`. The public `loop` namespace, `Loop*` diff --git a/packages/browser-loop/README.md b/packages/browser-loop/README.md index 7c483bd5..eef06024 100644 --- a/packages/browser-loop/README.md +++ b/packages/browser-loop/README.md @@ -357,10 +357,10 @@ than on the wire. [Models and native surfaces](docs/supported-models.md) lists which models carry which surface. Provider-native caller-visible names are fixed by protocol. Version/tool/model -mismatches fail during catalog compilation. Anthropic's GA -`computer_toolset_20260801` and `browser_toolset_20260801` declarations need no -beta header and may be selected together; Browser Loop preserves member -`toolset_name` fields and sequential batch semantics across pi's transcript. +mismatches fail during catalog compilation. Anthropic's +`computer_toolset_20260801` and `browser_toolset_20260801` declarations may be +selected together; Browser Loop preserves member `toolset_name` fields and +sequential batch semantics across pi's transcript. Every `loop.providers.*` tool surface exposes its first-party `source` or `sources`, and every returned provider spec carries the applicable URL. @@ -435,9 +435,9 @@ require different transports fails to compile. `openai-computer-use` api instead, which a Browser Loop adapter handles; that same adapter also covers tool-search namespace round-trips regardless of api, since pi's builtin transport does not replay them. -- **Anthropic**: GA client-toolset declarations, member/result transcript - adaptation, and adaptive model preparation. No beta header or api fork — - every Anthropic model streams through pi's builtin transport. +- **Anthropic**: client-toolset declarations, member/result transcript + adaptation, and adaptive model preparation. Every Anthropic model streams + through pi's builtin transport. - **Google**: a model selected without Google's native browser toolset streams through pi's builtin transport. Selecting `loop.providers.google.toolsets.browser()` derives the Browser Loop-owned diff --git a/packages/browser-loop/docs/supported-models.md b/packages/browser-loop/docs/supported-models.md index e34189c9..69728992 100644 --- a/packages/browser-loop/docs/supported-models.md +++ b/packages/browser-loop/docs/supported-models.md @@ -92,7 +92,7 @@ npx tsx packages/browser-loop/scripts/native-action-probe.ts --provider openai - Update that provider's adapter under `src/pi/providers/` to execute the actions the probe returns, then add or adjust the `COMPUTER_USE_NATIVE_SURFACES` entry, citing the -provider's documentation. Anthropic's GA client toolsets are adapted in +provider's documentation. Anthropic's client toolsets are adapted in `src/pi/providers/anthropic/toolsets.ts`; a new dated toolset version requires updating its declarations and member transcript adapter together. diff --git a/packages/browser-loop/src/core/anthropic-native.ts b/packages/browser-loop/src/core/anthropic-native.ts index 1fda1bb9..bf8280de 100644 --- a/packages/browser-loop/src/core/anthropic-native.ts +++ b/packages/browser-loop/src/core/anthropic-native.ts @@ -14,7 +14,7 @@ function asNativeInput(args: unknown): NativeInput { const MAX_KEY_REPEAT = 100; -/** Map one GA computer toolset member onto canonical computer-plane actions. */ +/** Map one Anthropic computer toolset member onto canonical computer-plane actions. */ export function mapNativeComputerInput(input: NativeInput): ComputerUseAction[] { switch (input.action) { case "screenshot": @@ -64,7 +64,7 @@ export function mapNativeComputerInput(input: NativeInput): ComputerUseAction[] } } -/** Map one GA browser toolset member onto canonical browser-plane actions. */ +/** Map one Anthropic browser toolset member onto canonical browser-plane actions. */ export function mapNativeBrowserInput(input: NativeInput): ComputerUseAction[] { const tab = tabId(input); switch (input.action) { diff --git a/packages/browser-loop/src/core/tool-catalog.ts b/packages/browser-loop/src/core/tool-catalog.ts index f420f0a4..929f1445 100644 --- a/packages/browser-loop/src/core/tool-catalog.ts +++ b/packages/browser-loop/src/core/tool-catalog.ts @@ -152,7 +152,7 @@ export interface LoopPayloadPlan { /** Identity-addressed native call dispatch passed to Loop custom provider streams. */ export interface LoopIncomingToolPlan { - /** GA Anthropic client toolsets selected for this request. */ + /** Anthropic client toolsets selected for this request. */ readonly anthropicToolsets: readonly ("computer" | "browser")[]; readonly openaiComputerName?: string; readonly googleNames: Readonly>; diff --git a/packages/browser-loop/src/core/translator/browser.ts b/packages/browser-loop/src/core/translator/browser.ts index 3cb044f3..9df153e1 100644 --- a/packages/browser-loop/src/core/translator/browser.ts +++ b/packages/browser-loop/src/core/translator/browser.ts @@ -1172,15 +1172,14 @@ export class BrowserExecutor { } private async closeTab(tabId: string): Promise { - const previousActiveTargetId = this.activeTargetId; const targetId = await this.resolveTarget(tabId); await this.cdp.send("Target.closeTarget", { targetId }); this.dropTarget(targetId); const targets = (await this.cdp.pageTargets()).filter((target) => target.targetId !== targetId); - const previousActiveStillOpen = targets.some((target) => target.targetId === previousActiveTargetId); - this.activeTargetId = previousActiveStillOpen ? previousActiveTargetId : targets[0]?.targetId; - if (!previousActiveStillOpen && this.activeTargetId) { - await this.cdp.send("Target.activateTarget", { targetId: this.activeTargetId }); + const activeStillOpen = targets.some((target) => target.targetId === this.activeTargetId); + if (!activeStillOpen) { + this.activeTargetId = targets[0]?.targetId; + if (this.activeTargetId) await this.cdp.send("Target.activateTarget", { targetId: this.activeTargetId }); } return this.browserState(undefined, targetId); } @@ -1193,14 +1192,19 @@ export class BrowserExecutor { if (targets.length > 0 && !targets.some((target) => target.targetId === this.activeTargetId)) { this.activeTargetId = targets[0]!.targetId; } + const visibleTargets = targets.slice(0, MAX_BROWSER_STATE_TABS); + const activeTarget = targets.find((target) => target.targetId === this.activeTargetId); + if (activeTarget && !visibleTargets.includes(activeTarget)) visibleTargets[visibleTargets.length - 1] = activeTarget; return { - tabs: targets.map((target) => ({ - tab_id: shortTabId(target.targetId), - title: target.title, - url: target.url, + tabs: visibleTargets.map((target) => ({ + tab_id: sanitizeBrowserStateValue(shortTabId(target.targetId)), + title: sanitizeBrowserStateValue(target.title), + url: sanitizeBrowserStateValue(target.url), ...(target.targetId === this.activeTargetId ? { active: true as const } : {}), })), - ...(stateChanges?.length ? { state_changes: stateChanges } : {}), + ...(stateChanges?.length ? { + state_changes: stateChanges.map((change) => ({ ...change, tab_id: sanitizeBrowserStateValue(change.tab_id) })), + } : {}), }; } @@ -1423,7 +1427,6 @@ export class BrowserExecutor { if (tabId) { const match = targets.find((target) => shortTabId(target.targetId) === tabId || target.targetId === tabId); if (!match) throw new Error(`unknown tab_id "${tabId}". Call list_tabs for current tabs.`); - this.activeTargetId = match.targetId; return match.targetId; } if (this.activeTargetId && targets.some((target) => target.targetId === this.activeTargetId)) { @@ -1489,6 +1492,14 @@ function normalizeState(value: unknown): boolean | "mixed" { return value === true || value === "true"; } +const MAX_BROWSER_STATE_TABS = 100; +const MAX_BROWSER_STATE_STRING_LENGTH = 4096; +const BROWSER_STATE_CONTROL_CHARACTERS = /[\u0000-\u001f\u007f-\u009f\u2028\u2029]/gu; + +function sanitizeBrowserStateValue(value: string): string { + return [...value.replace(BROWSER_STATE_CONTROL_CHARACTERS, "")].slice(0, MAX_BROWSER_STATE_STRING_LENGTH).join(""); +} + function shortTabId(targetId: string): string { return targetId.slice(0, 10).toUpperCase(); } diff --git a/packages/browser-loop/src/pi/attach.ts b/packages/browser-loop/src/pi/attach.ts index c2d3d89d..b77e5349 100644 --- a/packages/browser-loop/src/pi/attach.ts +++ b/packages/browser-loop/src/pi/attach.ts @@ -223,8 +223,11 @@ export function installLoopBehaviors( const offs: Array<() => void> = []; offs.push(harness.on("tool_result", (event: { details?: unknown }) => (hasExecutionError(event.details) ? { isError: true } : undefined))); - offs.push(harness.on("tool_call", () => - turnFailed && turnFailureStopMessage(manager) ? { block: true, reason: turnFailureStopMessage(manager) } : undefined)); + offs.push(harness.on("tool_call", (event: { toolName: string }) => { + if (!turnFailed) return undefined; + const reason = turnFailureStopMessage(manager, event.toolName); + return reason ? { block: true, reason } : undefined; + })); offs.push(harness.on("before_agent_start", () => { turnFailed = false; recoveryAttempts = 0; @@ -399,7 +402,12 @@ export function hasExecutionError(details: unknown): boolean { } /** @internal */ -export function turnFailureStopMessage(manager: LoopToolManager): string | undefined { +export function turnFailureStopMessage(manager: LoopToolManager, toolName?: string): string | undefined { + if (toolName) { + const entry = manager.catalog.entries.find((candidate) => candidate.name === toolName); + const execution = entry && manager.specFor(entry.identity)?.execution; + if (execution?.kind === "actions" && execution.stopTurnOnFailureMessage) return execution.stopTurnOnFailureMessage; + } for (const entry of manager.catalog.entries) { const execution = manager.specFor(entry.identity)?.execution; if (execution?.kind === "actions" && execution.stopTurnOnFailureMessage) return execution.stopTurnOnFailureMessage; diff --git a/packages/browser-loop/src/pi/providers.ts b/packages/browser-loop/src/pi/providers.ts index 55619496..4023f2f0 100644 --- a/packages/browser-loop/src/pi/providers.ts +++ b/packages/browser-loop/src/pi/providers.ts @@ -20,7 +20,7 @@ import { OPENAI_COMPUTER_USE_API, requiresOpenAINamespaceAdapter, streamOpenAICo * Build the pi `Models` collection Loop streams through: pi's builtin * providers, adjusted for Loop: * - * - `anthropic` adapts pi's function-tool transcript to the GA computer and + * - `anthropic` adapts pi's function-tool transcript to the computer and * browser client-toolset protocol, including member names and toolset results. * - `openai` streams through pi's builtin `openai-responses` transport and its * automatic prompt caching by default; a model compiled with OpenAI's native diff --git a/packages/browser-loop/src/pi/providers/anthropic/capabilities.ts b/packages/browser-loop/src/pi/providers/anthropic/capabilities.ts index 303c3e29..0bde667a 100644 --- a/packages/browser-loop/src/pi/providers/anthropic/capabilities.ts +++ b/packages/browser-loop/src/pi/providers/anthropic/capabilities.ts @@ -1,11 +1,11 @@ const NATIVE_TOOLSET_FAMILIES = ["claude-fable-5", "claude-mythos-5", "claude-opus-4-8", "claude-opus-5", "claude-sonnet-5"] as const; -/** Return whether an Anthropic model ID supports the GA browser toolset. */ +/** Return whether an Anthropic model ID supports the browser toolset. */ export function supportsAnthropicNativeBrowser(modelId: string): boolean { return NATIVE_TOOLSET_FAMILIES.some((family) => modelFamily(modelId, family)); } -/** Return whether an Anthropic model ID supports the GA computer toolset. */ +/** Return whether an Anthropic model ID supports the computer toolset. */ export function supportsAnthropicNativeComputer(modelId: string): boolean { return NATIVE_TOOLSET_FAMILIES.some((family) => modelFamily(modelId, family)); } diff --git a/packages/browser-loop/src/pi/providers/anthropic/toolsets.ts b/packages/browser-loop/src/pi/providers/anthropic/toolsets.ts index cc29fd63..8bdbbfff 100644 --- a/packages/browser-loop/src/pi/providers/anthropic/toolsets.ts +++ b/packages/browser-loop/src/pi/providers/anthropic/toolsets.ts @@ -22,7 +22,7 @@ type AnthropicToolsetName = "computer" | "browser"; const ENCODED_MEMBER_PREFIX = "loop_anthropic_toolset_"; const TAB_MEMBERS = new Set(["new_tab", "list_tabs", "switch_tab", "close_tab"]); -/** Adapt pi-ai's function-tool transcript to Anthropic's GA client-toolset protocol. */ +/** Adapt pi-ai's function-tool transcript to Anthropic's client-toolset protocol. */ export function withAnthropicToolsets(base: Provider): Provider { return { ...base, @@ -175,7 +175,15 @@ export function rewriteAnthropicToolsetPayload( const needsState = call.toolset === "browser" && TAB_MEMBERS.has(call.member) && block.is_error !== true; const state = needsState ? states.get(block.tool_use_id) : undefined; if (needsState && !state) { - throw new Error(`successful Anthropic browser member "${call.member}" is missing browser_state execution details`); + return { + ...block, + toolset_name: call.toolset, + is_error: true, + content: [{ + type: "text", + text: `Browser state is unavailable for this earlier ${call.member} result. Call list_tabs to refresh it.`, + }], + }; } return { ...block, diff --git a/packages/browser-loop/test/anthropic-native.integration.test.ts b/packages/browser-loop/test/anthropic-native.integration.test.ts index 3c77ed24..3b648984 100644 --- a/packages/browser-loop/test/anthropic-native.integration.test.ts +++ b/packages/browser-loop/test/anthropic-native.integration.test.ts @@ -20,7 +20,7 @@ const cases = [ }, ] as const; -describe("Anthropic GA client toolsets", () => { +describe("Anthropic client toolsets", () => { for (const current of cases) { liveIt(`${current.name} survives catalog and pi-ai serialization`, async () => { const catalog = compileLoopToolCatalog({ diff --git a/packages/browser-loop/test/anthropic-toolsets.test.ts b/packages/browser-loop/test/anthropic-toolsets.test.ts index 7a7d2b7a..6d14516e 100644 --- a/packages/browser-loop/test/anthropic-toolsets.test.ts +++ b/packages/browser-loop/test/anthropic-toolsets.test.ts @@ -7,8 +7,8 @@ import { const both = new Set(["computer", "browser"] as const); -describe("Anthropic GA client toolsets", () => { - it("declares the released toolsets without preview fields or beta requirements", () => { +describe("Anthropic client toolsets", () => { + it("declares the client toolsets with their documented fields", () => { expect(loop.providers.anthropic.tools.computer({ version: "20260801", zoom: false }).providerBinding).toEqual({ kind: "anthropic-native", toolsetName: "computer", @@ -27,7 +27,7 @@ describe("Anthropic GA client toolsets", () => { }); }); - it("maps the GA browser members added after the preview", () => { + it("maps the browser toolset members", () => { const tool = loop.providers.anthropic.tools.browser(); if (tool.execution.kind !== "actions") throw new Error("expected action tool"); expect(tool.execution.toActions({ action: "middle_click", target: { type: "coordinate", x: 1, y: 2 } })).toEqual([ @@ -108,14 +108,21 @@ describe("Anthropic GA client toolsets", () => { }); }); - it("rejects a successful tab result without structured browser state", () => { + it("keeps transcripts usable when an earlier tab result lacks structured browser state", () => { const payload = { messages: [ { role: "assistant", content: [{ type: "tool_use", id: "tab-1", name: "browser", input: { action: "list_tabs" } }] }, { role: "user", content: [{ type: "tool_result", tool_use_id: "tab-1", content: "Available tabs", is_error: false }] }, ], }; - expect(() => rewriteAnthropicToolsetPayload(payload, both)).toThrow(/missing browser_state execution details/); + const rewritten = rewriteAnthropicToolsetPayload(payload, both) as typeof payload; + expect(rewritten.messages[1]!.content[0]).toEqual({ + type: "tool_result", + tool_use_id: "tab-1", + toolset_name: "browser", + is_error: true, + content: [{ type: "text", text: "Browser state is unavailable for this earlier list_tabs result. Call list_tabs to refresh it." }], + }); }); it("marks streamed member calls without touching ordinary tools", () => { diff --git a/packages/browser-loop/test/attach.test.ts b/packages/browser-loop/test/attach.test.ts index f6d64d71..f1b47afa 100644 --- a/packages/browser-loop/test/attach.test.ts +++ b/packages/browser-loop/test/attach.test.ts @@ -15,7 +15,7 @@ import { } from "../src/pi/index"; import { type KernelBrowser, loop } from "../src/index"; import type Kernel from "@onkernel/sdk"; -import { installLoopBehaviors } from "../src/pi/attach"; +import { installLoopBehaviors, turnFailureStopMessage } from "../src/pi/attach"; const browser = { session_id: "browser_123", viewport: { width: 1440, height: 900 } } as KernelBrowser; const client = {} as Kernel; @@ -205,6 +205,26 @@ describe("attach", () => { expect(attempted).toEqual(["continue", "continue"]); }); + it("uses the blocked tool's own failure-stop message", () => { + const entries = [ + { identity: "computer", name: "computer" }, + { identity: "browser", name: "browser" }, + ]; + const messages = new Map([ + ["computer", "computer stopped"], + ["browser", "browser stopped"], + ]); + const manager = { + catalog: { entries }, + specFor: (identity: string) => ({ + execution: { kind: "actions", stopTurnOnFailureMessage: messages.get(identity) }, + }), + }; + + expect(turnFailureStopMessage(manager as never, "browser")).toBe("browser stopped"); + expect(turnFailureStopMessage(manager as never, "computer")).toBe("computer stopped"); + }); + it("disposes the shared execution pool once, not per compile", async () => { const handle = attach({ browser, client }); handle.compile({ model: "openai:gpt-5.5", tools: [] }); diff --git a/packages/browser-loop/test/extension.test.ts b/packages/browser-loop/test/extension.test.ts index 02f7806a..f4b07c5f 100644 --- a/packages/browser-loop/test/extension.test.ts +++ b/packages/browser-loop/test/extension.test.ts @@ -115,7 +115,7 @@ describe("pi extension activation", () => { expect(() => getHandler(pi, "session_start")({}, ctx)).toThrow('unknown browser tool selector "nope"'); }); - it("registers the Loop Anthropic provider and serializes the GA computer toolset", async () => { + it("registers the Loop Anthropic provider and serializes the computer toolset", async () => { const pi = makePi({ "browser-tools": "anthropic-computer", "browser-coordinates": "pixels", diff --git a/packages/browser-loop/test/selection.test.ts b/packages/browser-loop/test/selection.test.ts index ca701e6d..302f7008 100644 --- a/packages/browser-loop/test/selection.test.ts +++ b/packages/browser-loop/test/selection.test.ts @@ -123,13 +123,13 @@ describe("Loop pi selectors", () => { expect(() => parseSelection("browser", "screen")).toThrow("coordinates"); }); - it("combines Anthropic's GA computer and browser toolsets", () => { + it("combines Anthropic's computer and browser toolsets", () => { const both = expandSelection(parseSelection("anthropic-computer,anthropic-browser", "pixels")); const catalog = compileSpecs(getLoopModel("anthropic:claude-opus-5"), both); expect(catalog.incoming.anthropicToolsets).toEqual(["computer", "browser"]); }); - it("reports no conflict between Anthropic's GA toolsets", () => { + it("reports no conflict between Anthropic's toolsets", () => { const byName = new Map( selectorAvailability(getLoopModel("anthropic:claude-opus-5"), parseSelection(undefined, "pixels")).map((e) => [e.selector, e]), ); diff --git a/packages/browser-loop/test/tool-catalog.test.ts b/packages/browser-loop/test/tool-catalog.test.ts index 0573e6df..e64b7616 100644 --- a/packages/browser-loop/test/tool-catalog.test.ts +++ b/packages/browser-loop/test/tool-catalog.test.ts @@ -218,7 +218,7 @@ describe("compileLoopToolCatalog", () => { expect(catalog.incoming.openaiComputerName).toBe("computer"); }); - it("composes Anthropic GA toolset declarations and ordinary functions", async () => { + it("composes Anthropic toolset declarations and ordinary functions", async () => { const catalog = compile("anthropic:claude-opus-5", [ loop.providers.anthropic.tools.browser(), loop.tools.browser.snapshot(), @@ -234,7 +234,7 @@ describe("compileLoopToolCatalog", () => { expect(catalog.entries[0]?.dynamicLoading).toBe("eager-only"); }); - it("allows Anthropic's GA computer and browser toolsets together", async () => { + it("allows Anthropic's computer and browser toolsets together", async () => { const catalog = compile("anthropic:claude-opus-5", [ loop.providers.anthropic.tools.computer(), loop.providers.anthropic.tools.browser(), diff --git a/packages/browser-loop/test/translator-browser.test.ts b/packages/browser-loop/test/translator-browser.test.ts index ad08cc5b..65e548a5 100644 --- a/packages/browser-loop/test/translator-browser.test.ts +++ b/packages/browser-loop/test/translator-browser.test.ts @@ -1136,6 +1136,50 @@ describe("BrowserExecutor navigation stabilization", () => { expect(fake.sent.filter((command) => command.method === "Target.activateTarget")).toEqual([]); }); + it("does not report a targeted background action as a tab switch", async () => { + const fake = createFakeCdp(BUTTON_TREE); + fake.setTargetProvider(() => [ + { targetId: "TARGET-1", type: "page", title: "First", url: "https://a.test/" }, + { targetId: "TARGET-2", type: "page", title: "Second", url: "https://b.test/" }, + ]); + const executor = new BrowserExecutor(fake.cdp); + await executor.execute({ type: "browser_list_tabs" }); + await executor.execute({ type: "browser_text", tab_id: "TARGET-2" } as BrowserAction); + + await expect(executor.execute({ type: "browser_list_tabs" })).resolves.toEqual([ + expect.objectContaining({ + type: "browser_state", + state: { tabs: [ + { tab_id: "TARGET-1", title: "First", url: "https://a.test/", active: true }, + { tab_id: "TARGET-2", title: "Second", url: "https://b.test/" }, + ] }, + }), + ]); + }); + + it("sanitizes and caps browser state while retaining the active tab", async () => { + const fake = createFakeCdp(BUTTON_TREE); + const targets = Array.from({ length: 101 }, (_, index) => ({ + targetId: `TARGET-${String(index + 1).padStart(4, "0")}`, + type: "page", + title: index === 100 ? `Bad\n\u2028${"x".repeat(4100)}` : `Tab ${index + 1}`, + url: index === 100 ? `https://example.test/${"y".repeat(4100)}\u0000` : `https://example.test/${index + 1}`, + })); + fake.setTargetProvider(() => targets); + const executor = new BrowserExecutor(fake.cdp); + + const [result] = await executor.execute({ type: "browser_switch_tab", tab_id: "TARGET-0101" }); + expect(result.type).toBe("browser_state"); + if (result.type !== "browser_state") throw new Error("expected browser_state"); + expect(result.state.tabs).toHaveLength(100); + const active = result.state.tabs.find((tab) => tab.active); + expect(active?.tab_id).toBe("TARGET-010"); + expect(active?.title).toHaveLength(4096); + expect(active?.title).not.toMatch(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/u); + expect(active?.url).toHaveLength(4096); + expect(active?.url).not.toMatch(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/u); + }); + it("completes a non-bfcache history navigation after its fresh loader emits load", async () => { const fake = createFakeCdp(BUTTON_TREE); fake.setHistoryNavigationMode("load"); From e5d72ca9348e08da3288cebc6c503ec39f74e948 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 21 Aug 2026 14:04:45 +0000 Subject: [PATCH 4/7] Accept Anthropic bearer token credentials --- packages/browser-loop/README.md | 6 +++--- packages/browser-loop/src/pi/api-keys.ts | 2 +- packages/browser-loop/test/api-keys.test.ts | 7 +++++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/packages/browser-loop/README.md b/packages/browser-loop/README.md index eef06024..fd1f8b08 100644 --- a/packages/browser-loop/README.md +++ b/packages/browser-loop/README.md @@ -456,9 +456,9 @@ import { } from "@onkernel/browser-loop/pi"; ``` -Conventional variables are `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, -`GOOGLE_API_KEY`/`GEMINI_API_KEY`, `XAI_API_KEY`, and -`MOONSHOT_API_KEY`. +Conventional variables are `OPENAI_API_KEY`, `ANTHROPIC_AUTH_TOKEN`/ +`ANTHROPIC_OAUTH_TOKEN`/`ANTHROPIC_API_KEY`, `GOOGLE_API_KEY`/ +`GEMINI_API_KEY`, `XAI_API_KEY`, and `MOONSHOT_API_KEY`. ## pi extension diff --git a/packages/browser-loop/src/pi/api-keys.ts b/packages/browser-loop/src/pi/api-keys.ts index 5ecf68fa..736f84db 100644 --- a/packages/browser-loop/src/pi/api-keys.ts +++ b/packages/browser-loop/src/pi/api-keys.ts @@ -12,7 +12,7 @@ import { parseLoopModelRef, providerForModel, type LoopModelRef } from "./models */ const LOOP_PROVIDER_API_KEY_ENV_VARS: Readonly> = { openai: ["OPENAI_API_KEY"], - anthropic: ["ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_API_KEY"], + anthropic: ["ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_API_KEY"], google: ["GOOGLE_API_KEY", "GEMINI_API_KEY"], xai: ["XAI_API_KEY"], moonshotai: ["MOONSHOT_API_KEY"], diff --git a/packages/browser-loop/test/api-keys.test.ts b/packages/browser-loop/test/api-keys.test.ts index ecc04433..df0e6f07 100644 --- a/packages/browser-loop/test/api-keys.test.ts +++ b/packages/browser-loop/test/api-keys.test.ts @@ -8,6 +8,7 @@ import { const ENV_KEYS = [ "OPENAI_API_KEY", + "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_API_KEY", "GOOGLE_API_KEY", @@ -30,6 +31,7 @@ afterEach(() => { describe("loop api key helpers", () => { it("maps provider names to expected environment variables", () => { expect(loopApiKeyEnvVarsForProvider("openai")).toEqual(["OPENAI_API_KEY"]); + expect(loopApiKeyEnvVarsForProvider("anthropic")).toEqual(["ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_API_KEY"]); expect(loopApiKeyEnvVarsForProvider("google")).toEqual(["GOOGLE_API_KEY", "GEMINI_API_KEY"]); expect(loopApiKeyEnvVarsForProvider("gemini")).toEqual(["GOOGLE_API_KEY", "GEMINI_API_KEY"]); expect(loopApiKeyEnvVarsForProvider("xai")).toEqual(["XAI_API_KEY"]); @@ -40,6 +42,11 @@ describe("loop api key helpers", () => { }); it("resolves provider api keys with fallback order", () => { + delete process.env.ANTHROPIC_OAUTH_TOKEN; + delete process.env.ANTHROPIC_API_KEY; + process.env.ANTHROPIC_AUTH_TOKEN = "anthropic-auth"; + expect(getLoopEnvApiKey("anthropic")).toBe("anthropic-auth"); + delete process.env.GOOGLE_API_KEY; process.env.GEMINI_API_KEY = "gemini"; expect(getLoopEnvApiKey("google")).toBe("gemini"); From a02b9d5c04748cce270549ce3a24f131768821ec Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 21 Aug 2026 14:08:56 +0000 Subject: [PATCH 5/7] Keep bearer tokens out of API key helpers --- packages/browser-loop/README.md | 6 +++--- packages/browser-loop/src/pi/api-keys.ts | 2 +- packages/browser-loop/test/api-keys.test.ts | 7 ------- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/packages/browser-loop/README.md b/packages/browser-loop/README.md index fd1f8b08..eef06024 100644 --- a/packages/browser-loop/README.md +++ b/packages/browser-loop/README.md @@ -456,9 +456,9 @@ import { } from "@onkernel/browser-loop/pi"; ``` -Conventional variables are `OPENAI_API_KEY`, `ANTHROPIC_AUTH_TOKEN`/ -`ANTHROPIC_OAUTH_TOKEN`/`ANTHROPIC_API_KEY`, `GOOGLE_API_KEY`/ -`GEMINI_API_KEY`, `XAI_API_KEY`, and `MOONSHOT_API_KEY`. +Conventional variables are `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, +`GOOGLE_API_KEY`/`GEMINI_API_KEY`, `XAI_API_KEY`, and +`MOONSHOT_API_KEY`. ## pi extension diff --git a/packages/browser-loop/src/pi/api-keys.ts b/packages/browser-loop/src/pi/api-keys.ts index 736f84db..5ecf68fa 100644 --- a/packages/browser-loop/src/pi/api-keys.ts +++ b/packages/browser-loop/src/pi/api-keys.ts @@ -12,7 +12,7 @@ import { parseLoopModelRef, providerForModel, type LoopModelRef } from "./models */ const LOOP_PROVIDER_API_KEY_ENV_VARS: Readonly> = { openai: ["OPENAI_API_KEY"], - anthropic: ["ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_API_KEY"], + anthropic: ["ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_API_KEY"], google: ["GOOGLE_API_KEY", "GEMINI_API_KEY"], xai: ["XAI_API_KEY"], moonshotai: ["MOONSHOT_API_KEY"], diff --git a/packages/browser-loop/test/api-keys.test.ts b/packages/browser-loop/test/api-keys.test.ts index df0e6f07..ecc04433 100644 --- a/packages/browser-loop/test/api-keys.test.ts +++ b/packages/browser-loop/test/api-keys.test.ts @@ -8,7 +8,6 @@ import { const ENV_KEYS = [ "OPENAI_API_KEY", - "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_API_KEY", "GOOGLE_API_KEY", @@ -31,7 +30,6 @@ afterEach(() => { describe("loop api key helpers", () => { it("maps provider names to expected environment variables", () => { expect(loopApiKeyEnvVarsForProvider("openai")).toEqual(["OPENAI_API_KEY"]); - expect(loopApiKeyEnvVarsForProvider("anthropic")).toEqual(["ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_API_KEY"]); expect(loopApiKeyEnvVarsForProvider("google")).toEqual(["GOOGLE_API_KEY", "GEMINI_API_KEY"]); expect(loopApiKeyEnvVarsForProvider("gemini")).toEqual(["GOOGLE_API_KEY", "GEMINI_API_KEY"]); expect(loopApiKeyEnvVarsForProvider("xai")).toEqual(["XAI_API_KEY"]); @@ -42,11 +40,6 @@ describe("loop api key helpers", () => { }); it("resolves provider api keys with fallback order", () => { - delete process.env.ANTHROPIC_OAUTH_TOKEN; - delete process.env.ANTHROPIC_API_KEY; - process.env.ANTHROPIC_AUTH_TOKEN = "anthropic-auth"; - expect(getLoopEnvApiKey("anthropic")).toBe("anthropic-auth"); - delete process.env.GOOGLE_API_KEY; process.env.GEMINI_API_KEY = "gemini"; expect(getLoopEnvApiKey("google")).toBe("gemini"); From 80ad0d526924e38ab872c303031a5b85205e451e Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 21 Aug 2026 14:11:50 +0000 Subject: [PATCH 6/7] Test Anthropic bearer token isolation --- packages/browser-loop/test/api-keys.test.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/browser-loop/test/api-keys.test.ts b/packages/browser-loop/test/api-keys.test.ts index ecc04433..7d35fb27 100644 --- a/packages/browser-loop/test/api-keys.test.ts +++ b/packages/browser-loop/test/api-keys.test.ts @@ -8,6 +8,7 @@ import { const ENV_KEYS = [ "OPENAI_API_KEY", + "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_API_KEY", "GOOGLE_API_KEY", @@ -39,6 +40,13 @@ describe("loop api key helpers", () => { expect(loopApiKeyEnvVarsForProvider("unknown")).toEqual([]); }); + it("does not return Anthropic bearer tokens as API keys", () => { + process.env.ANTHROPIC_AUTH_TOKEN = "bearer"; + delete process.env.ANTHROPIC_OAUTH_TOKEN; + delete process.env.ANTHROPIC_API_KEY; + expect(getLoopEnvApiKey("anthropic")).toBeUndefined(); + }); + it("resolves provider api keys with fallback order", () => { delete process.env.GOOGLE_API_KEY; process.env.GEMINI_API_KEY = "gemini"; From c71fad667aa2c312703e178e66ce2a062a703640 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 21 Aug 2026 14:45:47 +0000 Subject: [PATCH 7/7] Document Anthropic browser JavaScript option --- packages/browser-loop/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/browser-loop/README.md b/packages/browser-loop/README.md index eef06024..1b393a44 100644 --- a/packages/browser-loop/README.md +++ b/packages/browser-loop/README.md @@ -341,6 +341,16 @@ loop.providers.google.toolsets.browser({ exclude: ["right_click"] }); loop.toolsets.browser(); ``` +Anthropic's browser toolset leaves `javascript_exec` disabled unless it is +enabled explicitly: + +```ts +loop.providers.anthropic.tools.browser({ + version: "20260801", + javascript: true, +}); +``` + The Google browser set exposes the current predefined action names and uses normalized coordinates in `[0, 999]`. Its native `computer_use` declaration excludes every unselected browser action. If Google emits an excluded name