diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d72384d979..c84ba8a9c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -289,6 +289,7 @@ jobs: packages/integrations/deepagents-sdk/dist/** packages/integrations/fx-sdk/dist/** packages/integrations/cursor-sdk/dist/** + packages/integrations/grok-build-sdk/dist/** packages/evals/dist/** retention-days: 1 diff --git a/packages/docs/docs.json b/packages/docs/docs.json index 5cdf45161e..1fd6cbf5ef 100644 --- a/packages/docs/docs.json +++ b/packages/docs/docs.json @@ -70,6 +70,7 @@ "v4/integrations/cli-agents/overview", "v4/integrations/cli-agents/claude-code", "v4/integrations/cli-agents/codex", + "v4/integrations/cli-agents/grok-build", "v4/integrations/cli-agents/cursor", "v4/integrations/cli-agents/fx", "v4/integrations/cli-agents/pi" diff --git a/packages/docs/images/integrations/grok-build.svg b/packages/docs/images/integrations/grok-build.svg new file mode 100644 index 0000000000..7342b543cd --- /dev/null +++ b/packages/docs/images/integrations/grok-build.svg @@ -0,0 +1 @@ +Grok Build diff --git a/packages/docs/v4/integrations/cli-agents/grok-build.mdx b/packages/docs/v4/integrations/cli-agents/grok-build.mdx new file mode 100644 index 0000000000..834c621107 --- /dev/null +++ b/packages/docs/v4/integrations/cli-agents/grok-build.mdx @@ -0,0 +1,117 @@ +--- +title: "Grok Build" +description: "Give Grok Build persistent Stagehand browser tools over MCP/stdio." +--- + +The Grok Build integration connects the Grok CLI to the Stagehand facade MCP server over MCP/stdio. One server process owns the browser, so page state survives across `run`, `snapshot`, and `screenshot` calls. + + +Stagehand ships this experimental integration from the repository rather than publishing it as a standalone adapter. + + +## Prerequisites + +- Node.js 24 or newer +- pnpm 11.10.0 +- The Grok Build CLI and either `XAI_API_KEY` or an existing `grok login` +- A current Google Chrome installation for local browser mode + +## Quickstart + + + +```bash +git clone https://github.com/browserbase/stagehand.git +cd stagehand +pnpm install --frozen-lockfile +pnpm exec turbo run build \ + --filter @browserbasehq/stagehand-integrations +``` + + +```bash +npm install --global @xai-official/grok +grok login +# or: export XAI_API_KEY="your-xai-api-key" +``` + + +Copy `packages/integrations/grok-build/.grok/config.toml` into the project where Grok will run, then replace the facade path and browser credentials. Grok merges project MCP configuration over its user settings: + +```toml +[mcp_servers.stagehand] +command = "node" +args = [ + "/absolute/path/to/stagehand/packages/integrations/core/dist/facade/stdio-server.mjs", + "--max-screenshot-base64-bytes=60000", +] +startup_timeout_sec = 60 +tool_timeout_sec = 300 + +[mcp_servers.stagehand.env] +STAGEHAND_BROWSER = "browserbase" +BROWSERBASE_API_KEY = "bb_live_..." +``` + +Start Grok from that project so it loads `.grok/config.toml` and the included `AGENTS.md`. + + +The example defaults to Browserbase when `BROWSERBASE_API_KEY` is set in the MCP server env. For local Chrome: + +```toml +[mcp_servers.stagehand.env] +STAGEHAND_BROWSER = "local" +``` + + +```bash +cd packages/integrations/grok-build +grok +``` + +At the prompt, enter: + +```text +Use the Stagehand browser tools to open https://example.com, take a snapshot, +and report the heading with its snapshot ID. +``` + + + +## Configuration + +| Variable | Purpose | +| --- | --- | +| `XAI_API_KEY` | Credential for the Grok CLI. Grok does not need to forward it to the MCP child. | +| `STAGEHAND_BROWSER` | Select `local` or `browserbase`. | +| `BROWSERBASE_API_KEY` | Required for Browserbase. | +| `BROWSERBASE_PROJECT_ID` | Optional Browserbase project ID. | +| `STAGEHAND_MODEL_NAME` | Optional model for Stagehand AI methods called inside `run`. | +| `STAGEHAND_MODEL_API_KEY` | Required with `STAGEHAND_MODEL_NAME`; the MCP child does not receive Grok credentials. | + +Put Stagehand and Browserbase values in `[mcp_servers.stagehand.env]`. Grok does not expand shell variables in config values; paste the real keys or generate this file from your environment. + +## Keep the browser session alive + +The package mounts one facade MCP server for the Grok session and raises MCP startup and tool timeouts because browser launches exceed the defaults. Preserve that lifetime if you adapt the integration; a new process per tool call starts a new browser. + +The MCP child receives only Stagehand and Browserbase configuration plus the process values required to launch Node. The host's model credential remains in the Grok process. + +## Headless one-shot runs + +Interactive sessions can approve tool calls when prompted. A `-p` run cannot accept approval responses, so tasks that require approval may fail. Pass `--always-approve` only when you want Grok to run tools without asking: + +```bash +cd packages/integrations/grok-build +grok \ + --always-approve \ + -p "Open https://example.com, snapshot it, and report the heading." +``` + + +`run` executes model-authored JavaScript in the browser. Use Browserbase for untrusted tasks and review the [integration security boundary](/v4/integrations/cli-agents/overview#security-boundary). + + + + Read the CLI configuration, MCP mount, and Stagehand tool guidance. + diff --git a/packages/docs/v4/integrations/cli-agents/overview.mdx b/packages/docs/v4/integrations/cli-agents/overview.mdx index d609b652f9..71799c4409 100644 --- a/packages/docs/v4/integrations/cli-agents/overview.mdx +++ b/packages/docs/v4/integrations/cli-agents/overview.mdx @@ -14,6 +14,8 @@ CLI agents that support stdio MCP can connect to the Stagehand MCP adapter. One | --- | --- | --- | | Claude Code | MCP server configured in a project `.mcp.json`. | [Claude Code](/v4/integrations/cli-agents/claude-code#connect-a-running-claude-code-cli) | | Codex | MCP server configured in `~/.codex/config.toml`. | [Codex](/v4/integrations/cli-agents/codex#connect-a-running-codex-cli) | +| Grok Build | MCP server configured in a project `.grok/config.toml`. | [Grok Build](/v4/integrations/cli-agents/grok-build#quickstart) | +| Cursor | Local Cursor SDK session with a project MCP server. | [Cursor](/v4/integrations/cli-agents/cursor) | | fx | MCP server configured in `~/.fx/mcp.json`. | [fx](/v4/integrations/cli-agents/fx) | | Pi | Native extension that registers Stagehand tools directly. | [Pi](/v4/integrations/cli-agents/pi) | diff --git a/packages/docs/v4/integrations/overview.mdx b/packages/docs/v4/integrations/overview.mdx index 169951e2c3..5820b9cd14 100644 --- a/packages/docs/v4/integrations/overview.mdx +++ b/packages/docs/v4/integrations/overview.mdx @@ -41,6 +41,9 @@ Give a coding agent browser tools for tasks such as checking a running applicati Give a Codex agent persistent Stagehand browser tools over MCP/stdio. + + Give a Grok Build agent persistent Stagehand browser tools. + Give a local Cursor SDK agent persistent Stagehand browser tools over MCP/stdio. diff --git a/packages/evals/docs/harness-contract.md b/packages/evals/docs/harness-contract.md index 70f9e7380d..cef9a95130 100644 --- a/packages/evals/docs/harness-contract.md +++ b/packages/evals/docs/harness-contract.md @@ -20,13 +20,13 @@ historical harness default. Only positive safe integers are accepted. and requested reasoning settings where supported. Equal budget numbers do not mean equal work: -| Harness | Counted unit | -| --------------------------------------- | --------------------- | -| Codex, Cursor, DeepAgents | Tool calls | -| Eve | Successful tool calls | -| Mastra | Model steps | -| fx | Agent steps | -| Claude Code, Pi, Claude CUA, Gemini CUA | Turns | +| Harness | Counted unit | +| --------------------------------------------------- | --------------------- | +| Codex, Cursor, DeepAgents | Tool calls | +| Eve | Successful tool calls | +| Mastra | Model steps | +| fx | Agent steps | +| Claude Code, Pi, Claude CUA, Gemini CUA, Grok Build | Turns | These are execution limits, not comparable measures of model efficiency. diff --git a/packages/evals/framework/benchHarness.ts b/packages/evals/framework/benchHarness.ts index a732a06acc..2f014a032e 100644 --- a/packages/evals/framework/benchHarness.ts +++ b/packages/evals/framework/benchHarness.ts @@ -27,6 +27,8 @@ import { runFxAgent } from "./fxRunner.js"; import { FX_TOOL_SURFACES, prepareFxToolAdapter } from "./fxToolAdapter.js"; import { runCursorAgent } from "./cursorRunner.js"; import { CURSOR_TOOL_SURFACES, prepareCursorToolAdapter } from "./cursorToolAdapter.js"; +import { runGrokBuildAgent } from "./grokBuildRunner.js"; +import { GROK_BUILD_TOOL_SURFACES, prepareGrokBuildToolAdapter } from "./grokBuildToolAdapter.js"; import { buildExternalHarnessTaskPlan, type ExternalHarnessTaskPlan, @@ -383,6 +385,14 @@ export const cursorHarness = defineExternalHarness({ runAgent: runCursorAgent, }); +export const grokBuildHarness = defineExternalHarness({ + harness: "grok_build", + supportedToolSurfaces: GROK_BUILD_TOOL_SURFACES, + defaultModels: ["grok-build/auto" as AvailableModel], + prepareToolAdapter: prepareGrokBuildToolAdapter, + runAgent: runGrokBuildAgent, +}); + export const claudeCuaHarness = defineExternalHarness({ harness: "claude_cua", supportedToolSurfaces: CLAUDE_CUA_TOOL_SURFACES, @@ -409,6 +419,7 @@ const harnessRegistry = new Map([ ["deepagents", deepagentsHarness], ["fx", fxHarness], ["cursor", cursorHarness], + ["grok_build", grokBuildHarness], ["claude_cua", claudeCuaHarness], ["gemini_cua", geminiCuaHarness], ]); diff --git a/packages/evals/framework/costEstimate.ts b/packages/evals/framework/costEstimate.ts index b35439b466..2627e2ee20 100644 --- a/packages/evals/framework/costEstimate.ts +++ b/packages/evals/framework/costEstimate.ts @@ -52,6 +52,7 @@ export interface BilledCost { * | mastra | never (AI SDK usage has no dollars) | — | provider SDK with our key → computed _api | * | deepagents | never (LangChain usage_metadata) | — | provider SDK with our key → computed _api | * | cursor | never | — | subscription → unavailable | + * | grok_build | end event `total_cost_usd` | xai | unreported → unavailable | */ const REPORTED_CHANNEL: Readonly> = { claude_code: "anthropic_api", @@ -60,6 +61,7 @@ const REPORTED_CHANNEL: Readonly> = { eve: "ai_gateway", pi: "pi_catalog", fx: "fx_gateway", + grok_build: "xai", }; /** Harnesses whose unreported bill is our own provider-API spend, priceable at list. */ diff --git a/packages/evals/framework/grokBuildRunner.ts b/packages/evals/framework/grokBuildRunner.ts new file mode 100644 index 0000000000..377c58f36d --- /dev/null +++ b/packages/evals/framework/grokBuildRunner.ts @@ -0,0 +1,205 @@ +import { + buildGrokBuildTranscript, + extractGrokBuildToolCall, + runGrokBuildSession, + stringifyError, + type GrokBuildProcessRunner, + type GrokBuildTokenUsage, +} from "@browserbasehq/stagehand-integrations-grok-build-sdk"; +import type { AvailableModel } from "stagehand-v3"; +import type { EvalLogger } from "../logger.js"; +import { EvalsError } from "../errors.js"; +import type { ExternalHarnessTaskPlan } from "./externalHarnessPlan.js"; +import type { PreparedGrokBuildToolAdapter } from "./grokBuildToolAdapter.js"; +import { grokBuildAdapter } from "./harnesses/grokBuildAdapter.js"; +import { + buildExternalHarnessPrompt, + metricValue, + parseEvalResult, + runExternalHarnessTask, + type ExternalHarnessToolAdapterLike, + type MetricValue, + type ParsedEvalResult, +} from "./harnesses/externalRunner.js"; +import { resolveStepBudget } from "./stepBudget.js"; +import type { TaskResult } from "./types.js"; +import type { ExternalHarnessVerifierConfig } from "./verifierAdapter.js"; + +export type { GrokBuildProcessRunner } from "@browserbasehq/stagehand-integrations-grok-build-sdk"; + +export interface GrokBuildRunnerInput { + plan: ExternalHarnessTaskPlan; + model: AvailableModel; + logger: EvalLogger; + toolAdapter?: PreparedGrokBuildToolAdapter; + signal?: AbortSignal; + runProcess?: GrokBuildProcessRunner; + verifier?: ExternalHarnessVerifierConfig; +} + +export interface ParsedGrokBuildResult extends ParsedEvalResult {} + +const MCP_ONLY_LINE = + "Your only browser access is the MCP server configured in this workspace. Never launch a browser yourself or run shell commands to browse."; + +function composeGrokBuildToolInstructions(toolInstructions?: string): string { + return [ + toolInstructions ?? "Use the available browser tools to complete the task.", + MCP_ONLY_LINE, + "For Stagehand snapshots, start with includeIframes:false. Include iframes only when the task needs embedded-frame content; unrelated advertising frames can stall snapshots.", + "Before selecting a result, check every requested constraint against the exact corresponding field in the observed page data. Do not substitute similarly named metrics or relax numerical requirements.", + "Before your final answer, recheck that your chosen result satisfies all constraints together. If it does not, select another verified candidate rather than reporting success for a near match.", + "Do not edit repository files.", + ].join("\n"); +} + +export function buildGrokBuildPrompt( + plan: ExternalHarnessTaskPlan, + toolInstructions?: string, +): string { + return buildExternalHarnessPrompt({ + plan, + toolInstructions: composeGrokBuildToolInstructions(toolInstructions), + resultContract: "marker", + }); +} + +export function parseGrokBuildResult(raw: string): ParsedGrokBuildResult { + return parseEvalResult(raw); +} + +function resolveGrokBuildAlwaysApprove(): boolean { + const value = process.env.EVAL_GROK_BUILD_ALWAYS_APPROVE; + if (value === undefined || value === "true") return true; + if (value === "false") return false; + throw new EvalsError("EVAL_GROK_BUILD_ALWAYS_APPROVE must be true or false."); +} + +export async function runGrokBuildAgent({ + plan, + model, + logger, + toolAdapter, + signal, + runProcess, + verifier, +}: GrokBuildRunnerInput): Promise { + const alwaysApprove = resolveGrokBuildAlwaysApprove(); + const adapterLike: ExternalHarnessToolAdapterLike = { + promptInstructions: composeGrokBuildToolInstructions(toolAdapter?.promptInstructions), + captureEvidence: toolAdapter?.captureEvidence, + drainStepObservations: toolAdapter?.drainStepObservations, + observedToolMatcher: toolAdapter?.observedToolMatcher, + browserSessionLoss: toolAdapter?.browserSessionLoss, + }; + const maxTurns = resolveStepBudget({ + harnessEnvKey: "EVAL_GROK_BUILD_MAX_TURNS", + dataset: plan.dataset, + harnessDefault: 50, + }); + return runExternalHarnessTask({ + harness: "grok_build", + plan, + model, + logger, + systemPromptMode: "native", + implementation: { name: "cli", version: 1 }, + configuration: { alwaysApprove }, + toolAdapter: adapterLike, + verifier, + resultContract: "marker", + fallbackErrorMessage: "Grok Build did not report success", + stepBudget: maxTurns, + stepBudgetUnit: "turns", + parseResult: parseGrokBuildResult, + runSession: async (prompt, systemPrompt) => { + const sessionResult = await runGrokBuildSession({ + prompt, + model, + logger, + signal, + runProcess, + session: { + alwaysApprove, + ...(toolAdapter?.cwd && { cwd: toolAdapter.cwd }), + ...(toolAdapter?.env && { env: toolAdapter.env }), + ...(process.env.EVAL_GROK_BUILD_PATH && { + binaryPath: process.env.EVAL_GROK_BUILD_PATH, + }), + maxTurns, + ...(systemPrompt && { rules: systemPrompt }), + ...(process.env.EVAL_GROK_BUILD_SANDBOX && { + sandbox: process.env.EVAL_GROK_BUILD_SANDBOX, + }), + }, + onToolResult: toolAdapter?.onToolResult + ? (name, view) => toolAdapter.onToolResult!(name, view) + : undefined, + }); + const usage = sessionResult.tokenUsage; + return { + raw: sessionResult, + resultText: sessionResult.resultText, + transcriptText: buildGrokBuildTranscript(sessionResult.events), + iterationError: sessionResult.iterationError, + status: sessionResult.status, + stopReason: + sessionResult.stopReason || + (sessionResult.status === "sdk_error" + ? stringifyError(sessionResult.iterationError) || undefined + : undefined), + usage: { + reported: usage.reported, + inputTokens: usage.inputTokens, + outputTokens: usage.outputTokens, + totalTokens: usage.totalTokens, + cachedInputTokens: usage.cachedInputTokens, + cacheCreationInputTokens: usage.cacheCreationInputTokens, + reasoningOutputTokens: usage.reasoningOutputTokens, + }, + costUsd: sessionResult.costUsd, + metrics: buildGrokBuildMetrics(usage, sessionResult.endEvent, sessionResult.events), + }; + }, + toTrajectory: ( + { raw, parsed, finalObservation, stepObservations, observedToolName, status }, + taskSpec, + ) => + grokBuildAdapter.fromHarnessResult( + { + events: raw.events, + ...(finalObservation && { finalObservation }), + ...(stepObservations?.length && { stepObservations }), + ...(observedToolName && { observedToolName }), + finalAnswer: parsed.finalAnswer ?? raw.resultText, + status, + usage: { + input_tokens: raw.tokenUsage.inputTokens, + output_tokens: raw.tokenUsage.outputTokens, + cached_input_tokens: raw.tokenUsage.cachedInputTokens, + reasoning_tokens: raw.tokenUsage.reasoningOutputTokens, + }, + }, + taskSpec, + ), + }); +} + +function buildGrokBuildMetrics( + usage: GrokBuildTokenUsage, + endEvent: Record | undefined, + events: Array>, +): Record { + const toolSteps = events.filter( + (event) => extractGrokBuildToolCall(event)?.subtype === "completed", + ).length; + return { + grok_build_input_tokens: metricValue(usage.inputTokens), + grok_build_output_tokens: metricValue(usage.outputTokens), + grok_build_total_tokens: metricValue(usage.totalTokens), + grok_build_cached_input_tokens: metricValue(usage.cachedInputTokens), + grok_build_reasoning_tokens: metricValue(usage.reasoningOutputTokens), + grok_build_num_turns: metricValue(endEvent?.num_turns), + grok_build_tool_steps: metricValue(toolSteps), + }; +} diff --git a/packages/evals/framework/grokBuildToolAdapter.ts b/packages/evals/framework/grokBuildToolAdapter.ts new file mode 100644 index 0000000000..8e5cc93f17 --- /dev/null +++ b/packages/evals/framework/grokBuildToolAdapter.ts @@ -0,0 +1,313 @@ +import fsp from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { stringify } from "smol-toml"; +import type { GrokBuildToolCallView } from "@browserbasehq/stagehand-integrations-grok-build-sdk"; +import type { ProbeEvidence } from "stagehand-v3"; +import type { BrowserSessionLoss, StartupProfile, ToolSurface } from "../core/contracts/tool.js"; +import type { BrowserSessionInfo } from "./browserSession.js"; +import { EvalsError } from "../errors.js"; +import type { EvalLogger } from "../logger.js"; +import { startAgentToolRuntime } from "./agentToolRuntime.js"; +import type { ExternalHarnessTaskPlan } from "./externalHarnessPlan.js"; +import { resolveStartupProfile, resolveToolSurface } from "./harnesses/toolSurfaceResolution.js"; +import { ObservationRecorder, type StepObservation } from "./observationRecorder.js"; + +export interface GrokBuildToolAdapterInput { + toolSurface?: ToolSurface; + startupProfile?: StartupProfile; + environment: "LOCAL" | "BROWSERBASE"; + plan: ExternalHarnessTaskPlan; + logger: EvalLogger; +} + +export interface PreparedGrokBuildToolAdapter { + toolSurface: ToolSurface; + startupProfile: StartupProfile; + cwd: string; + home: string; + grokHome: string; + env: Record; + mcpConfigPath: string; + mcpServerNames: string[]; + promptInstructions: string; + browserSession?: BrowserSessionInfo; + browserSessionLoss?: () => BrowserSessionLoss | undefined; + captureEvidence?: () => Promise; + drainStepObservations?: () => Promise; + onToolResult?: (toolName: string, view?: GrokBuildToolCallView) => void; + observedToolMatcher?: (name: string) => boolean; + cleanup: () => Promise; +} + +type GrokBuildMcpServerSpec = { + command: string; + args?: string[]; + env?: Record; +}; + +export const GROK_BUILD_TOOL_SURFACES: ToolSurface[] = [ + "stagehand_facade", + "playwright_mcp", + "chrome_devtools_mcp", +]; + +export function buildGrokBuildMcpConfig( + mcpServers: Record, +): Record { + const normalized: Record = {}; + for (const [serverName, rawSpec] of Object.entries(mcpServers)) { + if (!isRecord(rawSpec) || typeof rawSpec.command !== "string") { + throw new EvalsError(`Invalid Grok Build MCP launch spec for server "${serverName}".`); + } + const spec = rawSpec as GrokBuildMcpServerSpec; + normalized[serverName] = { + command: spec.command, + args: stringArray(spec.args), + ...(isStringRecord(spec.env) && { env: spec.env }), + startup_timeout_sec: 60, + tool_timeout_sec: 300, + }; + } + return { mcp_servers: normalized }; +} + +export async function writeGrokBuildWorkspace( + grokHome: string, + mcpServers: Record, +): Promise<{ mcpConfigPath: string }> { + const mcpConfigPath = path.join(grokHome, "config.toml"); + await fsp.mkdir(grokHome, { recursive: true }); + await fsp.writeFile( + mcpConfigPath, + stringify({ + cli: { auto_update: false, use_leader: false }, + compat: { claude: { mcps: false }, cursor: { mcps: false } }, + subagents: { enabled: false }, + memory: { enabled: false }, + ...buildGrokBuildMcpConfig(mcpServers), + }), + { mode: 0o600 }, + ); + return { mcpConfigPath }; +} + +export function resolveGrokBuildAuthHome( + env: NodeJS.ProcessEnv, + platform: NodeJS.Platform = process.platform, +): string | undefined { + const configured = env.GROK_HOME?.trim(); + if (configured) return configured; + const userHome = + platform === "win32" + ? env.USERPROFILE?.trim() || env.HOME?.trim() + : env.HOME?.trim() || env.USERPROFILE?.trim(); + return userHome ? path.join(userHome, ".grok") : undefined; +} + +export async function copyGrokBuildAuth( + env: NodeJS.ProcessEnv, + targetGrokHome: string, +): Promise { + if (env.XAI_API_KEY?.trim()) return false; + const sourceHome = resolveGrokBuildAuthHome(env); + if (!sourceHome) return false; + try { + await fsp.copyFile(path.join(sourceHome, "auth.json"), path.join(targetGrokHome, "auth.json")); + return true; + } catch { + return false; + } +} + +export function isGrokBuildMountToolName(serverNames: string[], toolName: string): boolean { + return serverNames.some( + (server) => + toolName === server || + toolName.startsWith(`${server}.`) || + toolName.startsWith(`${server}__`) || + toolName === `mcp__${server}` || + toolName.startsWith(`mcp__${server}__`), + ); +} + +export async function prepareGrokBuildToolAdapter( + input: GrokBuildToolAdapterInput, +): Promise { + const toolSurface = resolveToolSurface( + { harness: "grok_build", supportedToolSurfaces: GROK_BUILD_TOOL_SURFACES }, + input.toolSurface, + ); + if (toolSurface === undefined) { + throw new EvalsError("grok_build harness requires a tool surface."); + } + const startupProfile = resolveStartupProfile( + toolSurface, + input.environment, + input.startupProfile, + ); + const runtime = await startAgentToolRuntime({ + toolSurface, + startupProfile, + environment: input.environment, + logger: input.logger, + }); + + let root: string | undefined; + try { + const mount = runtime.running.agentMount; + if (!mount) { + throw new EvalsError(`Tool surface "${toolSurface}" does not provide an agent mount.`); + } + if (mount.via !== "mcp") { + throw new EvalsError( + `Grok Build does not support agent mounts delivered via "${mount.via}" yet.`, + ); + } + + root = await fsp.mkdtemp( + path.join(os.tmpdir(), `stagehand-evals-grok-build-${toolSurface.replace(/_/gu, "-")}-`), + ); + const capturedRoot = root; + const home = path.join(root, "home"); + const cwd = path.join(root, "workspace"); + const grokHome = path.join(home, ".grok"); + await Promise.all([ + fsp.mkdir(cwd, { recursive: true }), + fsp.mkdir(grokHome, { recursive: true }), + ]); + await copyGrokBuildAuth(process.env, grokHome); + const { mcpConfigPath } = await writeGrokBuildWorkspace(grokHome, mount.mcpServers); + const mcpServerNames = Object.keys(mount.mcpServers); + const recorder = runtime.running.captureEvidence + ? new ObservationRecorder(runtime.running.captureEvidence) + : undefined; + const observedToolMatcher = (name: string): boolean => + isGrokBuildMountToolName(mcpServerNames, name); + let cleanupPromise: Promise | undefined; + + input.logger.log({ + category: "grok_build", + message: `Initialized ${toolSurface} MCP mount for Grok Build (servers: ${mcpServerNames.join(", ")}).`, + level: 1, + auxiliary: { + startupProfile: { value: startupProfile, type: "string" }, + environment: { value: input.environment, type: "string" }, + }, + }); + + return { + toolSurface, + startupProfile, + cwd, + home, + grokHome, + env: stringEnv({ + ...process.env, + HOME: home, + USERPROFILE: home, + GROK_HOME: grokHome, + GROK_DISABLE_AUTOUPDATER: "1", + }), + mcpConfigPath, + mcpServerNames, + promptInstructions: mount.promptInstructions, + browserSession: runtime.browserSession, + browserSessionLoss: runtime.running.browserSessionLoss, + ...(runtime.running.captureEvidence && { + captureEvidence: boundedCaptureEvidence(runtime.running.captureEvidence), + }), + ...(recorder && { + drainStepObservations: async () => { + await recorder.settle(); + return recorder.drain(); + }, + onToolResult: (toolName, view) => { + if (observedToolMatcher(toolName)) void recorder.record(view?.callId); + }, + }), + observedToolMatcher, + cleanup: async () => { + cleanupPromise ??= (async () => { + try { + await withTimeout( + runtime.cleanup(), + readPositiveIntEnv("EVAL_AGENT_MOUNT_CLEANUP_TIMEOUT_MS", 30_000), + ); + } catch { + // Best effort only. + } finally { + await fsp.rm(capturedRoot, { recursive: true, force: true }); + } + })(); + await cleanupPromise; + }, + }; + } catch (error) { + await withTimeout( + runtime.cleanup(), + readPositiveIntEnv("EVAL_AGENT_MOUNT_CLEANUP_TIMEOUT_MS", 30_000), + ).catch((): undefined => undefined); + if (root) await fsp.rm(root, { recursive: true, force: true }); + throw error; + } +} + +function boundedCaptureEvidence( + capture: () => Promise, +): () => Promise { + return async () => { + try { + return await withTimeout( + capture(), + readPositiveIntEnv("EVAL_CAPTURE_EVIDENCE_TIMEOUT_MS", 15_000), + ); + } catch { + return {}; + } + }; +} + +function readPositiveIntEnv(key: string, fallback: number): number { + const parsed = Number.parseInt(process.env[key] ?? "", 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +function withTimeout(promise: Promise, timeoutMs: number): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error(`grok_build adapter operation timed out after ${timeoutMs}ms`)), + timeoutMs, + ); + promise.then( + (value) => { + clearTimeout(timer); + resolve(value); + }, + (error) => { + clearTimeout(timer); + reject(error); + }, + ); + }); +} + +function stringArray(value: unknown): string[] { + return Array.isArray(value) + ? value.filter((item): item is string => typeof item === "string") + : []; +} + +function stringEnv(env: NodeJS.ProcessEnv): Record { + return Object.fromEntries( + Object.entries(env).filter((entry): entry is [string, string] => typeof entry[1] === "string"), + ); +} + +function isStringRecord(value: unknown): value is Record { + return isRecord(value) && Object.values(value).every((item) => typeof item === "string"); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/packages/evals/framework/harnesses/grokBuildAdapter.ts b/packages/evals/framework/harnesses/grokBuildAdapter.ts new file mode 100644 index 0000000000..79e53012b8 --- /dev/null +++ b/packages/evals/framework/harnesses/grokBuildAdapter.ts @@ -0,0 +1,136 @@ +import { extractGrokBuildToolCall } from "@browserbasehq/stagehand-integrations-grok-build-sdk"; +import type { ProbeEvidence, TaskSpec, Trajectory } from "stagehand-v3"; +import type { StepObservation } from "../observationRecorder.js"; +import { + buildTrajectory, + type NormalizedToolCall, + type TrajectoryAdapter, +} from "./trajectoryAdapter.js"; + +export interface GrokBuildRunResult { + events: Array>; + finalAnswer?: string; + status?: Trajectory["status"]; + usage?: Partial; + finalObservation?: ProbeEvidence; + stepObservations?: StepObservation[]; + observedToolName?: (name: string) => boolean; +} + +export class GrokBuildTrajectoryAdapter implements TrajectoryAdapter { + fromHarnessResult(result: GrokBuildRunResult, taskSpec: TaskSpec): Trajectory { + const toolCalls: NormalizedToolCall[] = []; + const openCalls = new Map(); + let pendingReasoning = ""; + + for (const event of result.events) { + if ((event.type === "thought" || event.type === "text") && typeof event.data === "string") { + pendingReasoning = appendText(pendingReasoning, event.data); + continue; + } + const view = extractGrokBuildToolCall(event); + if (!view) continue; + if (view.subtype === "started") { + const call: NormalizedToolCall = { + ...(view.callId && { id: view.callId }), + name: view.name ?? "tool", + args: view.args, + result: undefined, + ok: true, + reasoning: pendingReasoning.trim() || undefined, + }; + toolCalls.push(call); + if (view.callId) openCalls.set(view.callId, call); + pendingReasoning = ""; + continue; + } + + const call = view.callId ? openCalls.get(view.callId) : undefined; + if (call) { + call.result = normalizeToolResult(view.result); + call.ok = view.ok; + if (view.error) call.error = view.error; + openCalls.delete(view.callId); + } else { + toolCalls.push({ + ...(view.callId && { id: view.callId }), + name: view.name ?? "tool", + args: view.args, + result: normalizeToolResult(view.result), + ok: view.ok, + ...(view.error && { error: view.error }), + reasoning: pendingReasoning.trim() || undefined, + }); + pendingReasoning = ""; + } + } + + for (const open of openCalls.values()) { + open.ok = false; + open.result = "no tool result"; + open.error = "no tool result"; + } + + attachStepObservations(toolCalls, result); + return buildTrajectory({ + taskSpec, + toolCalls, + finalAnswer: result.finalAnswer, + status: result.status ?? "complete", + usage: result.usage, + ...(result.finalObservation && { finalObservation: result.finalObservation }), + }); + } +} + +export const grokBuildAdapter = new GrokBuildTrajectoryAdapter(); + +function normalizeToolResult(result: unknown): unknown { + if (isRecord(result) && result.type === "MCP" && isRecord(result.output)) { + return result.output.OkayOutput ?? result.output.Error ?? result.output; + } + if (!isRecord(result) || !Array.isArray(result.content)) return result; + const text = result.content + .filter(isRecord) + .map((block) => (typeof block.text === "string" ? block.text : undefined)) + .filter((part): part is string => part !== undefined); + return text.length > 0 ? text.join("\n") : result; +} + +function attachStepObservations(toolCalls: NormalizedToolCall[], result: GrokBuildRunResult): void { + const keyed = new Map( + (result.stepObservations ?? []) + .filter((observation) => observation.toolCallId) + .map((observation) => [observation.toolCallId, observation.evidence]), + ); + for (const call of toolCalls) { + if (call.id && keyed.has(call.id)) call.probeEvidence = keyed.get(call.id); + } + const observations = (result.stepObservations ?? []).filter( + (observation) => !observation.toolCallId, + ); + if (observations.length === 0) return; + const isObservedTool = + result.observedToolName ?? ((name: string) => name.startsWith("mcp") || name.includes(".")); + const observedCalls = toolCalls.filter((call) => isObservedTool(call.name)); + const totalObservedRuns = + Math.max(...observations.map((observation) => observation.runIndex)) + 1; + // A failed final capture leaves earlier observations valid at their original indices. + if (totalObservedRuns > observedCalls.length) return; + const byRunIndex = new Map( + observations.map((observation) => [observation.runIndex, observation.evidence]), + ); + observedCalls.forEach((call, ordinal) => { + if (call.probeEvidence) return; + const observation = byRunIndex.get(ordinal); + if (observation) call.probeEvidence = observation; + }); +} + +function appendText(current: string, next: string): string { + return current + next; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/packages/evals/framework/usageNormalization.ts b/packages/evals/framework/usageNormalization.ts index 4f991238d7..4fd8c8edb9 100644 --- a/packages/evals/framework/usageNormalization.ts +++ b/packages/evals/framework/usageNormalization.ts @@ -67,6 +67,7 @@ export interface NormalizeUsageInput { * | cursor | SDK input excludes cache; historical CLI records explicitly report no usage | * | cursor_sdk | SDK `totalTokens` sums input, output, cacheRead and cacheWrite; input excludes cache | * | claude_cua | raw Messages API `usage.input_tokens` + separate `cache_read/creation_input_tokens` | + * | grok_build | end-event `input_tokens` plus separate `cache_read/creation_input_tokens` | */ const HARNESS_CONVENTIONS: Readonly> = { claude_code: "anthropic_cache_separate", @@ -80,6 +81,7 @@ const HARNESS_CONVENTIONS: Readonly> = { cursor_sdk: "uncached_only", claude_cua: "anthropic_cache_separate", gemini_cua: "openai_cached_subset", + grok_build: "anthropic_cache_separate", }; /** Unknown harnesses get the most common SDK shape; the metric is still labelled. */ diff --git a/packages/evals/package.json b/packages/evals/package.json index 34ea9be703..8c4095bb29 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -33,6 +33,7 @@ "@browserbasehq/stagehand-integrations-eve-sdk": "workspace:*", "@browserbasehq/stagehand-integrations-fx-sdk": "workspace:*", "@browserbasehq/stagehand-integrations-gemini-cua-sdk": "workspace:*", + "@browserbasehq/stagehand-integrations-grok-build-sdk": "workspace:*", "@browserbasehq/stagehand-integrations-mastra-sdk": "workspace:*", "@browserbasehq/stagehand-integrations-pi-sdk": "workspace:*", "@clack/prompts": "^1.3.0", @@ -49,6 +50,7 @@ "openai": "^4.104.0", "playwright": ">=1.55.1 <1.57.0", "sharp": "^0.35.4", + "smol-toml": "catalog:", "stagehand-v3": "npm:@browserbasehq/stagehand@3.7.1", "tsx": "catalog:", "ws": "^8.21.0", diff --git a/packages/evals/tests/framework/benchHarness.test.ts b/packages/evals/tests/framework/benchHarness.test.ts index ebe6bc0fc6..32eaab4f3f 100644 --- a/packages/evals/tests/framework/benchHarness.test.ts +++ b/packages/evals/tests/framework/benchHarness.test.ts @@ -13,6 +13,7 @@ import { mastraHarness, piHarness, cursorHarness, + grokBuildHarness, fxHarness, deepagentsHarness, eveHarness, @@ -21,6 +22,7 @@ import { import { MASTRA_TOOL_SURFACES } from "../../framework/mastraToolAdapter.js"; import { PI_TOOL_SURFACES } from "../../framework/piToolAdapter.js"; import { CURSOR_TOOL_SURFACES } from "../../framework/cursorToolAdapter.js"; +import { GROK_BUILD_TOOL_SURFACES } from "../../framework/grokBuildToolAdapter.js"; import { defaultModelsEnvKey } from "../../framework/benchPlanner.js"; import type { BenchMatrixRow } from "../../framework/benchTypes.js"; import type { DiscoveredTask } from "../../framework/types.js"; @@ -39,6 +41,7 @@ describe("bench harness registry", () => { "deepagents", "fx", "cursor", + "grok_build", "claude_cua", "gemini_cua", ]); @@ -48,7 +51,7 @@ describe("bench harness registry", () => { expect(parseBenchHarness(undefined)).toBe("stagehand"); expect(parseBenchHarness("codex")).toBe("codex"); expect(() => parseBenchHarness("nope")).toThrow( - /Unknown harness "nope"\. Supported: stagehand, claude_code, codex, mastra, pi, eve, deepagents, fx, cursor, claude_cua, gemini_cua\./, + /Unknown harness "nope"\. Supported: stagehand, claude_code, codex, mastra, pi, eve, deepagents, fx, cursor, grok_build, claude_cua, gemini_cua\./, ); }); @@ -191,6 +194,20 @@ describe("bench harness registry", () => { expect(harness.defaultModels).toEqual(["cursor/auto"]); }); + it("registers grok_build as a concrete executable harness", () => { + const harness = getBenchHarness("grok_build"); + + expect(harness).toBe(grokBuildHarness); + expect(parseBenchHarness("grok_build")).toBe("grok_build"); + expect(isExecutableBenchHarness("grok_build")).toBe(true); + expect(harness.supportedTaskKinds).toEqual(["agent", "suite"]); + expect(harness.supportsApi).toBe(false); + expect(harness.execute).toBeDefined(); + expect(harness.supportedToolSurfaces).toEqual(GROK_BUILD_TOOL_SURFACES); + expect(harness.supportedToolSurfaces[0]).toBe("stagehand_facade"); + expect(harness.defaultModels).toEqual(["grok-build/auto"]); + }); + it("registers a new harness and rejects duplicate ids", () => { const fakeHarness = { harness: "fake_harness", diff --git a/packages/evals/tests/framework/costEstimate.test.ts b/packages/evals/tests/framework/costEstimate.test.ts index 9f9296105a..18b0beed2a 100644 --- a/packages/evals/tests/framework/costEstimate.test.ts +++ b/packages/evals/tests/framework/costEstimate.test.ts @@ -254,6 +254,15 @@ describe("resolveBilledCost", () => { priceMap, }).billing_channel, ).toBe("fx_gateway"); + expect( + resolveBilledCost({ + harness: "grok_build", + model: "grok-build/grok-4.6", + usage, + reportedCostUsd: 0.02, + priceMap, + }).billing_channel, + ).toBe("xai"); // A reported figure wins even for a priced model on a direct-API harness. expect( resolveBilledCost({ diff --git a/packages/evals/tests/framework/grokBuildAdapter.test.ts b/packages/evals/tests/framework/grokBuildAdapter.test.ts new file mode 100644 index 0000000000..027916546f --- /dev/null +++ b/packages/evals/tests/framework/grokBuildAdapter.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, it } from "vitest"; +import type { TaskSpec } from "stagehand-v3"; +import { grokBuildAdapter } from "../../framework/harnesses/grokBuildAdapter.js"; + +const taskSpec: TaskSpec = { id: "grok-build-test", instruction: "do the task" }; + +describe("Grok Build trajectory adapter", () => { + it("pairs wrapped MCP calls across partial updates and retains browser evidence", () => { + const trajectory = grokBuildAdapter.fromHarnessResult( + { + events: [ + { type: "thought", data: "Inspect" }, + { type: "thought", data: " the page." }, + { + type: "tool_call", + toolCallId: "call-1", + toolName: "use_tool", + rawInput: { tool_name: "stagehand__snapshot", tool_input: { includeIframes: false } }, + }, + { + type: "tool_call_update", + toolCallId: "call-1", + status: null, + content: [], + rawOutput: null, + }, + { + type: "tool_call_update", + toolCallId: "call-1", + status: "completed", + rawOutput: { + type: "MCP", + server_name: "stagehand", + tool_name: "snapshot", + output: { OkayOutput: "Recipe: vegetarian lasagna" }, + }, + }, + ], + observedToolName: (name) => name.startsWith("stagehand__"), + stepObservations: [ + { runIndex: 0, toolCallId: "call-1", evidence: { url: "https://example.com/recipe" } }, + ], + }, + taskSpec, + ); + expect(trajectory.steps).toHaveLength(1); + expect(trajectory.steps[0]).toMatchObject({ + actionName: "stagehand__snapshot", + actionArgs: { includeIframes: false }, + reasoning: "Inspect the page.", + toolOutput: { ok: true, result: "Recipe: vegetarian lasagna" }, + probeEvidence: { url: "https://example.com/recipe" }, + }); + }); + + it("pairs native tool calls and carries thought text into reasoning", () => { + const trajectory = grokBuildAdapter.fromHarnessResult( + { + events: [ + { type: "thought", data: "I will inspect the page." }, + { + type: "tool_call", + toolCallId: "call-1", + toolName: "stagehand__snapshot", + rawInput: {}, + }, + { + type: "tool_call_update", + toolCallId: "call-1", + status: "completed", + rawOutput: { title: "Example" }, + }, + ], + finalAnswer: "Example", + }, + taskSpec, + ); + expect(trajectory.steps).toHaveLength(1); + expect(trajectory.steps[0]).toMatchObject({ + actionName: "stagehand__snapshot", + reasoning: "I will inspect the page.", + toolOutput: { ok: true, result: { title: "Example" } }, + }); + expect(trajectory.finalAnswer).toBe("Example"); + }); + + it("fails open tool calls closed and attaches matching observations", () => { + const trajectory = grokBuildAdapter.fromHarnessResult( + { + events: [ + { + type: "tool_call", + toolCallId: "call-1", + toolName: "stagehand__run", + rawInput: {}, + }, + ], + observedToolName: (name) => name.startsWith("stagehand__"), + stepObservations: [{ runIndex: 0, evidence: { url: "https://example.com" } }], + }, + taskSpec, + ); + expect(trajectory.steps[0].toolOutput).toMatchObject({ + ok: false, + error: "no tool result", + }); + expect(trajectory.steps[0].probeEvidence.url).toBe("https://example.com"); + }); + + it("keeps earlier evidence when the last browser observation is unavailable", () => { + const trajectory = grokBuildAdapter.fromHarnessResult( + { + events: ["first", "last"].flatMap((toolCallId) => [ + { type: "tool_call", toolCallId, toolName: "stagehand__snapshot", rawInput: {} }, + { type: "tool_call_update", toolCallId, status: "completed", rawOutput: "page" }, + ]), + observedToolName: (name) => name.startsWith("stagehand__"), + stepObservations: [{ runIndex: 0, evidence: { url: "https://example.com" } }], + }, + taskSpec, + ); + expect(trajectory.steps[0].probeEvidence.url).toBe("https://example.com"); + expect(trajectory.steps[1].probeEvidence.url).toBeUndefined(); + }); +}); diff --git a/packages/evals/tests/framework/grokBuildRunner.test.ts b/packages/evals/tests/framework/grokBuildRunner.test.ts new file mode 100644 index 0000000000..084ff11835 --- /dev/null +++ b/packages/evals/tests/framework/grokBuildRunner.test.ts @@ -0,0 +1,146 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { AvailableModel } from "stagehand-v3"; +import type { GrokBuildProcessRunner } from "@browserbasehq/stagehand-integrations-grok-build-sdk"; +import { buildGrokBuildPrompt, runGrokBuildAgent } from "../../framework/grokBuildRunner.js"; +import type { ExternalHarnessTaskPlan } from "../../framework/externalHarnessPlan.js"; +import { EVAL_SYSTEM_PROMPT } from "../../framework/evalSystemPrompt.js"; +import { EvalLogger } from "../../logger.js"; + +const plan: ExternalHarnessTaskPlan = { + dataset: "webvoyager", + taskId: "wv-1", + startUrl: "https://example.com", + instruction: "Report the heading", +}; + +describe("Grok Build runner", () => { + beforeEach(() => { + vi.stubEnv("EVAL_GROK_BUILD_ALWAYS_APPROVE", undefined); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("builds an MCP-only browser prompt", () => { + const prompt = buildGrokBuildPrompt(plan, "Use stagehand__run."); + expect(prompt).toContain("Dataset: webvoyager"); + expect(prompt).toContain("Start URL: https://example.com"); + expect(prompt).toContain("Use stagehand__run."); + expect(prompt).toContain("Your only browser access is the MCP server"); + expect(prompt).toContain("EVAL_RESULT:"); + expect(prompt).not.toContain(EVAL_SYSTEM_PROMPT); + }); + + it("runs the native stream and reports Grok Build metrics", async () => { + let capturedArgs: string[] = []; + const result = await runGrokBuildAgent({ + plan, + model: "grok-build/auto" as AvailableModel, + logger: new EvalLogger(false), + runProcess: scriptedRunner( + [ + { + type: "tool_call", + toolCallId: "1", + toolName: "stagehand__run", + rawInput: { code: "return 1" }, + }, + { + type: "tool_call_update", + toolCallId: "1", + status: "completed", + rawOutput: "done", + }, + { + type: "text", + data: 'EVAL_RESULT: {"success":true,"summary":"done","finalAnswer":"ok"}', + }, + { + type: "end", + stopReason: "end_turn", + num_turns: 2, + usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15 }, + total_cost_usd: 0.02, + }, + ], + 0, + (args) => { + capturedArgs = args; + }, + ), + }); + const metrics = result.metrics as Record; + expect(result._success).toBe(true); + expect(result.harnessStatus).toBe("completed"); + expect(result.grokBuildStatus).toBe("completed"); + expect(result.finalAnswer).toBe("ok"); + expect(metrics.grok_build_tool_steps.value).toBe(1); + expect(metrics.grok_build_num_turns.value).toBe(2); + expect(metrics.harness_total_tokens.value).toBe(15); + expect(metrics.harness_cost_usd.value).toBe(0.02); + expect(metrics.step_budget.value).toBe(50); + expect(result.harnessImplementation).toMatchObject({ name: "cli", version: 1 }); + expect(capturedArgs).toContain("--rules"); + expect(capturedArgs).toContain(EVAL_SYSTEM_PROMPT); + expect(capturedArgs).toContain("--max-turns"); + expect(capturedArgs).toContain("--always-approve"); + expect(result.harnessConfiguration).toMatchObject({ alwaysApprove: true }); + }); + + it.each(["true", "false"])("respects the approval override %s", async (value) => { + vi.stubEnv("EVAL_GROK_BUILD_ALWAYS_APPROVE", value); + let capturedArgs: string[] = []; + const result = await runGrokBuildAgent({ + plan, + model: "grok-build/auto" as AvailableModel, + logger: new EvalLogger(false), + runProcess: scriptedRunner([{ type: "end", stopReason: "end_turn" }], 0, (args) => { + capturedArgs = args; + }), + }); + expect(capturedArgs.includes("--always-approve")).toBe(value === "true"); + expect(result.harnessConfiguration).toMatchObject({ alwaysApprove: value === "true" }); + }); + + it.each(["", "1", "TRUE", "invalid"])( + "rejects invalid approval override %j before launch", + async (value) => { + vi.stubEnv("EVAL_GROK_BUILD_ALWAYS_APPROVE", value); + const runProcess = vi.fn(); + await expect( + runGrokBuildAgent({ + plan, + model: "grok-build/auto" as AvailableModel, + logger: new EvalLogger(false), + runProcess, + }), + ).rejects.toThrow("EVAL_GROK_BUILD_ALWAYS_APPROVE must be true or false."); + expect(runProcess).not.toHaveBeenCalled(); + }, + ); + + it("returns a failed result for a non-zero exit without an end event", async () => { + const result = await runGrokBuildAgent({ + plan, + model: "grok-build/auto" as AvailableModel, + logger: new EvalLogger(false), + runProcess: scriptedRunner([], 1), + }); + expect(result._success).toBe(false); + expect(result.harnessStatus).toBe("sdk_error"); + expect(result.error).toContain("exited with code 1"); + }); +}); + +function scriptedRunner( + events: Array>, + exitCode = 0, + onArgs?: (args: string[]) => void, +): GrokBuildProcessRunner { + return async (input) => { + onArgs?.(input.args); + for (const event of events) await input.onStdoutLine(JSON.stringify(event)); + return { exitCode, signal: null }; + }; +} diff --git a/packages/evals/tests/framework/grokBuildToolAdapter.test.ts b/packages/evals/tests/framework/grokBuildToolAdapter.test.ts new file mode 100644 index 0000000000..c7aceb598b --- /dev/null +++ b/packages/evals/tests/framework/grokBuildToolAdapter.test.ts @@ -0,0 +1,85 @@ +import fsp from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { parse } from "smol-toml"; +import { + buildGrokBuildMcpConfig, + copyGrokBuildAuth, + GROK_BUILD_TOOL_SURFACES, + isGrokBuildMountToolName, + resolveGrokBuildAuthHome, + writeGrokBuildWorkspace, +} from "../../framework/grokBuildToolAdapter.js"; + +const tempDirs: string[] = []; + +afterEach(async () => { + await Promise.all(tempDirs.splice(0).map((dir) => fsp.rm(dir, { recursive: true, force: true }))); +}); + +describe("Grok Build tool adapter helpers", () => { + it("supports MCP mounts and converts their config to Grok TOML shape", () => { + expect(GROK_BUILD_TOOL_SURFACES).toEqual([ + "stagehand_facade", + "playwright_mcp", + "chrome_devtools_mcp", + ]); + expect( + buildGrokBuildMcpConfig({ + stagehand: { command: "node", args: ["server.mjs"], env: { TOKEN: "value" } }, + }), + ).toEqual({ + mcp_servers: { + stagehand: { + command: "node", + args: ["server.mjs"], + env: { TOKEN: "value" }, + startup_timeout_sec: 60, + tool_timeout_sec: 300, + }, + }, + }); + }); + + it("writes the MCP config to the isolated user scope", async () => { + const root = await fsp.mkdtemp(path.join(os.tmpdir(), "grok-build-workspace-test-")); + tempDirs.push(root); + const grokHome = path.join(root, "home", ".grok"); + const result = await writeGrokBuildWorkspace(grokHome, { + stagehand: { command: "node", args: ["server.mjs"] }, + }); + const userConfig = parse(await fsp.readFile(path.join(grokHome, "config.toml"), "utf8")); + expect(result.mcpConfigPath).toBe(path.join(grokHome, "config.toml")); + expect(userConfig).toMatchObject({ + mcp_servers: { stagehand: { command: "node", args: ["server.mjs"] } }, + cli: { auto_update: false, use_leader: false }, + subagents: { enabled: false }, + memory: { enabled: false }, + }); + }); + + it("copies cached auth only when no API key is supplied", async () => { + const root = await fsp.mkdtemp(path.join(os.tmpdir(), "grok-build-auth-test-")); + tempDirs.push(root); + const source = path.join(root, "source"); + const target = path.join(root, "target"); + await Promise.all([fsp.mkdir(source), fsp.mkdir(target)]); + await fsp.writeFile(path.join(source, "auth.json"), '{"token":"cached"}\n'); + expect(resolveGrokBuildAuthHome({ GROK_HOME: source })).toBe(source); + await expect(copyGrokBuildAuth({ GROK_HOME: source }, target)).resolves.toBe(true); + await expect(fsp.readFile(path.join(target, "auth.json"), "utf8")).resolves.toContain("cached"); + await fsp.rm(path.join(target, "auth.json")); + await expect( + copyGrokBuildAuth({ GROK_HOME: source, XAI_API_KEY: "secret" }, target), + ).resolves.toBe(false); + }); + + it("matches Grok MCP tool identities", () => { + const matches = (name: string) => isGrokBuildMountToolName(["stagehand"], name); + for (const name of ["stagehand.run", "stagehand__run", "mcp__stagehand__run"]) { + expect(matches(name)).toBe(true); + } + expect(matches("shell")).toBe(false); + }); +}); diff --git a/packages/evals/tests/framework/usageNormalization.test.ts b/packages/evals/tests/framework/usageNormalization.test.ts index 2b0b9092c4..24de388d1b 100644 --- a/packages/evals/tests/framework/usageNormalization.test.ts +++ b/packages/evals/tests/framework/usageNormalization.test.ts @@ -37,7 +37,7 @@ describe("normalizeUsage", () => { expect(usage.input_total + usage.output + usage.reasoning).toBe(1250); }); - it.each(["eve", "mastra", "pi", "codex", "cursor", "cursor_sdk"])( + it.each(["eve", "mastra", "pi", "codex", "cursor", "cursor_sdk", "grok_build"])( "treats zero-filled %s telemetry with no presence flag as unknown", (harness) => { const raw = { inputTokens: 0, outputTokens: 0, totalTokens: 0 }; diff --git a/packages/integrations/README.md b/packages/integrations/README.md index 829b970881..32990150e3 100644 --- a/packages/integrations/README.md +++ b/packages/integrations/README.md @@ -8,19 +8,21 @@ else, never restated. ## Structure -| Directory | What it is | -| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `core/` | The `@browserbasehq/stagehand-integrations` package: the facade contract, `StagehandFacadeTools`, the `stagehand-facade` stdio MCP bin, and the code-mode MCP host scaffold. | -| `claude-code/` | Claude Agent SDK example (programmatic MCP mount) plus a `.mcp.json` for connecting a running Claude Code CLI. | -| `codex/` | Codex SDK example (config-override MCP mount) plus a `config.toml` template for the codex CLI. | -| `crewai/` | Python CrewAI example over MCP/stdio (uv project). | -| `cursor/` | Cursor SDK example over MCP/stdio via the Cursor Agent SDK. | -| `deepagents/` | Python LangChain Deep Agents integrations: a local stdio MCP server and a Managed Deep Agents project with native tools. | -| `eve/` | Eve example with the tools bound natively via `defineTool` (Eve has no external-process tool mounting). | -| `fx/` | fx configuration templates and skill — fx consumes the facade via its user-global MCP config. | -| `mastra/` | Mastra example over MCP/stdio via Mastra's `MCPClient`. | -| `pi/` | Pi extension registering the tools natively (Pi ships without built-in MCP). | -| `vercel-ai/` | Vercel AI SDK example over MCP/stdio via `createMCPClient`. | +| Directory | What it is | +| ----------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `core/` | The `@browserbasehq/stagehand-integrations` package: the facade contract, `StagehandFacadeTools`, the `stagehand-facade` stdio MCP bin, and the code-mode MCP host scaffold. | +| `claude-code/` | Claude Agent SDK example (programmatic MCP mount) plus a `.mcp.json` for connecting a running Claude Code CLI. | +| `codex/` | Codex SDK example (config-override MCP mount) plus a `config.toml` template for the codex CLI. | +| `crewai/` | Python CrewAI example over MCP/stdio (uv project). | +| `cursor/` | Cursor SDK example over MCP/stdio via the Cursor Agent SDK. | +| `deepagents/` | Python LangChain Deep Agents integrations: a local stdio MCP server and a Managed Deep Agents project with native tools. | +| `eve/` | Eve example with the tools bound natively via `defineTool` (Eve has no external-process tool mounting). | +| `fx/` | fx configuration templates and skill — fx consumes the facade via its user-global MCP config. | +| `grok-build/` | Grok Build CLI configuration template and AGENTS.md guidance. The CLI consumes the facade as a project MCP server. | +| `grok-build-sdk/` | Grok Build CLI session adapter used by evals. | +| `mastra/` | Mastra example over MCP/stdio via Mastra's `MCPClient`. | +| `pi/` | Pi extension registering the tools natively (Pi ships without built-in MCP). | +| `vercel-ai/` | Vercel AI SDK example over MCP/stdio via `createMCPClient`. | Each example is a self-contained project: install, export `BROWSERBASE_API_KEY`, and run — see the directory's README. TypeScript examples consume `core/` as a workspace dependency; the Python projects resolve the published diff --git a/packages/integrations/grok-build-sdk/package.json b/packages/integrations/grok-build-sdk/package.json new file mode 100644 index 0000000000..b618cf48f0 --- /dev/null +++ b/packages/integrations/grok-build-sdk/package.json @@ -0,0 +1,34 @@ +{ + "name": "@browserbasehq/stagehand-integrations-grok-build-sdk", + "version": "4.0.1", + "private": true, + "description": "Grok Build CLI harness adapter for Stagehand integrations", + "files": [ + "dist" + ], + "type": "module", + "exports": { + ".": { + "types": "./dist/index.d.mts", + "import": "./dist/index.mjs" + } + }, + "scripts": { + "build": "tsdown", + "test": "pnpm run build && vitest run --root ../../.. packages/integrations/grok-build-sdk/tests", + "test:unit": "vitest run --root ../../.. packages/integrations/grok-build-sdk/tests", + "typecheck": "tsc --noEmit -p tsconfig.json" + }, + "dependencies": { + "@browserbasehq/stagehand-integrations": "workspace:*" + }, + "devDependencies": { + "@types/node": "catalog:", + "tsdown": "catalog:", + "typescript": "catalog:", + "vitest": "catalog:" + }, + "engines": { + "node": ">=24.0.0" + } +} diff --git a/packages/integrations/grok-build-sdk/src/index.ts b/packages/integrations/grok-build-sdk/src/index.ts new file mode 100644 index 0000000000..9df91886ba --- /dev/null +++ b/packages/integrations/grok-build-sdk/src/index.ts @@ -0,0 +1 @@ +export * from "./session.js"; diff --git a/packages/integrations/grok-build-sdk/src/session.ts b/packages/integrations/grok-build-sdk/src/session.ts new file mode 100644 index 0000000000..6aa1afce53 --- /dev/null +++ b/packages/integrations/grok-build-sdk/src/session.ts @@ -0,0 +1,490 @@ +import { spawn } from "node:child_process"; +import { + HarnessAdapterError, + sanitizeErrorMessage, + type HarnessLogger, +} from "@browserbasehq/stagehand-integrations/harness"; + +export type GrokBuildEvent = Record; + +export type GrokBuildProcessExit = { + exitCode: number | null; + signal: NodeJS.Signals | null; +}; + +export type GrokBuildProcessRunner = (input: { + command: string; + args: string[]; + cwd?: string; + env?: Record; + signal: AbortSignal; + onStdoutLine: (line: string) => void | Promise; + onStderr: (chunk: string) => void; +}) => Promise; + +export type GrokBuildSessionConfig = { + cwd?: string; + env?: Record; + binaryPath?: string; + alwaysApprove?: boolean; + maxTurns?: number; + sandbox?: string; + /** Appended with `grok --rules` (`--append-system-prompt`). */ + rules?: string; + extraArgs?: string[]; +}; + +export type GrokBuildTokenUsage = { + inputTokens: number; + outputTokens: number; + cachedInputTokens: number; + cacheCreationInputTokens: number; + reasoningOutputTokens: number; + totalTokens: number; + reported: boolean; +}; + +export type GrokBuildSessionResult = { + events: GrokBuildEvent[]; + endEvent?: GrokBuildEvent; + resultText: string; + status: "completed" | "max_turns" | "sdk_error"; + stopReason?: string; + tokenUsage: GrokBuildTokenUsage; + costUsd?: number; + exit?: GrokBuildProcessExit; + stderr: string; + iterationError?: unknown; +}; + +export type GrokBuildToolCallView = { + callId: string; + subtype: "started" | "completed"; + name?: string; + args: Record; + result?: unknown; + ok: boolean; + error?: string; +}; + +export const GROK_BUILD_BINARY = "grok"; +const STDERR_LIMIT = 64 * 1024; + +export function resolveGrokBuildBinary(override?: string): string { + return override ?? process.env.GROK_BUILD_PATH ?? GROK_BUILD_BINARY; +} + +export function normalizeGrokBuildModel(model: string): string | undefined { + if (model === "grok-build/auto" || model === "auto") return undefined; + return model.includes("/") ? model.slice(model.indexOf("/") + 1) : model; +} + +export function buildGrokBuildArgs(input: { + prompt: string; + model?: string; + session: GrokBuildSessionConfig; +}): string[] { + const { session } = input; + return [ + "-p", + input.prompt, + "--output-format", + "streaming-json", + ...(session.alwaysApprove === true ? ["--always-approve"] : []), + "--tools", + "search_tool,use_tool", + "--disallowed-tools", + "Agent", + "--no-plan", + "--no-subagents", + "--disable-web-search", + ...(session.cwd ? ["--cwd", session.cwd] : []), + ...(input.model ? ["--model", input.model] : []), + ...(session.maxTurns ? ["--max-turns", String(session.maxTurns)] : []), + ...(session.sandbox ? ["--sandbox", session.sandbox] : []), + ...(session.rules ? ["--rules", session.rules] : []), + ...(session.extraArgs ?? []), + ]; +} + +export function parseGrokBuildStreamLine(line: string): GrokBuildEvent | undefined { + const trimmed = line.trim(); + if (!trimmed) return undefined; + try { + const parsed: unknown = JSON.parse(trimmed); + return isRecord(parsed) ? parsed : undefined; + } catch { + return undefined; + } +} + +export function extractGrokBuildToolCall(event: GrokBuildEvent): GrokBuildToolCallView | undefined { + if (event.type === "tool_call") { + const rawInput = isRecord(event.rawInput) ? event.rawInput : {}; + const wrappedTool = event.toolName === "use_tool" && readString(rawInput.tool_name); + return { + callId: readString(event.toolCallId) ?? "", + subtype: "started", + name: wrappedTool || readString(event.toolName), + args: wrappedTool && isRecord(rawInput.tool_input) ? rawInput.tool_input : rawInput, + ok: true, + }; + } + if (event.type !== "tool_call_update") return undefined; + const status = readString(event.status); + if (!status || !["completed", "failed", "cancelled", "rejected"].includes(status)) + return undefined; + const ok = status === "completed"; + const result = event.rawOutput ?? event.content; + return { + callId: readString(event.toolCallId) ?? "", + subtype: "completed", + args: {}, + ...(result !== undefined && { result }), + ok, + ...(!ok && { error: stringifyError(result) || `tool call ${status}` }), + }; +} + +export const defaultGrokBuildProcessRunner: GrokBuildProcessRunner = async (input) => { + return new Promise((resolve, reject) => { + const child = spawn(input.command, input.args, { + ...(input.cwd && { cwd: input.cwd }), + ...(input.env && { env: input.env }), + stdio: ["ignore", "pipe", "pipe"], + }); + let stdoutBuffer = ""; + let lineQueue = Promise.resolve(); + let lineFailed = false; + let lineError: unknown; + let killTimer: NodeJS.Timeout | undefined; + let settled = false; + + const queueLine = (line: string): void => { + lineQueue = lineQueue + .then(() => { + if (!lineFailed) return input.onStdoutLine(line); + }) + .catch((error) => { + lineFailed = true; + lineError = error; + abort(); + }); + }; + const removeAbort = (): void => input.signal.removeEventListener("abort", abort); + const abort = (): void => { + if (child.exitCode !== null || child.signalCode !== null) return; + child.kill("SIGTERM"); + killTimer = setTimeout(() => { + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + }, 5_000); + killTimer.unref(); + }; + + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + stdoutBuffer += chunk; + const lines = stdoutBuffer.split(/\r?\n/u); + stdoutBuffer = lines.pop() ?? ""; + for (const line of lines) queueLine(line); + }); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => input.onStderr(chunk)); + child.on("error", (error: NodeJS.ErrnoException) => { + if (settled) return; + settled = true; + removeAbort(); + if (killTimer) clearTimeout(killTimer); + if (error.code === "ENOENT") { + reject( + new HarnessAdapterError( + "Grok Build harness requires the `grok` CLI (install `@xai-official/grok` globally, or set GROK_BUILD_PATH).", + { cause: error }, + ), + ); + return; + } + reject(error); + }); + child.on("close", (exitCode, signal) => { + if (settled) return; + settled = true; + removeAbort(); + if (killTimer) clearTimeout(killTimer); + if (stdoutBuffer) queueLine(stdoutBuffer); + lineQueue.then( + () => (lineFailed ? reject(lineError) : resolve({ exitCode, signal })), + (error) => reject(error), + ); + }); + + if (input.signal.aborted) abort(); + else input.signal.addEventListener("abort", abort, { once: true }); + }); +}; + +export async function runGrokBuildSession(input: { + prompt: string; + model: string; + signal?: AbortSignal; + logger: HarnessLogger; + session: GrokBuildSessionConfig; + runProcess?: GrokBuildProcessRunner; + onToolResult?: (toolName: string, view: GrokBuildToolCallView) => void | Promise; +}): Promise { + const events: GrokBuildEvent[] = []; + const controller = new AbortController(); + const forwardAbort = (): void => controller.abort(input.signal?.reason); + if (input.signal) { + if (input.signal.aborted) controller.abort(input.signal.reason); + else input.signal.addEventListener("abort", forwardAbort, { once: true }); + } + + const textParts: string[] = []; + const toolNames = new Map(); + let endEvent: GrokBuildEvent | undefined; + let errorEvent: GrokBuildEvent | undefined; + let stderr = ""; + let exit: GrokBuildProcessExit | undefined; + let iterationError: unknown; + + try { + const model = normalizeGrokBuildModel(input.model); + exit = await (input.runProcess ?? defaultGrokBuildProcessRunner)({ + command: resolveGrokBuildBinary(input.session.binaryPath), + args: buildGrokBuildArgs({ prompt: input.prompt, model, session: input.session }), + ...(input.session.cwd && { cwd: input.session.cwd }), + env: stringEnv({ ...process.env, ...input.session.env }), + signal: controller.signal, + onStdoutLine: async (line) => { + const parsed = parseGrokBuildStreamLine(line); + if (!parsed) return; + const event = deepSanitize(parsed) as GrokBuildEvent; + events.push(event); + logGrokBuildEvent(input.logger, event); + if (event.type === "text" && typeof event.data === "string") textParts.push(event.data); + if (event.type === "end") endEvent = event; + if (event.type === "error") errorEvent = event; + const view = extractGrokBuildToolCall(event); + if (view?.subtype === "started" && view.callId && view.name) { + toolNames.set(view.callId, view.name); + } + if (view?.subtype === "completed") { + const toolName = toolNames.get(view.callId) ?? view.name ?? "tool"; + await input.onToolResult?.(toolName, view); + } + }, + onStderr: (chunk) => { + stderr = `${stderr}${chunk}`.slice(-STDERR_LIMIT); + }, + }); + } catch (error) { + iterationError = new HarnessAdapterError( + sanitizeErrorMessage(stringifyError(error)) || "Grok Build session failed.", + ); + input.logger.warn({ + category: "grok_build", + message: `Grok Build stopped before a normal result: ${sanitizeErrorMessage(stringifyError(error))}`, + level: 0, + }); + } finally { + input.signal?.removeEventListener("abort", forwardAbort); + } + + stderr = sanitizeErrorMessage(stderr); + if (stderr) input.logger.log({ category: "grok_build", message: stderr, level: 1 }); + const externalAbortReason = input.signal?.aborted + ? stringifyError(input.signal.reason) || "Grok Build session aborted" + : undefined; + const stopReason = buildGrokBuildStopReason({ + endEvent, + errorEvent, + iterationError, + exit, + stderr, + externalAbortReason, + }); + const tokenUsage = readGrokBuildUsage(endEvent); + const costUsd = finiteNumber(endEvent?.total_cost_usd); + return { + events, + ...(endEvent && { endEvent }), + resultText: textParts.join(""), + status: resolveGrokBuildStatus(endEvent, errorEvent, iterationError, stopReason), + ...(stopReason && { stopReason: sanitizeErrorMessage(stopReason) }), + tokenUsage, + ...(costUsd !== undefined && { costUsd }), + ...(exit && { exit }), + stderr, + ...(iterationError !== undefined && { iterationError }), + }; +} + +export function readGrokBuildUsage(event: GrokBuildEvent | undefined): GrokBuildTokenUsage { + const usage = isRecord(event?.usage) ? event.usage : undefined; + const inputTokens = finiteNumber(usage?.input_tokens) ?? 0; + const cachedInputTokens = finiteNumber(usage?.cache_read_input_tokens) ?? 0; + const cacheCreationInputTokens = finiteNumber(usage?.cache_creation_input_tokens) ?? 0; + const outputTokens = finiteNumber(usage?.output_tokens) ?? 0; + const reasoningOutputTokens = finiteNumber(usage?.reasoning_tokens) ?? 0; + const totalTokens = + finiteNumber(usage?.total_tokens) ?? + inputTokens + cachedInputTokens + cacheCreationInputTokens + outputTokens; + return { + inputTokens, + outputTokens, + cachedInputTokens, + cacheCreationInputTokens, + reasoningOutputTokens, + totalTokens, + reported: usage !== undefined, + }; +} + +export function resolveGrokBuildStatus( + endEvent: GrokBuildEvent | undefined, + errorEvent: GrokBuildEvent | undefined, + iterationError: unknown, + stopReason?: string, +): "completed" | "max_turns" | "sdk_error" { + const reason = readString(endEvent?.stopReason) ?? ""; + if (iterationError || errorEvent || (stopReason && stopReason !== "max turns reached")) { + return "sdk_error"; + } + if (/max_turn/iu.test(reason) || stopReason === "max turns reached") return "max_turns"; + if (!isNormalStopReason(reason)) return "sdk_error"; + return "completed"; +} + +export function buildGrokBuildStopReason(input: { + endEvent?: GrokBuildEvent; + errorEvent?: GrokBuildEvent; + iterationError?: unknown; + exit?: GrokBuildProcessExit; + stderr: string; + externalAbortReason?: string; +}): string | undefined { + if (input.externalAbortReason) return input.externalAbortReason; + if (input.iterationError) return stringifyError(input.iterationError); + if (input.errorEvent) { + return readString(input.errorEvent.message) ?? "Grok Build returned an error event"; + } + const reason = readString(input.endEvent?.stopReason); + if (input.exit?.signal) return `grok exited with signal ${input.exit.signal}`; + if (input.exit && input.exit.exitCode !== 0) { + const lastLine = input.stderr + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter(Boolean) + .at(-1); + return `grok exited with code ${String(input.exit.exitCode ?? "unknown")}${lastLine ? `: ${lastLine}` : ""}`; + } + if (reason && /max_turn/iu.test(reason)) return "max turns reached"; + if (reason && !isNormalStopReason(reason)) { + return `Grok Build stopped with ${reason}`; + } + if (!input.endEvent) { + return "Grok Build exited without a terminal end event"; + } + if (!reason) return "Grok Build end event is missing its stop reason"; + return undefined; +} + +function isNormalStopReason(reason: string): boolean { + return reason === "end_turn" || reason === "EndTurn"; +} + +export function buildGrokBuildTranscript(events: GrokBuildEvent[]): string { + return events + .map((event) => summarizeGrokBuildEvent(event).detail) + .filter((detail): detail is string => Boolean(detail)) + .join("\n"); +} + +export function logGrokBuildEvent(logger: HarnessLogger, event: GrokBuildEvent): void { + const summary = summarizeGrokBuildEvent(event); + logger.log({ + category: "grok_build", + message: summary.message, + level: 1, + auxiliary: { + type: { value: readString(event.type) ?? "unknown", type: "string" }, + ...(summary.detail && { detail: { value: summary.detail, type: "string" } }), + }, + }); +} + +export function summarizeGrokBuildEvent(event: GrokBuildEvent): { + message: string; + detail?: string; +} { + const type = readString(event.type) ?? "unknown"; + if ((type === "text" || type === "thought") && typeof event.data === "string") { + const detail = sanitizeErrorMessage(event.data); + return { message: `${type}: ${clip(detail, 500)}`, detail }; + } + const tool = extractGrokBuildToolCall(event); + if (tool) { + return { + message: `tool: ${tool.name ?? tool.callId ?? "unknown"} ${tool.subtype}${tool.ok ? "" : " failed"}`, + detail: sanitizeOptional(safeJson(event)), + }; + } + if (type === "end") { + return { message: `end: ${readString(event.stopReason) ?? "done"}`, detail: safeJson(event) }; + } + return { message: `${type} event`, detail: sanitizeOptional(safeJson(event)) }; +} + +function deepSanitize(value: unknown): unknown { + if (typeof value === "string") return sanitizeErrorMessage(value); + if (Array.isArray(value)) return value.map(deepSanitize); + if (!isRecord(value)) return value; + return Object.fromEntries( + Object.entries(value).map(([key, child]) => [key, deepSanitize(child)]), + ); +} + +function sanitizeOptional(value: string | undefined): string | undefined { + return value === undefined ? undefined : sanitizeErrorMessage(value); +} + +function stringEnv( + env: NodeJS.ProcessEnv | Record, +): Record { + return Object.fromEntries( + Object.entries(env).filter((entry): entry is [string, string] => typeof entry[1] === "string"), + ); +} + +function finiteNumber(value: unknown): number | undefined { + const number = typeof value === "number" ? value : Number.NaN; + return Number.isFinite(number) ? number : undefined; +} + +function readString(value: unknown): string | undefined { + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function safeJson(value: unknown): string | undefined { + try { + return JSON.stringify(value); + } catch { + return undefined; + } +} + +export function stringifyError(value: unknown): string { + if (!value) return ""; + if (value instanceof Error) return value.message; + if (typeof value === "string") return value; + return safeJson(value) ?? "Unknown error"; +} + +export function clip(value: string, maxLength: number): string { + return value.length <= maxLength ? value : `${value.slice(0, maxLength - 1)}…`; +} diff --git a/packages/integrations/grok-build-sdk/tests/session.test.ts b/packages/integrations/grok-build-sdk/tests/session.test.ts new file mode 100644 index 0000000000..85c3f8231c --- /dev/null +++ b/packages/integrations/grok-build-sdk/tests/session.test.ts @@ -0,0 +1,326 @@ +import { describe, expect, it, vi } from "vitest"; +import { + buildGrokBuildArgs, + buildGrokBuildStopReason, + defaultGrokBuildProcessRunner, + extractGrokBuildToolCall, + normalizeGrokBuildModel, + parseGrokBuildStreamLine, + readGrokBuildUsage, + resolveGrokBuildBinary, + runGrokBuildSession, + type GrokBuildProcessRunner, +} from "../src/session.ts"; + +const logger = { + log: vi.fn(), + warn: vi.fn(), + error: vi.fn(), +}; + +describe("Grok Build CLI session", () => { + it("builds a restricted one-shot CLI invocation", () => { + expect( + buildGrokBuildArgs({ + prompt: "do it", + model: "grok-build", + session: { + cwd: "/workspace", + maxTurns: 12, + sandbox: "off", + rules: "Do not ask for clarification.", + }, + }), + ).toEqual([ + "-p", + "do it", + "--output-format", + "streaming-json", + "--tools", + "search_tool,use_tool", + "--disallowed-tools", + "Agent", + "--no-plan", + "--no-subagents", + "--disable-web-search", + "--cwd", + "/workspace", + "--model", + "grok-build", + "--max-turns", + "12", + "--sandbox", + "off", + "--rules", + "Do not ask for clarification.", + ]); + }); + + it.each([undefined, false, true])("respects alwaysApprove=%s", (alwaysApprove) => { + const args = buildGrokBuildArgs({ prompt: "do it", session: { alwaysApprove } }); + expect(args.includes("--always-approve")).toBe(alwaysApprove === true); + }); + + it("resolves the binary and normalizes harness model ids", () => { + expect(resolveGrokBuildBinary("/custom/grok")).toBe("/custom/grok"); + expect(normalizeGrokBuildModel("grok-build/auto")).toBeUndefined(); + expect(normalizeGrokBuildModel("xai/grok-build")).toBe("grok-build"); + expect(normalizeGrokBuildModel("grok-build")).toBe("grok-build"); + }); + + it("parses native streaming events and tool updates", () => { + expect(parseGrokBuildStreamLine("not-json")).toBeUndefined(); + expect(parseGrokBuildStreamLine('{"type":"text","data":"hi"}')).toEqual({ + type: "text", + data: "hi", + }); + expect( + extractGrokBuildToolCall({ + type: "tool_call", + toolCallId: "call-1", + toolName: "stagehand__run", + rawInput: { code: "return 1" }, + }), + ).toEqual({ + callId: "call-1", + subtype: "started", + name: "stagehand__run", + args: { code: "return 1" }, + ok: true, + }); + expect( + extractGrokBuildToolCall({ + type: "tool_call_update", + toolCallId: "call-1", + status: "completed", + rawOutput: { value: 1 }, + }), + ).toMatchObject({ + callId: "call-1", + subtype: "completed", + result: { value: 1 }, + ok: true, + }); + }); + + it("normalizes usage from the terminal end event", () => { + expect( + readGrokBuildUsage({ + type: "end", + usage: { + input_tokens: 10, + cache_read_input_tokens: 20, + cache_creation_input_tokens: 2, + output_tokens: 5, + reasoning_tokens: 3, + }, + }), + ).toEqual({ + inputTokens: 10, + outputTokens: 5, + cachedInputTokens: 20, + cacheCreationInputTokens: 2, + reasoningOutputTokens: 3, + totalTokens: 37, + reported: true, + }); + }); + + it("ignores partial updates and observes wrapped MCP tools only on completion", async () => { + const onToolResult = vi.fn(); + const start = { + type: "tool_call", + toolCallId: "mcp-1", + toolName: "use_tool", + rawInput: { tool_name: "stagehand__run", tool_input: { code: "return 1" } }, + }; + expect(extractGrokBuildToolCall(start)).toMatchObject({ + name: "stagehand__run", + args: { code: "return 1" }, + }); + const partial = { + type: "tool_call_update", + toolCallId: "mcp-1", + status: null, + content: [], + rawOutput: null, + }; + expect(extractGrokBuildToolCall(partial)).toBeUndefined(); + expect(extractGrokBuildToolCall({ ...partial, status: undefined })).toBeUndefined(); + await runGrokBuildSession({ + prompt: "do it", + model: "grok-build/grok-4.6", + logger, + session: {}, + onToolResult, + runProcess: scriptedRunner([ + start, + partial, + { type: "tool_call_update", toolCallId: "mcp-1", status: "completed", rawOutput: "1" }, + { type: "end", stopReason: "end_turn" }, + ]), + }); + expect(onToolResult).toHaveBeenCalledTimes(1); + expect(onToolResult).toHaveBeenCalledWith( + "stagehand__run", + expect.objectContaining({ result: "1" }), + ); + }); + + it("runs the stream, joins text, and reports usage and cost", async () => { + const onToolResult = vi.fn(); + const result = await runGrokBuildSession({ + prompt: "do it", + model: "grok-build/auto", + logger, + session: { cwd: "/workspace" }, + runProcess: scriptedRunner([ + { type: "text", data: "EVAL_RESULT: " }, + { + type: "tool_call", + toolCallId: "call-1", + toolName: "stagehand__snapshot", + rawInput: {}, + }, + { + type: "tool_call_update", + toolCallId: "call-1", + status: "completed", + rawOutput: "snapshot", + }, + { type: "text", data: '{"success":true,"summary":"done","finalAnswer":"ok"}' }, + { + type: "end", + stopReason: "end_turn", + usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15 }, + total_cost_usd: 0.02, + num_turns: 2, + }, + ]), + onToolResult, + }); + + expect(result.status).toBe("completed"); + expect(result.resultText).toContain('EVAL_RESULT: {"success":true'); + expect(result.tokenUsage.totalTokens).toBe(15); + expect(result.costUsd).toBe(0.02); + expect(onToolResult).toHaveBeenCalledWith( + "stagehand__snapshot", + expect.objectContaining({ subtype: "completed" }), + ); + }); + + it("fails a non-zero exit without an end event", async () => { + const result = await runGrokBuildSession({ + prompt: "do it", + model: "grok-build/auto", + logger, + session: {}, + runProcess: scriptedRunner([], 1), + }); + expect(result.status).toBe("sdk_error"); + expect(result.stopReason).toContain("exited with code 1"); + }); + + it.each(["end_turn", "max_turn_requests"])( + "fails a non-zero exit after %s", + async (stopReason) => { + const result = await runGrokBuildSession({ + prompt: "do it", + model: "grok-build/auto", + logger, + session: {}, + runProcess: scriptedRunner([{ type: "end", stopReason }], 1), + }); + expect(result.status).toBe("sdk_error"); + expect(result.stopReason).toContain("exited with code 1"); + }, + ); + + it("reports signal termination even after an end event", async () => { + const result = await runGrokBuildSession({ + prompt: "do it", + model: "grok-build/auto", + logger, + session: {}, + runProcess: async (input) => { + await input.onStdoutLine(JSON.stringify({ type: "end", stopReason: "end_turn" })); + return { exitCode: null, signal: "SIGTERM" }; + }, + }); + expect(result.status).toBe("sdk_error"); + expect(result.stopReason).toContain("SIGTERM"); + }); + + it.each(["max_tokens", "refusal", "cancelled", "unknown_reason"])( + "does not report %s as normal completion", + async (stopReason) => { + const result = await runGrokBuildSession({ + prompt: "do it", + model: "grok-build/auto", + logger, + session: {}, + runProcess: scriptedRunner([{ type: "end", stopReason }]), + }); + expect(result.status).toBe("sdk_error"); + expect(result.stopReason).toContain(stopReason); + }, + ); + + it.each(["end_turn", "EndTurn"])("accepts normal completion %s", async (stopReason) => { + const result = await runGrokBuildSession({ + prompt: "do it", + model: "grok-build/auto", + logger, + session: {}, + runProcess: scriptedRunner([{ type: "end", stopReason }]), + }); + expect(result.status).toBe("completed"); + expect(result.stopReason).toBeUndefined(); + }); + + it("rejects an end event without a stop reason", async () => { + const result = await runGrokBuildSession({ + prompt: "do it", + model: "grok-build/auto", + logger, + session: {}, + runProcess: scriptedRunner([{ type: "end" }]), + }); + expect(result.status).toBe("sdk_error"); + expect(result.stopReason).toContain("missing its stop reason"); + }); + + it("terminates the child and rejects when a stream callback fails", async () => { + await expect( + defaultGrokBuildProcessRunner({ + command: process.execPath, + args: ["-e", "console.log('event'); setInterval(() => {}, 1000)"], + signal: new AbortController().signal, + onStdoutLine: async () => { + throw new Error("callback failed"); + }, + onStderr: () => {}, + }), + ).rejects.toThrow("callback failed"); + }); + + it("reports max-turn stops without treating them as SDK errors", () => { + expect( + buildGrokBuildStopReason({ + endEvent: { type: "end", stopReason: "max_turn_requests" }, + stderr: "", + }), + ).toBe("max turns reached"); + }); +}); + +function scriptedRunner( + events: Array>, + exitCode = 0, +): GrokBuildProcessRunner { + return async (input) => { + for (const event of events) await input.onStdoutLine(JSON.stringify(event)); + return { exitCode, signal: null }; + }; +} diff --git a/packages/integrations/grok-build-sdk/tsconfig.json b/packages/integrations/grok-build-sdk/tsconfig.json new file mode 100644 index 0000000000..3571933603 --- /dev/null +++ b/packages/integrations/grok-build-sdk/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "module": "NodeNext", + "moduleResolution": "NodeNext", + "target": "ES2022", + "types": ["node"], + "rootDir": ".", + "noEmit": true, + "skipLibCheck": true + }, + "include": ["src/**/*.ts", "tests/**/*.ts"], + "exclude": ["dist", "node_modules"] +} diff --git a/packages/integrations/grok-build-sdk/tsdown.config.ts b/packages/integrations/grok-build-sdk/tsdown.config.ts new file mode 100644 index 0000000000..49fbead000 --- /dev/null +++ b/packages/integrations/grok-build-sdk/tsdown.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from "tsdown"; + +export default defineConfig({ + entry: { index: "src/index.ts" }, + format: ["esm"], + dts: true, + sourcemap: true, + clean: true, + deps: { neverBundle: ["@browserbasehq/stagehand-integrations"] }, + outputOptions: { minify: false }, +}); diff --git a/packages/integrations/grok-build/.grok/config.toml b/packages/integrations/grok-build/.grok/config.toml new file mode 100644 index 0000000000..3981c0ac37 --- /dev/null +++ b/packages/integrations/grok-build/.grok/config.toml @@ -0,0 +1,15 @@ +# Grok Build project MCP entry for the Stagehand facade. Adjust the absolute +# path and credentials before copying this file into another project. + +[mcp_servers.stagehand] +command = "node" +args = [ + "/absolute/path/to/stagehand/packages/integrations/core/dist/facade/stdio-server.mjs", + "--max-screenshot-base64-bytes=60000", +] +startup_timeout_sec = 60 +tool_timeout_sec = 300 + +[mcp_servers.stagehand.env] +STAGEHAND_BROWSER = "browserbase" +BROWSERBASE_API_KEY = "bb_live_..." diff --git a/packages/integrations/grok-build/AGENTS.md b/packages/integrations/grok-build/AGENTS.md new file mode 100644 index 0000000000..85938290b0 --- /dev/null +++ b/packages/integrations/grok-build/AGENTS.md @@ -0,0 +1,9 @@ +# Stagehand browser tools in Grok Build + +The `stagehand` MCP server exposes `stagehand__run`, `stagehand__snapshot`, and +`stagehand__screenshot`. Use only those tools for browser work. + +There is no separate navigate or start tool. Open URLs with `stagehand__run`, for example +`await page.goto("https://example.com"); return { url: await page.url() };`. Use +`stagehand__snapshot` for the accessibility tree and element IDs. Use `stagehand__screenshot` +only when pixels matter. Never launch another browser or use shell commands for browsing. diff --git a/packages/integrations/grok-build/README.md b/packages/integrations/grok-build/README.md new file mode 100644 index 0000000000..79cc1ede0b --- /dev/null +++ b/packages/integrations/grok-build/README.md @@ -0,0 +1,65 @@ +# Grok Build CLI + Stagehand facade over MCP/stdio + +Grok Build's `grok` CLI consumes the Stagehand facade (`run` / `snapshot` / `screenshot`) as a +project MCP server through `.grok/config.toml`. + + + +## Setup + +Use Node.js 24 or newer. From the repository root, build the integrations package first: + +```bash +pnpm install --frozen-lockfile +pnpm exec turbo run build --filter @browserbasehq/stagehand-integrations +npm install --global @xai-official/grok +grok login +# or: export XAI_API_KEY=... +``` + +## Configure + +Copy `.grok/config.toml` from this directory to your project, then replace the facade path and +Browserbase key placeholders. Grok merges project MCP configuration over its user settings. + +## Run + +Run from the configured project so Grok sees both `.grok/config.toml` and `AGENTS.md`: + +```bash +grok \ + --output-format streaming-json \ + --always-approve \ + --tools search_tool,use_tool \ + --disallowed-tools Agent \ + --no-plan \ + --no-subagents \ + --disable-web-search \ + -p "Use the stagehand MCP tools: open https://example.com, snapshot it, and report the heading citing the snapshot ID." +``` + +The example uses `--always-approve` to let Grok run tools without asking for approval. +Remove this flag to use Grok's default permissions. This command runs without an interactive +session, so it cannot accept approval responses. Tasks that require approval may fail. + +For programmatic runs, set `GrokBuildSessionConfig.alwaysApprove` to `true` to add the flag. +When the option is `false` or unset, the adapter omits it. The adapter closes stdin and +does not support interactive approval. + +The eval harness uses the same CLI path with configuration in an isolated temporary Grok home: + +```bash +evals run b:webvoyager --harness grok_build --tool stagehand_facade -l 1 -t 1 -e browserbase +``` + +Set `EVAL_GROK_BUILD_PATH` to override the binary, `EVAL_GROK_BUILD_MAX_TURNS` to change the +50-turn default, or `EVAL_GROK_BUILD_SANDBOX` to pass a Grok sandbox profile. + +Evals enable auto-approval by default for unattended runs. Set +`EVAL_GROK_BUILD_ALWAYS_APPROVE=false` to omit the flag and retain Grok's native permission +policy. Only `true` and `false` are accepted. The effective setting is recorded in +`harnessConfiguration.alwaysApprove`. + +The shared Browserbase runtime enables verified mode by default. If your project does not +support it, set `EVAL_BROWSERBASE_VERIFIED=0`. The verifier uses `google/gemini-3.5-flash` +and a Google API key (`GOOGLE_GENERATIVE_AI_API_KEY` or `GEMINI_API_KEY`). diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 62f57a2a6e..5ce71d4a49 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,4 +1,5 @@ --- + lockfileVersion: '9.0' importers: @@ -641,6 +642,9 @@ importers: '@browserbasehq/stagehand-integrations-gemini-cua-sdk': specifier: workspace:* version: link:../integrations/gemini-cua-sdk + '@browserbasehq/stagehand-integrations-grok-build-sdk': + specifier: workspace:* + version: link:../integrations/grok-build-sdk '@browserbasehq/stagehand-integrations-mastra-sdk': specifier: workspace:* version: link:../integrations/mastra-sdk @@ -689,6 +693,9 @@ importers: sharp: specifier: 0.35.4 version: 0.35.4(@types/node@24.13.2) + smol-toml: + specifier: 'catalog:' + version: 1.7.1 stagehand-v3: specifier: npm:@browserbasehq/stagehand@3.7.1 version: '@browserbasehq/stagehand@3.7.1(playwright-core@1.56.1)(supports-color@8.1.1)(zod@4.4.3)' @@ -1153,6 +1160,25 @@ importers: specifier: 'catalog:' version: 4.1.11(@opentelemetry/api@1.9.1)(@types/node@24.13.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0)) + packages/integrations/grok-build-sdk: + dependencies: + '@browserbasehq/stagehand-integrations': + specifier: workspace:* + version: link:../core + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 24.13.2 + tsdown: + specifier: 'catalog:' + version: 0.22.3(publint@0.3.21)(tsx@4.23.1)(typescript@5.9.3) + typescript: + specifier: 'catalog:' + version: 5.9.3 + vitest: + specifier: 'catalog:' + version: 4.1.11(@opentelemetry/api@1.9.1)(@types/node@24.13.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0)) + packages/integrations/mastra: dependencies: '@ai-sdk/openai': diff --git a/turbo.json b/turbo.json index 87bdcf61eb..09decd09ff 100644 --- a/turbo.json +++ b/turbo.json @@ -94,6 +94,11 @@ "inputs": ["$TURBO_DEFAULT$", "!dist/**"], "outputs": ["dist/**"] }, + "@browserbasehq/stagehand-integrations-grok-build-sdk#build": { + "dependsOn": ["^build"], + "inputs": ["$TURBO_DEFAULT$", "!dist/**"], + "outputs": ["dist/**"] + }, "@browserbasehq/stagehand-evals#build": { "dependsOn": ["^build"], "inputs": ["$TURBO_DEFAULT$", "!dist/**"], @@ -247,6 +252,16 @@ "$TURBO_ROOT$/vitest.config.ts" ] }, + "@browserbasehq/stagehand-integrations-grok-build-sdk#test:unit": { + "dependsOn": ["^build", "@browserbasehq/stagehand-integrations-grok-build-sdk#build"], + "inputs": [ + "$TURBO_DEFAULT$", + "tests/**", + "src/**", + "**/*.test.ts", + "$TURBO_ROOT$/vitest.config.ts" + ] + }, "@browserbasehq/eve#typecheck": { "dependsOn": ["^build"], "inputs": [ @@ -303,6 +318,9 @@ "@browserbasehq/stagehand-integrations-cursor-sdk#typecheck": { "dependsOn": ["^build"] }, + "@browserbasehq/stagehand-integrations-grok-build-sdk#typecheck": { + "dependsOn": ["^build"] + }, "@browserbasehq/stagehand-docs#typecheck": {}, "test:unit": { "dependsOn": ["^build"], diff --git a/vitest.config.ts b/vitest.config.ts index 5f33f6f7a5..9b62c05f3b 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -21,6 +21,7 @@ export default defineConfig({ "packages/integrations/deepagents-sdk/tests/**/*.test.ts", "packages/integrations/fx-sdk/tests/**/*.test.ts", "packages/integrations/cursor-sdk/tests/**/*.test.ts", + "packages/integrations/grok-build-sdk/tests/**/*.test.ts", "packages/extension/tests/**/*.test.ts", "packages/sdk-ts/tests/**/*.test.ts", "packages/extension/understudy/**/*.test.ts",