diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ce240ccc3..53819bc098 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -267,6 +267,7 @@ jobs: packages/integrations/deepagents-sdk/dist/** packages/integrations/fx-sdk/dist/** packages/integrations/cursor-sdk/dist/** + packages/integrations/grok-build-sdk/dist/** packages/evals/dist/** retention-days: 1 diff --git a/packages/docs/docs.json b/packages/docs/docs.json index a62b1e8fef..a8975145ff 100644 --- a/packages/docs/docs.json +++ b/packages/docs/docs.json @@ -51,6 +51,7 @@ "v4/integrations/overview", "v4/integrations/claude-code", "v4/integrations/codex", + "v4/integrations/grok-build", "v4/integrations/eve", "v4/integrations/deep-agents", "v4/integrations/crewai", diff --git a/packages/docs/images/integrations/grok-build.svg b/packages/docs/images/integrations/grok-build.svg new file mode 100644 index 0000000000..7342b543cd --- /dev/null +++ b/packages/docs/images/integrations/grok-build.svg @@ -0,0 +1 @@ +Grok Build diff --git a/packages/docs/v4/integrations/grok-build.mdx b/packages/docs/v4/integrations/grok-build.mdx new file mode 100644 index 0000000000..913583a39d --- /dev/null +++ b/packages/docs/v4/integrations/grok-build.mdx @@ -0,0 +1,85 @@ +--- +title: "Grok Build" +description: "Give Grok Build persistent Stagehand browser tools through its CLI and project MCP configuration." +--- + +Grok Build can load the Stagehand facade as a project MCP server. The facade owns one persistent browser and exposes `run`, `snapshot`, and `screenshot` for the full CLI run. The integration uses Grok's headless streaming output directly. + + +Stagehand ships this experimental integration from the repository rather than publishing it as a standalone adapter. + + +## Prerequisites + +- Node.js 24 or newer +- pnpm 11.10.0 +- The Grok Build CLI and either `XAI_API_KEY` or an existing `grok login` +- Google Chrome for local browser mode + +## Quickstart + + + +```bash +git clone https://github.com/browserbase/stagehand.git +cd stagehand +pnpm install --frozen-lockfile +pnpm exec turbo run build --filter @browserbasehq/stagehand-integrations +``` + + +```bash +npm install --global @xai-official/grok +grok login +# or: export XAI_API_KEY="your-xai-api-key" +``` + + +Copy `packages/integrations/grok-build/.grok/config.toml` into the project where Grok will run. Replace the absolute facade path and browser credentials. + + +```bash +cd packages/integrations/grok-build +grok -p \ + --output-format streaming-json \ + --always-approve \ + --tools search_tool,use_tool \ + --disallowed-tools Agent \ + --no-plan \ + --no-subagents \ + --disable-web-search \ + "Open https://example.com, snapshot it, and report the heading." +``` + + + +## Eval harness + +The registered `grok_build` harness follows the same CLI pattern as the Cursor harness. It creates an isolated temporary home and workspace, copies cached Grok authentication only when no API key is present, writes the selected Stagehand MCP mount to project configuration, runs `grok -p --output-format streaming-json`, and maps native tool and usage events into the shared verifier trajectory. + +```bash +evals run b:webvoyager \ + --harness grok_build \ + --tool stagehand_facade \ + -l 1 -t 1 -e browserbase +``` + +| Variable | Purpose | +| --- | --- | +| `XAI_API_KEY` | Grok credential. The Stagehand MCP server receives only its own configured environment. | +| `GROK_HOME` | Source for cached `auth.json` when `XAI_API_KEY` is unset. | +| `EVAL_GROK_BUILD_PATH` | Optional path to the `grok` binary for eval runs. | +| `EVAL_GROK_BUILD_MAX_TURNS` | Grok turn limit. Defaults to 50. | +| `EVAL_GROK_BUILD_SANDBOX` | Optional Grok sandbox profile. | +| `STAGEHAND_BROWSER` | `local` or `browserbase`. | +| `BROWSERBASE_API_KEY` | Required for Browserbase. | + +The harness supports `stagehand_facade`, `playwright_mcp`, and `chrome_devtools_mcp`. It restricts Grok to its MCP discovery and invocation tools, disables subagents, plan mode, and web search, and asks Grok not to edit repository files. The temporary Grok home disables compatibility MCP imports, memory, and background leader reuse so user configuration cannot add unrelated tools. + + +`run` executes model-authored JavaScript in the browser. Use Browserbase for untrusted tasks and review the [integration security boundary](/v4/integrations/overview#security-boundary). + + + + Open the CLI configuration and eval harness instructions. + diff --git a/packages/docs/v4/integrations/overview.mdx b/packages/docs/v4/integrations/overview.mdx index 9aaee85459..a708f31cf0 100644 --- a/packages/docs/v4/integrations/overview.mdx +++ b/packages/docs/v4/integrations/overview.mdx @@ -1,7 +1,7 @@ --- title: "Integrations" sidebarTitle: "Overview" -description: "Connect Claude Code, Codex, CrewAI, Deep Agents, Eve, Mastra, fx, Pi, or the Vercel AI SDK to a persistent Stagehand browser." +description: "Connect Claude Code, Codex, Grok Build, CrewAI, Deep Agents, Eve, Mastra, fx, Pi, or the Vercel AI SDK to a persistent Stagehand browser." --- Each integration gives your agent one persistent browser and three tools: `run`, `snapshot`, and `screenshot`. Your agent decides how to navigate and interact while Stagehand manages the browser session. @@ -19,6 +19,9 @@ Stagehand ships these experimental integrations from the monorepo and does not p Give a Codex agent persistent Stagehand browser tools over MCP/stdio. + + Give a Grok Build agent persistent Stagehand browser tools. + Give Vercel's framework for building durable agents native Stagehand tools. @@ -78,7 +81,7 @@ Every integration exposes the same browser capabilities. The tools share a browser for the lifetime of the integration's client session. A navigation performed by `run` is visible to the next `snapshot`, and authentication and page state remain available across calls. -The Claude Code, Codex, CrewAI, Mastra, fx, Vercel AI SDK, and local Deep Agents examples keep one MCP client session open so the stdio server and browser stay alive. Eve, Pi, and Managed Deep Agents bind equivalent tools in-process. +The Claude Code, Codex, Grok Build, CrewAI, Mastra, fx, Vercel AI SDK, and local Deep Agents examples keep one MCP client session open so the stdio server and browser stay alive. Eve, Pi, and Managed Deep Agents bind equivalent tools in-process. The private [`core/` workspace package](https://github.com/browserbase/stagehand/tree/main/packages/integrations/core) owns the shared TypeScript tool contract, browser runtime, native bindings, and stdio MCP server. The TypeScript integrations and CrewAI use this package. @@ -88,7 +91,7 @@ Do not create a new MCP process for every tool call. Doing so starts a new brows ## Run the integrations from source -Claude Code, Codex, CrewAI, Mastra, fx, and the Vercel AI SDK use the shared TypeScript Stagehand facade MCP server. Eve and Pi use native in-process bindings from the same package. These integrations require Node.js 24 or newer and [pnpm](https://pnpm.io/installation) 11.10.0. CrewAI also requires Python 3.11–3.13 and [uv](https://docs.astral.sh/uv/). +Claude Code, Codex, Grok Build, CrewAI, Mastra, fx, and the Vercel AI SDK use the shared TypeScript Stagehand facade MCP server. Eve and Pi use native in-process bindings from the same package. These integrations require Node.js 24 or newer and [pnpm](https://pnpm.io/installation) 11.10.0. CrewAI also requires Python 3.11–3.13 and [uv](https://docs.astral.sh/uv/). diff --git a/packages/evals/framework/benchHarness.ts b/packages/evals/framework/benchHarness.ts index 99c8f3d610..40e83a3eb4 100644 --- a/packages/evals/framework/benchHarness.ts +++ b/packages/evals/framework/benchHarness.ts @@ -22,6 +22,8 @@ import { runFxAgent } from "./fxRunner.js"; import { FX_TOOL_SURFACES, prepareFxToolAdapter } from "./fxToolAdapter.js"; import { runCursorAgent } from "./cursorRunner.js"; import { CURSOR_TOOL_SURFACES, prepareCursorToolAdapter } from "./cursorToolAdapter.js"; +import { runGrokBuildAgent } from "./grokBuildRunner.js"; +import { GROK_BUILD_TOOL_SURFACES, prepareGrokBuildToolAdapter } from "./grokBuildToolAdapter.js"; import { buildExternalHarnessTaskPlan, type ExternalHarnessTaskPlan, @@ -346,6 +348,14 @@ export const cursorHarness = defineExternalHarness({ runAgent: runCursorAgent, }); +export const grokBuildHarness = defineExternalHarness({ + harness: "grok_build", + supportedToolSurfaces: GROK_BUILD_TOOL_SURFACES, + defaultModels: ["grok-build/auto" as AvailableModel], + prepareToolAdapter: prepareGrokBuildToolAdapter, + runAgent: runGrokBuildAgent, +}); + const harnessRegistry = new Map([ ["stagehand", stagehandHarness], ["claude_code", claudeCodeHarness], @@ -356,6 +366,7 @@ const harnessRegistry = new Map([ ["deepagents", deepagentsHarness], ["fx", fxHarness], ["cursor", cursorHarness], + ["grok_build", grokBuildHarness], ]); export function registerBenchHarness(harness: BenchHarness): () => void { diff --git a/packages/evals/framework/grokBuildRunner.ts b/packages/evals/framework/grokBuildRunner.ts new file mode 100644 index 0000000000..4debc2ab2d --- /dev/null +++ b/packages/evals/framework/grokBuildRunner.ts @@ -0,0 +1,187 @@ +import { + buildGrokBuildTranscript, + extractGrokBuildToolCall, + runGrokBuildSession, + stringifyError, + type GrokBuildProcessRunner, + type GrokBuildTokenUsage, +} from "@browserbasehq/stagehand-integrations-grok-build-sdk"; +import type { AvailableModel } from "stagehand-v3"; +import type { EvalLogger } from "../logger.js"; +import type { ExternalHarnessTaskPlan } from "./externalHarnessPlan.js"; +import type { PreparedGrokBuildToolAdapter } from "./grokBuildToolAdapter.js"; +import { grokBuildAdapter } from "./harnesses/grokBuildAdapter.js"; +import { + buildExternalHarnessPrompt, + metricValue, + parseEvalResult, + runExternalHarnessTask, + type ExternalHarnessToolAdapterLike, + type MetricValue, + type ParsedEvalResult, +} from "./harnesses/externalRunner.js"; +import type { TaskResult } from "./types.js"; +import type { ExternalHarnessVerifierConfig } from "./verifierAdapter.js"; + +export type { GrokBuildProcessRunner } from "@browserbasehq/stagehand-integrations-grok-build-sdk"; + +export interface GrokBuildRunnerInput { + plan: ExternalHarnessTaskPlan; + model: AvailableModel; + logger: EvalLogger; + toolAdapter?: PreparedGrokBuildToolAdapter; + signal?: AbortSignal; + runProcess?: GrokBuildProcessRunner; + verifier?: ExternalHarnessVerifierConfig; +} + +export interface ParsedGrokBuildResult extends ParsedEvalResult {} + +const MCP_ONLY_LINE = + "Your only browser access is the MCP server configured in this workspace. Never launch a browser yourself or run shell commands to browse."; + +function composeGrokBuildToolInstructions(toolInstructions?: string): string { + return [ + toolInstructions ?? "Use the available browser tools to complete the task.", + MCP_ONLY_LINE, + "Do not edit repository files.", + ].join("\n"); +} + +export function buildGrokBuildPrompt( + plan: ExternalHarnessTaskPlan, + toolInstructions?: string, +): string { + return buildExternalHarnessPrompt({ + plan, + toolInstructions: composeGrokBuildToolInstructions(toolInstructions), + resultContract: "marker", + }); +} + +export function parseGrokBuildResult(raw: string): ParsedGrokBuildResult { + return parseEvalResult(raw); +} + +export async function runGrokBuildAgent({ + plan, + model, + logger, + toolAdapter, + signal, + runProcess, + verifier, +}: GrokBuildRunnerInput): Promise { + const adapterLike: ExternalHarnessToolAdapterLike = { + promptInstructions: composeGrokBuildToolInstructions(toolAdapter?.promptInstructions), + captureEvidence: toolAdapter?.captureEvidence, + drainStepObservations: toolAdapter?.drainStepObservations, + observedToolMatcher: toolAdapter?.observedToolMatcher, + }; + return runExternalHarnessTask({ + harness: "grok_build", + plan, + logger, + toolAdapter: adapterLike, + verifier, + resultContract: "marker", + fallbackErrorMessage: "Grok Build did not report success", + parseResult: parseGrokBuildResult, + runSession: async (prompt) => { + const sessionResult = await runGrokBuildSession({ + prompt, + model, + logger, + signal, + runProcess, + session: { + ...(toolAdapter?.cwd && { cwd: toolAdapter.cwd }), + ...(toolAdapter?.env && { env: toolAdapter.env }), + ...(process.env.EVAL_GROK_BUILD_PATH && { + binaryPath: process.env.EVAL_GROK_BUILD_PATH, + }), + maxTurns: readGrokBuildMaxTurns(), + ...(process.env.EVAL_GROK_BUILD_SANDBOX && { + sandbox: process.env.EVAL_GROK_BUILD_SANDBOX, + }), + }, + onToolResult: toolAdapter?.onToolResult + ? (name) => toolAdapter.onToolResult!(name) + : undefined, + }); + const usage = sessionResult.tokenUsage; + return { + raw: sessionResult, + resultText: sessionResult.resultText, + transcriptText: buildGrokBuildTranscript(sessionResult.events), + iterationError: sessionResult.iterationError, + status: sessionResult.status, + stopReason: + sessionResult.stopReason || + (sessionResult.status === "sdk_error" + ? stringifyError(sessionResult.iterationError) || undefined + : undefined), + usage: { + inputTokens: usage.inputTokens, + outputTokens: usage.outputTokens, + totalTokens: usage.totalTokens, + ...(usage.reported && { + cachedInputTokens: usage.cachedInputTokens, + cacheCreationInputTokens: usage.cacheCreationInputTokens, + reasoningOutputTokens: usage.reasoningOutputTokens, + }), + }, + costUsd: sessionResult.costUsd, + metrics: buildGrokBuildMetrics(usage, sessionResult.endEvent, sessionResult.events), + }; + }, + toTrajectory: ( + { raw, parsed, finalObservation, stepObservations, observedToolName, status }, + taskSpec, + ) => + grokBuildAdapter.fromHarnessResult( + { + events: raw.events, + ...(finalObservation && { finalObservation }), + ...(stepObservations?.length && { stepObservations }), + ...(observedToolName && { observedToolName }), + finalAnswer: parsed.finalAnswer ?? raw.resultText, + status, + usage: { + input_tokens: raw.tokenUsage.inputTokens, + output_tokens: raw.tokenUsage.outputTokens, + cached_input_tokens: raw.tokenUsage.cachedInputTokens, + reasoning_tokens: raw.tokenUsage.reasoningOutputTokens, + }, + }, + taskSpec, + ), + }); +} + +export function readGrokBuildMaxTurns(): number { + for (const key of ["EVAL_GROK_BUILD_MAX_TURNS", "AGENT_EVAL_MAX_STEPS"]) { + const parsed = Number.parseInt(process.env[key] ?? "", 10); + if (Number.isFinite(parsed) && parsed > 0) return parsed; + } + return 50; +} + +function buildGrokBuildMetrics( + usage: GrokBuildTokenUsage, + endEvent: Record | undefined, + events: Array>, +): Record { + const toolSteps = events.filter( + (event) => extractGrokBuildToolCall(event)?.subtype === "completed", + ).length; + return { + grok_build_input_tokens: metricValue(usage.inputTokens), + grok_build_output_tokens: metricValue(usage.outputTokens), + grok_build_total_tokens: metricValue(usage.totalTokens), + grok_build_cached_input_tokens: metricValue(usage.cachedInputTokens), + grok_build_reasoning_tokens: metricValue(usage.reasoningOutputTokens), + grok_build_num_turns: metricValue(endEvent?.num_turns), + grok_build_tool_steps: metricValue(toolSteps), + }; +} diff --git a/packages/evals/framework/grokBuildToolAdapter.ts b/packages/evals/framework/grokBuildToolAdapter.ts new file mode 100644 index 0000000000..855674055e --- /dev/null +++ b/packages/evals/framework/grokBuildToolAdapter.ts @@ -0,0 +1,315 @@ +import fsp from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { stringify } from "smol-toml"; +import type { ProbeEvidence } from "stagehand-v3"; +import type { StartupProfile, ToolSurface } from "../core/contracts/tool.js"; +import { EvalsError } from "../errors.js"; +import type { EvalLogger } from "../logger.js"; +import { startAgentToolRuntime } from "./agentToolRuntime.js"; +import type { ExternalHarnessTaskPlan } from "./externalHarnessPlan.js"; +import { resolveStartupProfile, resolveToolSurface } from "./harnesses/toolSurfaceResolution.js"; +import { ObservationRecorder, type StepObservation } from "./observationRecorder.js"; + +export interface GrokBuildToolAdapterInput { + toolSurface?: ToolSurface; + startupProfile?: StartupProfile; + environment: "LOCAL" | "BROWSERBASE"; + plan: ExternalHarnessTaskPlan; + logger: EvalLogger; +} + +export interface PreparedGrokBuildToolAdapter { + toolSurface: ToolSurface; + startupProfile: StartupProfile; + cwd: string; + home: string; + grokHome: string; + env: Record; + mcpConfigPath: string; + mcpServerNames: string[]; + promptInstructions: string; + captureEvidence?: () => Promise; + drainStepObservations?: () => Promise; + onToolResult?: (toolName: string) => void; + observedToolMatcher?: (name: string) => boolean; + cleanup: () => Promise; +} + +type GrokBuildMcpServerSpec = { + command: string; + args?: string[]; + env?: Record; +}; + +export const GROK_BUILD_TOOL_SURFACES: ToolSurface[] = [ + "stagehand_facade", + "playwright_mcp", + "chrome_devtools_mcp", +]; + +export function buildGrokBuildMcpConfig( + mcpServers: Record, +): Record { + const normalized: Record = {}; + for (const [serverName, rawSpec] of Object.entries(mcpServers)) { + if (!isRecord(rawSpec) || typeof rawSpec.command !== "string") { + throw new EvalsError(`Invalid Grok Build MCP launch spec for server "${serverName}".`); + } + const spec = rawSpec as GrokBuildMcpServerSpec; + normalized[serverName] = { + command: spec.command, + args: stringArray(spec.args), + ...(isStringRecord(spec.env) && { env: spec.env }), + startup_timeout_sec: 60, + tool_timeout_sec: 300, + }; + } + return { mcp_servers: normalized }; +} + +export async function writeGrokBuildWorkspace( + cwd: string, + grokHome: string, + mcpServers: Record, +): Promise<{ mcpConfigPath: string }> { + const projectConfigDir = path.join(cwd, ".grok"); + const mcpConfigPath = path.join(projectConfigDir, "config.toml"); + await Promise.all([ + fsp.mkdir(projectConfigDir, { recursive: true }), + fsp.mkdir(grokHome, { recursive: true }), + ]); + await Promise.all([ + fsp.writeFile( + path.join(grokHome, "config.toml"), + stringify({ + cli: { auto_update: false, use_leader: false }, + compat: { claude: { mcps: false }, cursor: { mcps: false } }, + subagents: { enabled: false }, + memory: { enabled: false }, + }), + { mode: 0o600 }, + ), + fsp.writeFile(mcpConfigPath, stringify(buildGrokBuildMcpConfig(mcpServers)), { + mode: 0o600, + }), + ]); + return { mcpConfigPath }; +} + +export function resolveGrokBuildAuthHome( + env: NodeJS.ProcessEnv, + platform: NodeJS.Platform = process.platform, +): string | undefined { + const configured = env.GROK_HOME?.trim(); + if (configured) return configured; + const userHome = + platform === "win32" + ? env.USERPROFILE?.trim() || env.HOME?.trim() + : env.HOME?.trim() || env.USERPROFILE?.trim(); + return userHome ? path.join(userHome, ".grok") : undefined; +} + +export async function copyGrokBuildAuth( + env: NodeJS.ProcessEnv, + targetGrokHome: string, +): Promise { + if (env.XAI_API_KEY?.trim()) return false; + const sourceHome = resolveGrokBuildAuthHome(env); + if (!sourceHome) return false; + try { + await fsp.copyFile(path.join(sourceHome, "auth.json"), path.join(targetGrokHome, "auth.json")); + return true; + } catch { + return false; + } +} + +export function isGrokBuildMountToolName(serverNames: string[], toolName: string): boolean { + return serverNames.some( + (server) => + toolName === server || + toolName.startsWith(`${server}.`) || + toolName.startsWith(`${server}__`) || + toolName === `mcp__${server}` || + toolName.startsWith(`mcp__${server}__`), + ); +} + +export async function prepareGrokBuildToolAdapter( + input: GrokBuildToolAdapterInput, +): Promise { + const toolSurface = resolveToolSurface( + { harness: "grok_build", supportedToolSurfaces: GROK_BUILD_TOOL_SURFACES }, + input.toolSurface, + ); + if (toolSurface === undefined) { + throw new EvalsError("grok_build harness requires a tool surface."); + } + const startupProfile = resolveStartupProfile( + toolSurface, + input.environment, + input.startupProfile, + ); + const runtime = await startAgentToolRuntime({ + toolSurface, + startupProfile, + environment: input.environment, + logger: input.logger, + }); + + let root: string | undefined; + try { + const mount = runtime.running.agentMount; + if (!mount) { + throw new EvalsError(`Tool surface "${toolSurface}" does not provide an agent mount.`); + } + if (mount.via !== "mcp") { + throw new EvalsError( + `Grok Build does not support agent mounts delivered via "${mount.via}" yet.`, + ); + } + + root = await fsp.mkdtemp( + path.join(os.tmpdir(), `stagehand-evals-grok-build-${toolSurface.replace(/_/gu, "-")}-`), + ); + const capturedRoot = root; + const home = path.join(root, "home"); + const cwd = path.join(root, "workspace"); + const grokHome = path.join(home, ".grok"); + await Promise.all([ + fsp.mkdir(cwd, { recursive: true }), + fsp.mkdir(grokHome, { recursive: true }), + ]); + await copyGrokBuildAuth(process.env, grokHome); + const { mcpConfigPath } = await writeGrokBuildWorkspace(cwd, grokHome, mount.mcpServers); + const mcpServerNames = Object.keys(mount.mcpServers); + const recorder = runtime.running.captureEvidence + ? new ObservationRecorder(runtime.running.captureEvidence) + : undefined; + const observedToolMatcher = (name: string): boolean => + isGrokBuildMountToolName(mcpServerNames, name); + let cleanupPromise: Promise | undefined; + + input.logger.log({ + category: "grok_build", + message: `Initialized ${toolSurface} MCP mount for Grok Build (servers: ${mcpServerNames.join(", ")}).`, + level: 1, + auxiliary: { + startupProfile: { value: startupProfile, type: "string" }, + environment: { value: input.environment, type: "string" }, + }, + }); + + return { + toolSurface, + startupProfile, + cwd, + home, + grokHome, + env: stringEnv({ + ...process.env, + HOME: home, + USERPROFILE: home, + GROK_HOME: grokHome, + }), + mcpConfigPath, + mcpServerNames, + promptInstructions: mount.promptInstructions, + ...(runtime.running.captureEvidence && { + captureEvidence: boundedCaptureEvidence(runtime.running.captureEvidence), + }), + ...(recorder && { + drainStepObservations: async () => { + await recorder.settle(); + return recorder.drain(); + }, + onToolResult: (toolName: string) => { + if (observedToolMatcher(toolName)) void recorder.record(); + }, + }), + observedToolMatcher, + cleanup: async () => { + cleanupPromise ??= (async () => { + try { + await withTimeout( + runtime.cleanup(), + readPositiveIntEnv("EVAL_AGENT_MOUNT_CLEANUP_TIMEOUT_MS", 30_000), + ); + } catch { + // Best effort only. + } finally { + await fsp.rm(capturedRoot, { recursive: true, force: true }); + } + })(); + await cleanupPromise; + }, + }; + } catch (error) { + await withTimeout( + runtime.cleanup(), + readPositiveIntEnv("EVAL_AGENT_MOUNT_CLEANUP_TIMEOUT_MS", 30_000), + ).catch((): undefined => undefined); + if (root) await fsp.rm(root, { recursive: true, force: true }); + throw error; + } +} + +function boundedCaptureEvidence( + capture: () => Promise, +): () => Promise { + return async () => { + try { + return await withTimeout( + capture(), + readPositiveIntEnv("EVAL_CAPTURE_EVIDENCE_TIMEOUT_MS", 15_000), + ); + } catch { + return {}; + } + }; +} + +function readPositiveIntEnv(key: string, fallback: number): number { + const parsed = Number.parseInt(process.env[key] ?? "", 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +function withTimeout(promise: Promise, timeoutMs: number): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error(`grok_build adapter operation timed out after ${timeoutMs}ms`)), + timeoutMs, + ); + promise.then( + (value) => { + clearTimeout(timer); + resolve(value); + }, + (error) => { + clearTimeout(timer); + reject(error); + }, + ); + }); +} + +function stringArray(value: unknown): string[] { + return Array.isArray(value) + ? value.filter((item): item is string => typeof item === "string") + : []; +} + +function stringEnv(env: NodeJS.ProcessEnv): Record { + return Object.fromEntries( + Object.entries(env).filter((entry): entry is [string, string] => typeof entry[1] === "string"), + ); +} + +function isStringRecord(value: unknown): value is Record { + return isRecord(value) && Object.values(value).every((item) => typeof item === "string"); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/packages/evals/framework/harnesses/grokBuildAdapter.ts b/packages/evals/framework/harnesses/grokBuildAdapter.ts new file mode 100644 index 0000000000..0148f12b2c --- /dev/null +++ b/packages/evals/framework/harnesses/grokBuildAdapter.ts @@ -0,0 +1,119 @@ +import { extractGrokBuildToolCall } from "@browserbasehq/stagehand-integrations-grok-build-sdk"; +import type { ProbeEvidence, TaskSpec, Trajectory } from "stagehand-v3"; +import type { StepObservation } from "../observationRecorder.js"; +import { + buildTrajectory, + type NormalizedToolCall, + type TrajectoryAdapter, +} from "./trajectoryAdapter.js"; + +export interface GrokBuildRunResult { + events: Array>; + finalAnswer?: string; + status?: Trajectory["status"]; + usage?: Partial; + finalObservation?: ProbeEvidence; + stepObservations?: StepObservation[]; + observedToolName?: (name: string) => boolean; +} + +export class GrokBuildTrajectoryAdapter implements TrajectoryAdapter { + fromHarnessResult(result: GrokBuildRunResult, taskSpec: TaskSpec): Trajectory { + const toolCalls: NormalizedToolCall[] = []; + const openCalls = new Map(); + let pendingReasoning = ""; + + for (const event of result.events) { + if ((event.type === "thought" || event.type === "text") && typeof event.data === "string") { + pendingReasoning = appendText(pendingReasoning, event.data); + continue; + } + const view = extractGrokBuildToolCall(event); + if (!view) continue; + if (view.subtype === "started") { + const call: NormalizedToolCall = { + name: view.name ?? "tool", + args: view.args, + result: undefined, + ok: true, + reasoning: pendingReasoning.trim() || undefined, + }; + toolCalls.push(call); + if (view.callId) openCalls.set(view.callId, call); + pendingReasoning = ""; + continue; + } + + const call = view.callId ? openCalls.get(view.callId) : undefined; + if (call) { + call.result = normalizeToolResult(view.result); + call.ok = view.ok; + if (view.error) call.error = view.error; + openCalls.delete(view.callId); + } else { + toolCalls.push({ + name: view.name ?? "tool", + args: view.args, + result: normalizeToolResult(view.result), + ok: view.ok, + ...(view.error && { error: view.error }), + reasoning: pendingReasoning.trim() || undefined, + }); + pendingReasoning = ""; + } + } + + for (const open of openCalls.values()) { + open.ok = false; + open.result = "no tool result"; + open.error = "no tool result"; + } + + attachStepObservations(toolCalls, result); + return buildTrajectory({ + taskSpec, + toolCalls, + finalAnswer: result.finalAnswer, + status: result.status ?? "complete", + usage: result.usage, + ...(result.finalObservation && { finalObservation: result.finalObservation }), + }); + } +} + +export const grokBuildAdapter = new GrokBuildTrajectoryAdapter(); + +function normalizeToolResult(result: unknown): unknown { + if (!isRecord(result) || !Array.isArray(result.content)) return result; + const text = result.content + .filter(isRecord) + .map((block) => (typeof block.text === "string" ? block.text : undefined)) + .filter((part): part is string => part !== undefined); + return text.length > 0 ? text.join("\n") : result; +} + +function attachStepObservations(toolCalls: NormalizedToolCall[], result: GrokBuildRunResult): void { + const observations = result.stepObservations ?? []; + if (observations.length === 0) return; + const isObservedTool = + result.observedToolName ?? ((name: string) => name.startsWith("mcp") || name.includes(".")); + const observedCalls = toolCalls.filter((call) => isObservedTool(call.name)); + const totalObservedRuns = + Math.max(...observations.map((observation) => observation.runIndex)) + 1; + if (observedCalls.length !== totalObservedRuns) return; + const byRunIndex = new Map( + observations.map((observation) => [observation.runIndex, observation.evidence]), + ); + observedCalls.forEach((call, ordinal) => { + const observation = byRunIndex.get(ordinal); + if (observation) call.probeEvidence = observation; + }); +} + +function appendText(current: string, next: string): string { + return current ? `${current}\n${next}` : next; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/packages/evals/package.json b/packages/evals/package.json index 3b6c143ead..a897c8efe9 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -31,6 +31,7 @@ "@browserbasehq/stagehand-integrations-deepagents-sdk": "workspace:*", "@browserbasehq/stagehand-integrations-eve-sdk": "workspace:*", "@browserbasehq/stagehand-integrations-fx-sdk": "workspace:*", + "@browserbasehq/stagehand-integrations-grok-build-sdk": "workspace:*", "@browserbasehq/stagehand-integrations-mastra-sdk": "workspace:*", "@browserbasehq/stagehand-integrations-pi-sdk": "workspace:*", "@modelcontextprotocol/sdk": "catalog:", @@ -46,6 +47,7 @@ "openai": "^4.104.0", "playwright": ">=1.55.1 <1.57.0", "sharp": "^0.34.5", + "smol-toml": "catalog:", "stagehand-v3": "npm:@browserbasehq/stagehand@3.7.1", "tsx": "catalog:", "ws": "^8.21.0", diff --git a/packages/evals/tests/framework/benchHarness.test.ts b/packages/evals/tests/framework/benchHarness.test.ts index 7c0b2e968c..4a4de8c251 100644 --- a/packages/evals/tests/framework/benchHarness.test.ts +++ b/packages/evals/tests/framework/benchHarness.test.ts @@ -13,6 +13,7 @@ import { mastraHarness, piHarness, cursorHarness, + grokBuildHarness, fxHarness, deepagentsHarness, eveHarness, @@ -21,6 +22,7 @@ import { import { MASTRA_TOOL_SURFACES } from "../../framework/mastraToolAdapter.js"; import { PI_TOOL_SURFACES } from "../../framework/piToolAdapter.js"; import { CURSOR_TOOL_SURFACES } from "../../framework/cursorToolAdapter.js"; +import { GROK_BUILD_TOOL_SURFACES } from "../../framework/grokBuildToolAdapter.js"; import { defaultModelsEnvKey } from "../../framework/benchPlanner.js"; import type { BenchMatrixRow } from "../../framework/benchTypes.js"; import type { DiscoveredTask } from "../../framework/types.js"; @@ -39,6 +41,7 @@ describe("bench harness registry", () => { "deepagents", "fx", "cursor", + "grok_build", ]); }); @@ -46,7 +49,7 @@ describe("bench harness registry", () => { expect(parseBenchHarness(undefined)).toBe("stagehand"); expect(parseBenchHarness("codex")).toBe("codex"); expect(() => parseBenchHarness("nope")).toThrow( - /Unknown harness "nope"\. Supported: stagehand, claude_code, codex, mastra, pi, eve, deepagents, fx, cursor\./, + /Unknown harness "nope"\. Supported: stagehand, claude_code, codex, mastra, pi, eve, deepagents, fx, cursor, grok_build\./, ); }); @@ -185,6 +188,20 @@ describe("bench harness registry", () => { expect(harness.defaultModels).toEqual(["cursor/auto"]); }); + it("registers grok_build as a concrete executable harness", () => { + const harness = getBenchHarness("grok_build"); + + expect(harness).toBe(grokBuildHarness); + expect(parseBenchHarness("grok_build")).toBe("grok_build"); + expect(isExecutableBenchHarness("grok_build")).toBe(true); + expect(harness.supportedTaskKinds).toEqual(["agent", "suite"]); + expect(harness.supportsApi).toBe(false); + expect(harness.execute).toBeDefined(); + expect(harness.supportedToolSurfaces).toEqual(GROK_BUILD_TOOL_SURFACES); + expect(harness.supportedToolSurfaces[0]).toBe("stagehand_facade"); + expect(harness.defaultModels).toEqual(["grok-build/auto"]); + }); + it("registers a new harness and rejects duplicate ids", () => { const fakeHarness = { harness: "fake_harness", diff --git a/packages/evals/tests/framework/grokBuildAdapter.test.ts b/packages/evals/tests/framework/grokBuildAdapter.test.ts new file mode 100644 index 0000000000..036df5ee1d --- /dev/null +++ b/packages/evals/tests/framework/grokBuildAdapter.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest"; +import type { TaskSpec } from "stagehand-v3"; +import { grokBuildAdapter } from "../../framework/harnesses/grokBuildAdapter.js"; + +const taskSpec: TaskSpec = { id: "grok-build-test", instruction: "do the task" }; + +describe("Grok Build trajectory adapter", () => { + it("pairs native tool calls and carries thought text into reasoning", () => { + const trajectory = grokBuildAdapter.fromHarnessResult( + { + events: [ + { type: "thought", data: "I will inspect the page." }, + { + type: "tool_call", + toolCallId: "call-1", + toolName: "stagehand__snapshot", + rawInput: {}, + }, + { + type: "tool_call_update", + toolCallId: "call-1", + status: "completed", + rawOutput: { title: "Example" }, + }, + ], + finalAnswer: "Example", + }, + taskSpec, + ); + expect(trajectory.steps).toHaveLength(1); + expect(trajectory.steps[0]).toMatchObject({ + actionName: "stagehand__snapshot", + reasoning: "I will inspect the page.", + toolOutput: { ok: true, result: { title: "Example" } }, + }); + expect(trajectory.finalAnswer).toBe("Example"); + }); + + it("fails open tool calls closed and attaches matching observations", () => { + const trajectory = grokBuildAdapter.fromHarnessResult( + { + events: [ + { + type: "tool_call", + toolCallId: "call-1", + toolName: "stagehand__run", + rawInput: {}, + }, + ], + observedToolName: (name) => name.startsWith("stagehand__"), + stepObservations: [{ runIndex: 0, evidence: { url: "https://example.com" } }], + }, + taskSpec, + ); + expect(trajectory.steps[0].toolOutput).toMatchObject({ + ok: false, + error: "no tool result", + }); + expect(trajectory.steps[0].probeEvidence.url).toBe("https://example.com"); + }); +}); diff --git a/packages/evals/tests/framework/grokBuildRunner.test.ts b/packages/evals/tests/framework/grokBuildRunner.test.ts new file mode 100644 index 0000000000..83ab8216a0 --- /dev/null +++ b/packages/evals/tests/framework/grokBuildRunner.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from "vitest"; +import type { AvailableModel } from "stagehand-v3"; +import type { GrokBuildProcessRunner } from "@browserbasehq/stagehand-integrations-grok-build-sdk"; +import { buildGrokBuildPrompt, runGrokBuildAgent } from "../../framework/grokBuildRunner.js"; +import type { ExternalHarnessTaskPlan } from "../../framework/externalHarnessPlan.js"; +import { EvalLogger } from "../../logger.js"; + +const plan: ExternalHarnessTaskPlan = { + dataset: "webvoyager", + taskId: "wv-1", + startUrl: "https://example.com", + instruction: "Report the heading", +}; + +describe("Grok Build runner", () => { + it("builds an MCP-only browser prompt", () => { + const prompt = buildGrokBuildPrompt(plan, "Use stagehand__run."); + expect(prompt).toContain("Dataset: webvoyager"); + expect(prompt).toContain("Start URL: https://example.com"); + expect(prompt).toContain("Use stagehand__run."); + expect(prompt).toContain("Your only browser access is the MCP server"); + expect(prompt).toContain("EVAL_RESULT:"); + }); + + it("runs the native stream and reports Grok Build metrics", async () => { + const result = await runGrokBuildAgent({ + plan, + model: "grok-build/auto" as AvailableModel, + logger: new EvalLogger(false), + runProcess: scriptedRunner([ + { + type: "tool_call", + toolCallId: "1", + toolName: "stagehand__run", + rawInput: { code: "return 1" }, + }, + { + type: "tool_call_update", + toolCallId: "1", + status: "completed", + rawOutput: "done", + }, + { + type: "text", + data: 'EVAL_RESULT: {"success":true,"summary":"done","finalAnswer":"ok"}', + }, + { + type: "end", + stopReason: "end_turn", + num_turns: 2, + usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15 }, + total_cost_usd: 0.02, + }, + ]), + }); + const metrics = result.metrics as Record; + expect(result._success).toBe(true); + expect(result.harnessStatus).toBe("completed"); + expect(result.grokBuildStatus).toBe("completed"); + expect(result.finalAnswer).toBe("ok"); + expect(metrics.grok_build_tool_steps.value).toBe(1); + expect(metrics.grok_build_num_turns.value).toBe(2); + expect(metrics.harness_total_tokens.value).toBe(15); + expect(metrics.harness_cost_usd.value).toBe(0.02); + }); + + it("returns a failed result for a non-zero exit without an end event", async () => { + const result = await runGrokBuildAgent({ + plan, + model: "grok-build/auto" as AvailableModel, + logger: new EvalLogger(false), + runProcess: scriptedRunner([], 1), + }); + expect(result._success).toBe(false); + expect(result.harnessStatus).toBe("sdk_error"); + expect(result.error).toContain("exited with code 1"); + }); +}); + +function scriptedRunner( + events: Array>, + exitCode = 0, +): GrokBuildProcessRunner { + return async (input) => { + for (const event of events) await input.onStdoutLine(JSON.stringify(event)); + return { exitCode, signal: null }; + }; +} diff --git a/packages/evals/tests/framework/grokBuildToolAdapter.test.ts b/packages/evals/tests/framework/grokBuildToolAdapter.test.ts new file mode 100644 index 0000000000..ad09f3806b --- /dev/null +++ b/packages/evals/tests/framework/grokBuildToolAdapter.test.ts @@ -0,0 +1,88 @@ +import fsp from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { parse } from "smol-toml"; +import { + buildGrokBuildMcpConfig, + copyGrokBuildAuth, + GROK_BUILD_TOOL_SURFACES, + isGrokBuildMountToolName, + resolveGrokBuildAuthHome, + writeGrokBuildWorkspace, +} from "../../framework/grokBuildToolAdapter.js"; + +const tempDirs: string[] = []; + +afterEach(async () => { + await Promise.all(tempDirs.splice(0).map((dir) => fsp.rm(dir, { recursive: true, force: true }))); +}); + +describe("Grok Build tool adapter helpers", () => { + it("supports MCP mounts and converts their config to Grok TOML shape", () => { + expect(GROK_BUILD_TOOL_SURFACES).toEqual([ + "stagehand_facade", + "playwright_mcp", + "chrome_devtools_mcp", + ]); + expect( + buildGrokBuildMcpConfig({ + stagehand: { command: "node", args: ["server.mjs"], env: { TOKEN: "value" } }, + }), + ).toEqual({ + mcp_servers: { + stagehand: { + command: "node", + args: ["server.mjs"], + env: { TOKEN: "value" }, + startup_timeout_sec: 60, + tool_timeout_sec: 300, + }, + }, + }); + }); + + it("writes isolated user and project config", async () => { + const root = await fsp.mkdtemp(path.join(os.tmpdir(), "grok-build-workspace-test-")); + tempDirs.push(root); + const cwd = path.join(root, "workspace"); + const grokHome = path.join(root, "home", ".grok"); + const result = await writeGrokBuildWorkspace(cwd, grokHome, { + stagehand: { command: "node", args: ["server.mjs"] }, + }); + const projectConfig = parse(await fsp.readFile(result.mcpConfigPath, "utf8")); + const userConfig = parse(await fsp.readFile(path.join(grokHome, "config.toml"), "utf8")); + expect(projectConfig).toMatchObject({ + mcp_servers: { stagehand: { command: "node", args: ["server.mjs"] } }, + }); + expect(userConfig).toMatchObject({ + cli: { auto_update: false, use_leader: false }, + subagents: { enabled: false }, + memory: { enabled: false }, + }); + }); + + it("copies cached auth only when no API key is supplied", async () => { + const root = await fsp.mkdtemp(path.join(os.tmpdir(), "grok-build-auth-test-")); + tempDirs.push(root); + const source = path.join(root, "source"); + const target = path.join(root, "target"); + await Promise.all([fsp.mkdir(source), fsp.mkdir(target)]); + await fsp.writeFile(path.join(source, "auth.json"), '{"token":"cached"}\n'); + expect(resolveGrokBuildAuthHome({ GROK_HOME: source })).toBe(source); + await expect(copyGrokBuildAuth({ GROK_HOME: source }, target)).resolves.toBe(true); + await expect(fsp.readFile(path.join(target, "auth.json"), "utf8")).resolves.toContain("cached"); + await fsp.rm(path.join(target, "auth.json")); + await expect( + copyGrokBuildAuth({ GROK_HOME: source, XAI_API_KEY: "secret" }, target), + ).resolves.toBe(false); + }); + + it("matches Grok MCP tool identities", () => { + const matches = (name: string) => isGrokBuildMountToolName(["stagehand"], name); + for (const name of ["stagehand.run", "stagehand__run", "mcp__stagehand__run"]) { + expect(matches(name)).toBe(true); + } + expect(matches("shell")).toBe(false); + }); +}); diff --git a/packages/integrations/README.md b/packages/integrations/README.md index 0b88bdf580..f70c4a1efd 100644 --- a/packages/integrations/README.md +++ b/packages/integrations/README.md @@ -8,19 +8,21 @@ else, never restated. ## Structure -| Directory | What it is | -| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `core/` | The `@browserbasehq/stagehand-integrations` package: the facade contract, `StagehandFacadeTools`, the `stagehand-facade` stdio MCP bin, and the code-mode MCP host scaffold. | -| `claude-code/` | Claude Agent SDK example (programmatic MCP mount) plus a `.mcp.json` for connecting a running Claude Code CLI. | -| `codex/` | Codex SDK example (config-override MCP mount) plus a `config.toml` template for the codex CLI. | -| `cursor/` | Cursor agent CLI configuration template (`.cursor/mcp.json`) and AGENTS.md guidance — the CLI consumes the facade as a project MCP server. | -| `crewai/` | Python CrewAI example over MCP/stdio (uv project). | -| `deepagents/` | Python LangChain Deep Agents integrations: a local stdio MCP server and a Managed Deep Agents project with native tools. | -| `eve/` | Eve example with the tools bound natively via `defineTool` (Eve has no external-process tool mounting). | -| `fx/` | fx configuration templates and skill — fx consumes the facade via its user-global MCP config. | -| `mastra/` | Mastra example over MCP/stdio via Mastra's `MCPClient`. | -| `pi/` | Pi extension registering the tools natively (Pi ships without built-in MCP). | -| `vercel-ai/` | Vercel AI SDK example over MCP/stdio via `createMCPClient`. | +| Directory | What it is | +| ----------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `core/` | The `@browserbasehq/stagehand-integrations` package: the facade contract, `StagehandFacadeTools`, the `stagehand-facade` stdio MCP bin, and the code-mode MCP host scaffold. | +| `claude-code/` | Claude Agent SDK example (programmatic MCP mount) plus a `.mcp.json` for connecting a running Claude Code CLI. | +| `codex/` | Codex SDK example (config-override MCP mount) plus a `config.toml` template for the codex CLI. | +| `cursor/` | Cursor agent CLI configuration template (`.cursor/mcp.json`) and AGENTS.md guidance — the CLI consumes the facade as a project MCP server. | +| `crewai/` | Python CrewAI example over MCP/stdio (uv project). | +| `deepagents/` | Python LangChain Deep Agents integrations: a local stdio MCP server and a Managed Deep Agents project with native tools. | +| `eve/` | Eve example with the tools bound natively via `defineTool` (Eve has no external-process tool mounting). | +| `fx/` | fx configuration templates and skill — fx consumes the facade via its user-global MCP config. | +| `grok-build/` | Grok Build CLI configuration template and AGENTS.md guidance. The CLI consumes the facade as a project MCP server. | +| `grok-build-sdk/` | Grok Build CLI session adapter used by evals. | +| `mastra/` | Mastra example over MCP/stdio via Mastra's `MCPClient`. | +| `pi/` | Pi extension registering the tools natively (Pi ships without built-in MCP). | +| `vercel-ai/` | Vercel AI SDK example over MCP/stdio via `createMCPClient`. | Each example is a self-contained project: install, export `BROWSERBASE_API_KEY`, and run — see the directory's README. TypeScript examples consume `core/` as a workspace dependency; the Python projects resolve the published diff --git a/packages/integrations/grok-build-sdk/package.json b/packages/integrations/grok-build-sdk/package.json new file mode 100644 index 0000000000..b618cf48f0 --- /dev/null +++ b/packages/integrations/grok-build-sdk/package.json @@ -0,0 +1,34 @@ +{ + "name": "@browserbasehq/stagehand-integrations-grok-build-sdk", + "version": "4.0.1", + "private": true, + "description": "Grok Build CLI harness adapter for Stagehand integrations", + "files": [ + "dist" + ], + "type": "module", + "exports": { + ".": { + "types": "./dist/index.d.mts", + "import": "./dist/index.mjs" + } + }, + "scripts": { + "build": "tsdown", + "test": "pnpm run build && vitest run --root ../../.. packages/integrations/grok-build-sdk/tests", + "test:unit": "vitest run --root ../../.. packages/integrations/grok-build-sdk/tests", + "typecheck": "tsc --noEmit -p tsconfig.json" + }, + "dependencies": { + "@browserbasehq/stagehand-integrations": "workspace:*" + }, + "devDependencies": { + "@types/node": "catalog:", + "tsdown": "catalog:", + "typescript": "catalog:", + "vitest": "catalog:" + }, + "engines": { + "node": ">=24.0.0" + } +} diff --git a/packages/integrations/grok-build-sdk/src/index.ts b/packages/integrations/grok-build-sdk/src/index.ts new file mode 100644 index 0000000000..9df91886ba --- /dev/null +++ b/packages/integrations/grok-build-sdk/src/index.ts @@ -0,0 +1 @@ +export * from "./session.js"; diff --git a/packages/integrations/grok-build-sdk/src/session.ts b/packages/integrations/grok-build-sdk/src/session.ts new file mode 100644 index 0000000000..c4c63367df --- /dev/null +++ b/packages/integrations/grok-build-sdk/src/session.ts @@ -0,0 +1,460 @@ +import { spawn } from "node:child_process"; +import { + HarnessAdapterError, + sanitizeErrorMessage, + type HarnessLogger, +} from "@browserbasehq/stagehand-integrations/harness"; + +export type GrokBuildEvent = Record; + +export type GrokBuildProcessExit = { + exitCode: number | null; + signal: NodeJS.Signals | null; +}; + +export type GrokBuildProcessRunner = (input: { + command: string; + args: string[]; + cwd?: string; + env?: Record; + signal: AbortSignal; + onStdoutLine: (line: string) => void | Promise; + onStderr: (chunk: string) => void; +}) => Promise; + +export type GrokBuildSessionConfig = { + cwd?: string; + env?: Record; + binaryPath?: string; + maxTurns?: number; + sandbox?: string; + extraArgs?: string[]; +}; + +export type GrokBuildTokenUsage = { + inputTokens: number; + outputTokens: number; + cachedInputTokens: number; + cacheCreationInputTokens: number; + reasoningOutputTokens: number; + totalTokens: number; + reported: boolean; +}; + +export type GrokBuildSessionResult = { + events: GrokBuildEvent[]; + endEvent?: GrokBuildEvent; + resultText: string; + status: "completed" | "max_turns" | "sdk_error"; + stopReason?: string; + tokenUsage: GrokBuildTokenUsage; + costUsd?: number; + exit?: GrokBuildProcessExit; + stderr: string; + iterationError?: unknown; +}; + +export type GrokBuildToolCallView = { + callId: string; + subtype: "started" | "completed"; + name?: string; + args: Record; + result?: unknown; + ok: boolean; + error?: string; +}; + +export const GROK_BUILD_BINARY = "grok"; +const STDERR_LIMIT = 64 * 1024; + +export function resolveGrokBuildBinary(override?: string): string { + return override ?? process.env.GROK_BUILD_PATH ?? GROK_BUILD_BINARY; +} + +export function normalizeGrokBuildModel(model: string): string | undefined { + if (model === "grok-build/auto" || model === "auto") return undefined; + return model.includes("/") ? model.slice(model.indexOf("/") + 1) : model; +} + +export function buildGrokBuildArgs(input: { + prompt: string; + model?: string; + session: GrokBuildSessionConfig; +}): string[] { + const { session } = input; + return [ + "-p", + input.prompt, + "--output-format", + "streaming-json", + "--always-approve", + "--tools", + "search_tool,use_tool", + "--disallowed-tools", + "Agent", + "--no-plan", + "--no-subagents", + "--disable-web-search", + ...(session.cwd ? ["--cwd", session.cwd] : []), + ...(input.model ? ["--model", input.model] : []), + ...(session.maxTurns ? ["--max-turns", String(session.maxTurns)] : []), + ...(session.sandbox ? ["--sandbox", session.sandbox] : []), + ...(session.extraArgs ?? []), + ]; +} + +export function parseGrokBuildStreamLine(line: string): GrokBuildEvent | undefined { + const trimmed = line.trim(); + if (!trimmed) return undefined; + try { + const parsed: unknown = JSON.parse(trimmed); + return isRecord(parsed) ? parsed : undefined; + } catch { + return undefined; + } +} + +export function extractGrokBuildToolCall(event: GrokBuildEvent): GrokBuildToolCallView | undefined { + if (event.type === "tool_call") { + return { + callId: readString(event.toolCallId) ?? "", + subtype: "started", + name: readString(event.toolName), + args: isRecord(event.rawInput) ? event.rawInput : {}, + ok: true, + }; + } + if (event.type !== "tool_call_update") return undefined; + const status = readString(event.status) ?? "completed"; + if (!["completed", "failed", "cancelled", "rejected"].includes(status)) return undefined; + const ok = status === "completed"; + const result = event.rawOutput ?? event.content; + return { + callId: readString(event.toolCallId) ?? "", + subtype: "completed", + args: {}, + ...(result !== undefined && { result }), + ok, + ...(!ok && { error: stringifyError(result) || `tool call ${status}` }), + }; +} + +export const defaultGrokBuildProcessRunner: GrokBuildProcessRunner = async (input) => { + return new Promise((resolve, reject) => { + const child = spawn(input.command, input.args, { + ...(input.cwd && { cwd: input.cwd }), + ...(input.env && { env: input.env }), + stdio: ["ignore", "pipe", "pipe"], + }); + let stdoutBuffer = ""; + let lineQueue = Promise.resolve(); + let killTimer: NodeJS.Timeout | undefined; + let settled = false; + + const queueLine = (line: string): void => { + lineQueue = lineQueue.then(() => input.onStdoutLine(line)); + }; + const removeAbort = (): void => input.signal.removeEventListener("abort", abort); + const abort = (): void => { + if (child.exitCode !== null || child.signalCode !== null) return; + child.kill("SIGTERM"); + killTimer = setTimeout(() => { + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + }, 5_000); + killTimer.unref(); + }; + + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + stdoutBuffer += chunk; + const lines = stdoutBuffer.split(/\r?\n/u); + stdoutBuffer = lines.pop() ?? ""; + for (const line of lines) queueLine(line); + }); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => input.onStderr(chunk)); + child.on("error", (error: NodeJS.ErrnoException) => { + if (settled) return; + settled = true; + removeAbort(); + if (killTimer) clearTimeout(killTimer); + if (error.code === "ENOENT") { + reject( + new HarnessAdapterError( + "Grok Build harness requires the `grok` CLI (install `@xai-official/grok` globally, or set GROK_BUILD_PATH).", + { cause: error }, + ), + ); + return; + } + reject(error); + }); + child.on("close", (exitCode, signal) => { + if (settled) return; + settled = true; + removeAbort(); + if (killTimer) clearTimeout(killTimer); + if (stdoutBuffer) queueLine(stdoutBuffer); + lineQueue.then( + () => resolve({ exitCode, signal }), + (error) => reject(error), + ); + }); + + if (input.signal.aborted) abort(); + else input.signal.addEventListener("abort", abort, { once: true }); + }); +}; + +export async function runGrokBuildSession(input: { + prompt: string; + model: string; + signal?: AbortSignal; + logger: HarnessLogger; + session: GrokBuildSessionConfig; + runProcess?: GrokBuildProcessRunner; + onToolResult?: (toolName: string, view: GrokBuildToolCallView) => void | Promise; +}): Promise { + const events: GrokBuildEvent[] = []; + const controller = new AbortController(); + const forwardAbort = (): void => controller.abort(input.signal?.reason); + if (input.signal) { + if (input.signal.aborted) controller.abort(input.signal.reason); + else input.signal.addEventListener("abort", forwardAbort, { once: true }); + } + + const textParts: string[] = []; + const toolNames = new Map(); + let endEvent: GrokBuildEvent | undefined; + let errorEvent: GrokBuildEvent | undefined; + let stderr = ""; + let exit: GrokBuildProcessExit | undefined; + let iterationError: unknown; + + try { + const model = normalizeGrokBuildModel(input.model); + exit = await (input.runProcess ?? defaultGrokBuildProcessRunner)({ + command: resolveGrokBuildBinary(input.session.binaryPath), + args: buildGrokBuildArgs({ prompt: input.prompt, model, session: input.session }), + ...(input.session.cwd && { cwd: input.session.cwd }), + env: stringEnv({ ...process.env, ...input.session.env }), + signal: controller.signal, + onStdoutLine: async (line) => { + const parsed = parseGrokBuildStreamLine(line); + if (!parsed) return; + const event = deepSanitize(parsed) as GrokBuildEvent; + events.push(event); + logGrokBuildEvent(input.logger, event); + if (event.type === "text" && typeof event.data === "string") textParts.push(event.data); + if (event.type === "end") endEvent = event; + if (event.type === "error") errorEvent = event; + const view = extractGrokBuildToolCall(event); + if (view?.subtype === "started" && view.callId && view.name) { + toolNames.set(view.callId, view.name); + } + if (view?.subtype === "completed") { + const toolName = toolNames.get(view.callId) ?? view.name ?? "tool"; + await input.onToolResult?.(toolName, view); + } + }, + onStderr: (chunk) => { + stderr = `${stderr}${chunk}`.slice(-STDERR_LIMIT); + }, + }); + } catch (error) { + iterationError = new HarnessAdapterError( + sanitizeErrorMessage(stringifyError(error)) || "Grok Build session failed.", + ); + input.logger.warn({ + category: "grok_build", + message: `Grok Build stopped before a normal result: ${sanitizeErrorMessage(stringifyError(error))}`, + level: 0, + }); + } finally { + input.signal?.removeEventListener("abort", forwardAbort); + } + + stderr = sanitizeErrorMessage(stderr); + if (stderr) input.logger.log({ category: "grok_build", message: stderr, level: 1 }); + const externalAbortReason = input.signal?.aborted + ? stringifyError(input.signal.reason) || "Grok Build session aborted" + : undefined; + const stopReason = buildGrokBuildStopReason({ + endEvent, + errorEvent, + iterationError, + exit, + stderr, + externalAbortReason, + }); + const tokenUsage = readGrokBuildUsage(endEvent); + const costUsd = finiteNumber(endEvent?.total_cost_usd); + return { + events, + ...(endEvent && { endEvent }), + resultText: textParts.join(""), + status: resolveGrokBuildStatus(endEvent, errorEvent, iterationError, stopReason), + ...(stopReason && { stopReason: sanitizeErrorMessage(stopReason) }), + tokenUsage, + ...(costUsd !== undefined && { costUsd }), + ...(exit && { exit }), + stderr, + ...(iterationError !== undefined && { iterationError }), + }; +} + +export function readGrokBuildUsage(event: GrokBuildEvent | undefined): GrokBuildTokenUsage { + const usage = isRecord(event?.usage) ? event.usage : undefined; + const inputTokens = finiteNumber(usage?.input_tokens) ?? 0; + const cachedInputTokens = finiteNumber(usage?.cache_read_input_tokens) ?? 0; + const cacheCreationInputTokens = finiteNumber(usage?.cache_creation_input_tokens) ?? 0; + const outputTokens = finiteNumber(usage?.output_tokens) ?? 0; + const reasoningOutputTokens = finiteNumber(usage?.reasoning_tokens) ?? 0; + const totalTokens = + finiteNumber(usage?.total_tokens) ?? + inputTokens + cachedInputTokens + cacheCreationInputTokens + outputTokens; + return { + inputTokens, + outputTokens, + cachedInputTokens, + cacheCreationInputTokens, + reasoningOutputTokens, + totalTokens, + reported: usage !== undefined, + }; +} + +export function resolveGrokBuildStatus( + endEvent: GrokBuildEvent | undefined, + errorEvent: GrokBuildEvent | undefined, + iterationError: unknown, + stopReason?: string, +): "completed" | "max_turns" | "sdk_error" { + const reason = readString(endEvent?.stopReason) ?? ""; + if (/max_turn/iu.test(reason) || stopReason === "max turns reached") return "max_turns"; + if (iterationError || errorEvent || stopReason || !endEvent) return "sdk_error"; + return "completed"; +} + +export function buildGrokBuildStopReason(input: { + endEvent?: GrokBuildEvent; + errorEvent?: GrokBuildEvent; + iterationError?: unknown; + exit?: GrokBuildProcessExit; + stderr: string; + externalAbortReason?: string; +}): string | undefined { + if (input.externalAbortReason) return input.externalAbortReason; + if (input.iterationError) return stringifyError(input.iterationError); + if (input.errorEvent) { + return readString(input.errorEvent.message) ?? "Grok Build returned an error event"; + } + const reason = readString(input.endEvent?.stopReason); + if (reason && /max_turn/iu.test(reason)) return "max turns reached"; + if (!input.endEvent) { + const lastLine = input.stderr + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter(Boolean) + .at(-1); + return input.exit?.exitCode !== 0 + ? `grok exited with code ${String(input.exit?.exitCode ?? "unknown")}${lastLine ? `: ${lastLine}` : ""}` + : "Grok Build exited without a terminal end event"; + } + return undefined; +} + +export function buildGrokBuildTranscript(events: GrokBuildEvent[]): string { + return events + .map((event) => summarizeGrokBuildEvent(event).detail) + .filter((detail): detail is string => Boolean(detail)) + .join("\n"); +} + +export function logGrokBuildEvent(logger: HarnessLogger, event: GrokBuildEvent): void { + const summary = summarizeGrokBuildEvent(event); + logger.log({ + category: "grok_build", + message: summary.message, + level: 1, + auxiliary: { + type: { value: readString(event.type) ?? "unknown", type: "string" }, + ...(summary.detail && { detail: { value: summary.detail, type: "string" } }), + }, + }); +} + +export function summarizeGrokBuildEvent(event: GrokBuildEvent): { + message: string; + detail?: string; +} { + const type = readString(event.type) ?? "unknown"; + if ((type === "text" || type === "thought") && typeof event.data === "string") { + const detail = sanitizeErrorMessage(event.data); + return { message: `${type}: ${clip(detail, 500)}`, detail }; + } + const tool = extractGrokBuildToolCall(event); + if (tool) { + return { + message: `tool: ${tool.name ?? tool.callId ?? "unknown"} ${tool.subtype}${tool.ok ? "" : " failed"}`, + detail: sanitizeOptional(safeJson(event)), + }; + } + if (type === "end") { + return { message: `end: ${readString(event.stopReason) ?? "done"}`, detail: safeJson(event) }; + } + return { message: `${type} event`, detail: sanitizeOptional(safeJson(event)) }; +} + +function deepSanitize(value: unknown): unknown { + if (typeof value === "string") return sanitizeErrorMessage(value); + if (Array.isArray(value)) return value.map(deepSanitize); + if (!isRecord(value)) return value; + return Object.fromEntries( + Object.entries(value).map(([key, child]) => [key, deepSanitize(child)]), + ); +} + +function sanitizeOptional(value: string | undefined): string | undefined { + return value === undefined ? undefined : sanitizeErrorMessage(value); +} + +function stringEnv( + env: NodeJS.ProcessEnv | Record, +): Record { + return Object.fromEntries( + Object.entries(env).filter((entry): entry is [string, string] => typeof entry[1] === "string"), + ); +} + +function finiteNumber(value: unknown): number | undefined { + const number = typeof value === "number" ? value : Number.NaN; + return Number.isFinite(number) ? number : undefined; +} + +function readString(value: unknown): string | undefined { + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function safeJson(value: unknown): string | undefined { + try { + return JSON.stringify(value); + } catch { + return undefined; + } +} + +export function stringifyError(value: unknown): string { + if (!value) return ""; + if (value instanceof Error) return value.message; + if (typeof value === "string") return value; + return safeJson(value) ?? "Unknown error"; +} + +export function clip(value: string, maxLength: number): string { + return value.length <= maxLength ? value : `${value.slice(0, maxLength - 1)}…`; +} diff --git a/packages/integrations/grok-build-sdk/tests/session.test.ts b/packages/integrations/grok-build-sdk/tests/session.test.ts new file mode 100644 index 0000000000..b8cea9b569 --- /dev/null +++ b/packages/integrations/grok-build-sdk/tests/session.test.ts @@ -0,0 +1,190 @@ +import { describe, expect, it, vi } from "vitest"; +import { + buildGrokBuildArgs, + buildGrokBuildStopReason, + extractGrokBuildToolCall, + normalizeGrokBuildModel, + parseGrokBuildStreamLine, + readGrokBuildUsage, + resolveGrokBuildBinary, + runGrokBuildSession, + type GrokBuildProcessRunner, +} from "../src/session.ts"; + +const logger = { + log: vi.fn(), + warn: vi.fn(), + error: vi.fn(), +}; + +describe("Grok Build CLI session", () => { + it("builds a restricted one-shot CLI invocation", () => { + expect( + buildGrokBuildArgs({ + prompt: "do it", + model: "grok-build", + session: { cwd: "/workspace", maxTurns: 12, sandbox: "off" }, + }), + ).toEqual([ + "-p", + "do it", + "--output-format", + "streaming-json", + "--always-approve", + "--tools", + "search_tool,use_tool", + "--disallowed-tools", + "Agent", + "--no-plan", + "--no-subagents", + "--disable-web-search", + "--cwd", + "/workspace", + "--model", + "grok-build", + "--max-turns", + "12", + "--sandbox", + "off", + ]); + }); + + it("resolves the binary and normalizes harness model ids", () => { + expect(resolveGrokBuildBinary("/custom/grok")).toBe("/custom/grok"); + expect(normalizeGrokBuildModel("grok-build/auto")).toBeUndefined(); + expect(normalizeGrokBuildModel("xai/grok-build")).toBe("grok-build"); + expect(normalizeGrokBuildModel("grok-build")).toBe("grok-build"); + }); + + it("parses native streaming events and tool updates", () => { + expect(parseGrokBuildStreamLine("not-json")).toBeUndefined(); + expect(parseGrokBuildStreamLine('{"type":"text","data":"hi"}')).toEqual({ + type: "text", + data: "hi", + }); + expect( + extractGrokBuildToolCall({ + type: "tool_call", + toolCallId: "call-1", + toolName: "stagehand__run", + rawInput: { code: "return 1" }, + }), + ).toEqual({ + callId: "call-1", + subtype: "started", + name: "stagehand__run", + args: { code: "return 1" }, + ok: true, + }); + expect( + extractGrokBuildToolCall({ + type: "tool_call_update", + toolCallId: "call-1", + status: "completed", + rawOutput: { value: 1 }, + }), + ).toMatchObject({ + callId: "call-1", + subtype: "completed", + result: { value: 1 }, + ok: true, + }); + }); + + it("normalizes usage from the terminal end event", () => { + expect( + readGrokBuildUsage({ + type: "end", + usage: { + input_tokens: 10, + cache_read_input_tokens: 20, + cache_creation_input_tokens: 2, + output_tokens: 5, + reasoning_tokens: 3, + }, + }), + ).toEqual({ + inputTokens: 10, + outputTokens: 5, + cachedInputTokens: 20, + cacheCreationInputTokens: 2, + reasoningOutputTokens: 3, + totalTokens: 37, + reported: true, + }); + }); + + it("runs the stream, joins text, and reports usage and cost", async () => { + const onToolResult = vi.fn(); + const result = await runGrokBuildSession({ + prompt: "do it", + model: "grok-build/auto", + logger, + session: { cwd: "/workspace" }, + runProcess: scriptedRunner([ + { type: "text", data: "EVAL_RESULT: " }, + { + type: "tool_call", + toolCallId: "call-1", + toolName: "stagehand__snapshot", + rawInput: {}, + }, + { + type: "tool_call_update", + toolCallId: "call-1", + status: "completed", + rawOutput: "snapshot", + }, + { type: "text", data: '{"success":true,"summary":"done","finalAnswer":"ok"}' }, + { + type: "end", + stopReason: "end_turn", + usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15 }, + total_cost_usd: 0.02, + num_turns: 2, + }, + ]), + onToolResult, + }); + + expect(result.status).toBe("completed"); + expect(result.resultText).toContain('EVAL_RESULT: {"success":true'); + expect(result.tokenUsage.totalTokens).toBe(15); + expect(result.costUsd).toBe(0.02); + expect(onToolResult).toHaveBeenCalledWith( + "stagehand__snapshot", + expect.objectContaining({ subtype: "completed" }), + ); + }); + + it("fails a non-zero exit without an end event", async () => { + const result = await runGrokBuildSession({ + prompt: "do it", + model: "grok-build/auto", + logger, + session: {}, + runProcess: scriptedRunner([], 1), + }); + expect(result.status).toBe("sdk_error"); + expect(result.stopReason).toContain("exited with code 1"); + }); + + it("reports max-turn stops without treating them as SDK errors", () => { + expect( + buildGrokBuildStopReason({ + endEvent: { type: "end", stopReason: "max_turn_requests" }, + stderr: "", + }), + ).toBe("max turns reached"); + }); +}); + +function scriptedRunner( + events: Array>, + exitCode = 0, +): GrokBuildProcessRunner { + return async (input) => { + for (const event of events) await input.onStdoutLine(JSON.stringify(event)); + return { exitCode, signal: null }; + }; +} diff --git a/packages/integrations/grok-build-sdk/tsconfig.json b/packages/integrations/grok-build-sdk/tsconfig.json new file mode 100644 index 0000000000..3571933603 --- /dev/null +++ b/packages/integrations/grok-build-sdk/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "module": "NodeNext", + "moduleResolution": "NodeNext", + "target": "ES2022", + "types": ["node"], + "rootDir": ".", + "noEmit": true, + "skipLibCheck": true + }, + "include": ["src/**/*.ts", "tests/**/*.ts"], + "exclude": ["dist", "node_modules"] +} diff --git a/packages/integrations/grok-build-sdk/tsdown.config.ts b/packages/integrations/grok-build-sdk/tsdown.config.ts new file mode 100644 index 0000000000..49fbead000 --- /dev/null +++ b/packages/integrations/grok-build-sdk/tsdown.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from "tsdown"; + +export default defineConfig({ + entry: { index: "src/index.ts" }, + format: ["esm"], + dts: true, + sourcemap: true, + clean: true, + deps: { neverBundle: ["@browserbasehq/stagehand-integrations"] }, + outputOptions: { minify: false }, +}); diff --git a/packages/integrations/grok-build/.grok/config.toml b/packages/integrations/grok-build/.grok/config.toml new file mode 100644 index 0000000000..3981c0ac37 --- /dev/null +++ b/packages/integrations/grok-build/.grok/config.toml @@ -0,0 +1,15 @@ +# Grok Build project MCP entry for the Stagehand facade. Adjust the absolute +# path and credentials before copying this file into another project. + +[mcp_servers.stagehand] +command = "node" +args = [ + "/absolute/path/to/stagehand/packages/integrations/core/dist/facade/stdio-server.mjs", + "--max-screenshot-base64-bytes=60000", +] +startup_timeout_sec = 60 +tool_timeout_sec = 300 + +[mcp_servers.stagehand.env] +STAGEHAND_BROWSER = "browserbase" +BROWSERBASE_API_KEY = "bb_live_..." diff --git a/packages/integrations/grok-build/AGENTS.md b/packages/integrations/grok-build/AGENTS.md new file mode 100644 index 0000000000..85938290b0 --- /dev/null +++ b/packages/integrations/grok-build/AGENTS.md @@ -0,0 +1,9 @@ +# Stagehand browser tools in Grok Build + +The `stagehand` MCP server exposes `stagehand__run`, `stagehand__snapshot`, and +`stagehand__screenshot`. Use only those tools for browser work. + +There is no separate navigate or start tool. Open URLs with `stagehand__run`, for example +`await page.goto("https://example.com"); return { url: await page.url() };`. Use +`stagehand__snapshot` for the accessibility tree and element IDs. Use `stagehand__screenshot` +only when pixels matter. Never launch another browser or use shell commands for browsing. diff --git a/packages/integrations/grok-build/README.md b/packages/integrations/grok-build/README.md new file mode 100644 index 0000000000..f535a158c8 --- /dev/null +++ b/packages/integrations/grok-build/README.md @@ -0,0 +1,48 @@ +# Grok Build CLI + Stagehand facade over MCP/stdio + +Grok Build's `grok` CLI consumes the Stagehand facade (`run` / `snapshot` / `screenshot`) as a +project MCP server through `.grok/config.toml`. + + + +## Setup + +Use Node.js 24 or newer. From the repository root, build the integrations package first: + +```bash +pnpm install --frozen-lockfile +pnpm exec turbo run build --filter @browserbasehq/stagehand-integrations +npm install --global @xai-official/grok +grok login +# or: export XAI_API_KEY=... +``` + +## Configure + +Copy `.grok/config.toml` from this directory to your project, then replace the facade path and +Browserbase key placeholders. Grok merges project MCP configuration over its user settings. + +## Run + +Run from the configured project so Grok sees both `.grok/config.toml` and `AGENTS.md`: + +```bash +grok -p \ + --output-format streaming-json \ + --always-approve \ + --tools search_tool,use_tool \ + --disallowed-tools Agent \ + --no-plan \ + --no-subagents \ + --disable-web-search \ + "Use the stagehand MCP tools: open https://example.com, snapshot it, and report the heading citing the snapshot ID." +``` + +The eval harness uses the same CLI path with an isolated temporary Grok home and project config: + +```bash +evals run b:webvoyager --harness grok_build --tool stagehand_facade -l 1 -t 1 -e browserbase +``` + +Set `EVAL_GROK_BUILD_PATH` to override the binary, `EVAL_GROK_BUILD_MAX_TURNS` to change the +50-turn default, or `EVAL_GROK_BUILD_SANDBOX` to pass a Grok sandbox profile. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 26c9db5b54..0561917452 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -281,6 +281,9 @@ importers: '@browserbasehq/stagehand-integrations-fx-sdk': specifier: workspace:* version: link:../integrations/fx-sdk + '@browserbasehq/stagehand-integrations-grok-build-sdk': + specifier: workspace:* + version: link:../integrations/grok-build-sdk '@browserbasehq/stagehand-integrations-mastra-sdk': specifier: workspace:* version: link:../integrations/mastra-sdk @@ -326,6 +329,9 @@ importers: sharp: specifier: ^0.34.5 version: 0.34.5 + smol-toml: + specifier: 'catalog:' + version: 1.7.0 stagehand-v3: specifier: npm:@browserbasehq/stagehand@3.7.1 version: '@browserbasehq/stagehand@3.7.1(playwright-core@1.56.1)(zod@4.4.3)' @@ -669,6 +675,25 @@ importers: specifier: 'catalog:' version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@24.13.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0)) + packages/integrations/grok-build-sdk: + dependencies: + '@browserbasehq/stagehand-integrations': + specifier: workspace:* + version: link:../core + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 24.13.2 + tsdown: + specifier: 'catalog:' + version: 0.22.3(publint@0.3.21)(tsx@4.23.1)(typescript@5.9.3) + typescript: + specifier: 'catalog:' + version: 5.9.3 + vitest: + specifier: 'catalog:' + version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@24.13.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0)) + packages/integrations/mastra: dependencies: '@ai-sdk/openai': diff --git a/turbo.json b/turbo.json index 9f4dd6a96f..1a14bc03bc 100644 --- a/turbo.json +++ b/turbo.json @@ -76,6 +76,11 @@ "inputs": ["$TURBO_DEFAULT$", "!dist/**"], "outputs": ["dist/**"] }, + "@browserbasehq/stagehand-integrations-grok-build-sdk#build": { + "dependsOn": ["^build"], + "inputs": ["$TURBO_DEFAULT$", "!dist/**"], + "outputs": ["dist/**"] + }, "@browserbasehq/stagehand-evals#build": { "dependsOn": ["^build"], "inputs": ["$TURBO_DEFAULT$", "!dist/**"], @@ -229,6 +234,16 @@ "$TURBO_ROOT$/vitest.config.ts" ] }, + "@browserbasehq/stagehand-integrations-grok-build-sdk#test:unit": { + "dependsOn": ["^build", "@browserbasehq/stagehand-integrations-grok-build-sdk#build"], + "inputs": [ + "$TURBO_DEFAULT$", + "tests/**", + "src/**", + "**/*.test.ts", + "$TURBO_ROOT$/vitest.config.ts" + ] + }, "@browserbasehq/stagehand-integrations-example-mastra-facade#typecheck": { "dependsOn": ["^build"] }, @@ -256,6 +271,9 @@ "@browserbasehq/stagehand-integrations-cursor-sdk#typecheck": { "dependsOn": ["^build"] }, + "@browserbasehq/stagehand-integrations-grok-build-sdk#typecheck": { + "dependsOn": ["^build"] + }, "@browserbasehq/stagehand-docs#typecheck": {}, "test:unit": { "dependsOn": ["^build"], diff --git a/vitest.config.ts b/vitest.config.ts index 14c0cba156..34156ed53b 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -19,6 +19,7 @@ export default defineConfig({ "packages/integrations/deepagents-sdk/tests/**/*.test.ts", "packages/integrations/fx-sdk/tests/**/*.test.ts", "packages/integrations/cursor-sdk/tests/**/*.test.ts", + "packages/integrations/grok-build-sdk/tests/**/*.test.ts", "packages/extension/tests/**/*.test.ts", "packages/sdk-ts/tests/**/*.test.ts", "packages/extension/understudy/**/*.test.ts",