Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 23 additions & 3 deletions apps/server/src/mcp/McpHttpServer.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,22 @@ it.effect.each([
),
);

it.effect("rejects a metadata-only response when the screenshot is saved", () =>
Effect.scoped(
Effect.gen(function* () {
yield* serveSnapshots("mcp-missing-pixels", {
...snapshotResult,
screenshot: { width: 10, height: 5 },
});
const result = yield* callSnapshot({ save: true });
expect(result.isError).toBe(true);
expect(result.content).toEqual([
{ type: "text", text: expect.stringContaining("did not contain requested PNG pixels") },
]);
}),
).pipe(Effect.provide(layerTest)),
);

const layerRecordingTools = (calls: Array<unknown>) => {
const RecordingToolkit = Toolkit.make(
OrchestratorToolkit.tools.delegate_task,
Expand Down Expand Up @@ -418,7 +434,9 @@ it.effect.each([
tabId: alternateTabId,
threadId,
});
expect(event.request.input).toEqual({});
expect(event.request.input).toEqual(
images && requests !== 7 ? {} : { pixelsRequired: false },
);
return broker.respond({
clientId: "mcp-image-option-client",
connectionId: event.connectionId,
Expand All @@ -427,7 +445,8 @@ it.effect.each([
result: {
...page,
title: `Snapshot ${requests}`,
screenshot: { ...screenshot, data: png },
screenshot:
images || requests === 7 ? { ...screenshot, data: png } : { width: 1, height: 1 },
},
});
}).pipe(Effect.forkScoped);
Expand Down Expand Up @@ -545,11 +564,12 @@ it.effect("names the screenshot destination when saving it fails", () =>
Effect.gen(function* () {
const config = yield* ServerConfig.ServerConfig;
yield* (yield* FileSystem.FileSystem).writeFileString(config.browserArtifactsDir, "occupied");
yield* serveSnapshots("mcp-save-failure-client", snapshotResult);
const inputs = yield* serveSnapshots("mcp-save-failure-client", snapshotResult);

const snapshot = yield* callSnapshot({ tabId: alternateTabId, save: true });
const text = snapshot.content[0]?.type === "text" ? snapshot.content[0].text : "";

expect(inputs).toEqual([{ fullScreenshot: true }]);
expect(snapshot.isError).toBe(true);
expect(text).toMatch(/^Preview snapshot failed: Could not save preview screenshot to /);
expect(text).toContain(config.browserArtifactsDir);
Expand Down
30 changes: 21 additions & 9 deletions apps/server/src/mcp/McpHttpServer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -574,7 +574,9 @@ const previewSnapshotFailure = Effect.fn("previewSnapshotFailure")(function <E>(
: "PreviewSnapshotError";
// Preview errors contain server-authored recovery guidance, never page output.
const message =
isPreviewAutomationError(firstFailure) || isPreviewScreenshotSaveError(firstFailure)
isPreviewAutomationError(firstFailure) ||
isPreviewScreenshotSaveError(firstFailure) ||
isOrchestratorMcpFailure(firstFailure)
? firstFailure.message
: parameterValidationMessage(firstFailure);
const result = new McpSchema.CallToolResult({
Expand Down Expand Up @@ -641,17 +643,28 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
const snapshot = encodedResult as SnapshotMetadata & {
readonly url: string;
readonly screenshot: {
readonly mimeType: "image/png";
readonly data: string;
readonly mimeType?: "image/png";
readonly data?: string;
readonly width: number;
readonly height: number;
};
readonly fullScreenshot?: { readonly data: string };
};
const { screenshot, fullScreenshot, ...page } = snapshot;
const png = new Uint8Array(Buffer.from(screenshot.data, "base64"));
const includeImage = payload?.includeImage === true;
const needsPixels = payload?.save === true || includeImage;
if (needsPixels && (screenshot.mimeType !== "image/png" || !screenshot.data)) {
return yield* new OrchestratorMcpFailure({
code: "invalid_request",
message: "Preview snapshot did not contain requested PNG pixels.",
});
}
const png =
needsPixels && screenshot.data
? new Uint8Array(Buffer.from(screenshot.data, "base64"))
: undefined;
const screenshotPath =
payload?.save === true
payload?.save === true && png
? yield* saveScreenshot(
snapshot.url,
fullScreenshot
Expand All @@ -660,7 +673,6 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
)
: undefined;
// Providers replay images on later requests, and some reject them; include images only when requested.
const includeImage = payload?.includeImage === true;
if (screenshotPath !== undefined && !includeImage) {
const saved = {
url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS),
Expand All @@ -675,7 +687,7 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
const metadata = {
...page,
screenshot: {
mimeType: screenshot.mimeType,
mimeType: "image/png" as const,
width: screenshot.width,
height: screenshot.height,
},
Expand Down Expand Up @@ -704,8 +716,8 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
text: `Snapshot text was bounded. Omitted: ${bounded.omitted.join("; ")}.`,
},
]),
...(includeImage
? [{ type: "image" as const, data: png, mimeType: screenshot.mimeType }]
...(includeImage && png
? [{ type: "image" as const, data: png, mimeType: "image/png" as const }]
: []),
],
});
Expand Down
15 changes: 10 additions & 5 deletions apps/server/src/mcp/toolkits/preview/handlers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import {
type PreviewAutomationResizeResult,
type PreviewAutomationSetColorSchemeResult,
type PreviewAutomationSnapshot,
type PreviewAutomationSnapshotMetadata,
type PreviewAutomationStatus,
type PreviewTabId,
} from "@t3tools/contracts";
Expand Down Expand Up @@ -221,12 +222,16 @@ const handlers = {
invokeTargeted<PreviewAutomationSetColorSchemeResult>("setColorScheme", input),
),
preview_snapshot: McpToolAccess.readsAsCaller((input) => {
const { includeImage: _includeImage, save, ...operationInput } = input ?? {};
const { includeImage, save, ...operationInput } = input ?? {};
// The saved file is for people, so it keeps the resolution the agent's image is capped below.
return invokeTargeted<PreviewAutomationSnapshot>("snapshot", {
...operationInput,
...(save === true ? { fullScreenshot: true } : {}),
});
return invokeTargeted<PreviewAutomationSnapshot | PreviewAutomationSnapshotMetadata>(
"snapshot",
{
...operationInput,
...(save === true ? { fullScreenshot: true } : {}),
...(includeImage === true || save === true ? {} : { pixelsRequired: false }),
},
);
}),
preview_act: McpToolAccess.actsAsCaller(({ action, ...input }) =>
decodeActInput[action](input).pipe(
Expand Down
3 changes: 2 additions & 1 deletion apps/server/src/mcp/toolkits/preview/tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import {
PreviewAutomationSetColorSchemeInput,
PreviewAutomationSetColorSchemeResult,
PreviewAutomationSnapshot,
PreviewAutomationSnapshotMetadata,
PreviewAutomationStatus,
PreviewAutomationTabTargetInput,
PreviewAutomationWaitForInput,
Expand Down Expand Up @@ -151,7 +152,7 @@ export const PreviewSnapshotTool = safeBrowserTool(
}),
),
}),
success: PreviewAutomationSnapshot,
success: Schema.Union([PreviewAutomationSnapshot, PreviewAutomationSnapshotMetadata]),
failure: PreviewToolFailure,
dependencies,
}).annotate(Tool.Title, "Inspect browser page"),
Expand Down
14 changes: 9 additions & 5 deletions apps/server/src/preview/ServerBrowser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2830,12 +2830,16 @@ const make = Effect.gen(function* () {
return { tabId: tab.tabId, colorScheme };
}
case "snapshot": {
const snapshotInput = input as PreviewAutomationSnapshotInput | undefined;
const rendering = {
...tab,
renderScale: tab.desktop ? "display" : RENDER_SCALE,
fullScreenshot: snapshotInput?.fullScreenshot,
} as const;
const snapshot = await withScreencastsPaused(tab, () =>
ServerBrowserPage.snapshot({
...tab,
renderScale: tab.desktop ? "display" : RENDER_SCALE,
fullScreenshot: (input as PreviewAutomationSnapshotInput | undefined)?.fullScreenshot,
}),
snapshotInput?.pixelsRequired === false
? ServerBrowserPage.snapshotMetadata(rendering)
: ServerBrowserPage.snapshot(rendering),
);
const loadError = loadErrorOf(tab);
return loadError ? { ...snapshot, url: loadError.url, loadError } : snapshot;
Expand Down
36 changes: 31 additions & 5 deletions apps/server/src/preview/ServerBrowserPage.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,24 @@ import { SessionControl } from "./SessionControl.ts";

const decodeEvaluateInput = Schema.decodeUnknownSync(PreviewAutomationEvaluateInput);

describe("JPEG screenshot dimensions", () => {
const encode = (bytes: number[]) => Buffer.from(bytes).toString("base64");
const frame = [0xff, 0xd8, 0xff, 0xc0, 0, 7, 8, 0, 24, 0, 32];

it("reads valid dimensions and rejects invalid or incomplete frames", () => {
expect(ServerBrowserPage.jpegSize(encode(frame))).toEqual({ width: 32, height: 24 });
for (const bytes of [
[],
[0xff, 0xd8, 0xff, 0xe1, 0, 20, 1],
frame.slice(0, -1),
[...frame.slice(0, 7), 0, 0, 0, 32],
[...frame.slice(0, 9), 0, 0],
]) {
expect(() => ServerBrowserPage.jpegSize(encode(bytes))).toThrow("Invalid JPEG screenshot.");
}
});
});

describe("snapshot deadline diagnostics", () => {
const never = () => new Promise<never>(() => {});
const stalledSnapshot = async (options: {
Expand Down Expand Up @@ -591,15 +609,16 @@ describe("server browser snapshot screenshot", () => {
const screenshotOf = async (html: string) => {
const page = await context.newPage();
await page.setContent(html);
const { screenshot, fullScreenshot } = await ServerBrowserPage.snapshot({
const input = {
page,
cdp: await context.newCDPSession(page),
renderScale: 2,
fullScreenshot: true,
consoleEntries: [],
networkEntries: [],
actionTimeline: [],
});
};
const { screenshot, fullScreenshot } = await ServerBrowserPage.snapshot(input);
// The server has no DOM types, so the decoder runs as page source.
const points = [
[0.15, 0.75],
Expand All @@ -618,7 +637,7 @@ describe("server browser snapshot screenshot", () => {
Array.from(context.getImageData(x * bitmap.width, y * bitmap.height, 1, 1).data),
);
})()`)) as Array<[number, number, number, number]>;
return { screenshot, fullScreenshot, pixels };
return { screenshot, fullScreenshot, pixels, input };
};
const colorOf = ([red, green, blue]: readonly [number, number, number, number]) =>
blue - red > 100 ? "blue" : red - blue > 100 ? "red" : `rgb(${red}, ${green}, ${blue})`;
Expand All @@ -633,13 +652,20 @@ describe("server browser snapshot screenshot", () => {
});

it("captures a plain page at the shrunk size with its content in place", async () => {
const { screenshot, fullScreenshot, pixels } = await screenshotOf(halves(""));
const { screenshot, fullScreenshot, pixels, input } = await screenshotOf(halves(""));
const metadata = await ServerBrowserPage.snapshotMetadata(input);

expect({
size: [screenshot.width, screenshot.height],
fullSize: [fullScreenshot?.width, fullScreenshot?.height],
metadataSize: metadata.screenshot,
colors: pixels.map(colorOf),
}).toEqual({ size: [1280, 800], fullSize: [2560, 1600], colors: ["blue", "red", "red"] });
}).toEqual({
size: [1280, 800],
fullSize: [2560, 1600],
metadataSize: { width: 1280, height: 800 },
colors: ["blue", "red", "red"],
});
});

it("records the destination of a finite entrance animation without waiting on infinite ones", async () => {
Expand Down
Loading
Loading