diff --git a/apps/mobile/src/features/browser/preview-stream.browser.ts b/apps/mobile/src/features/browser/preview-stream.browser.ts index 75cf5608b475e..f6640c6f79ce7 100644 --- a/apps/mobile/src/features/browser/preview-stream.browser.ts +++ b/apps/mobile/src/features/browser/preview-stream.browser.ts @@ -243,7 +243,8 @@ export function start(configuration: PreviewStreamConfiguration) { const height = Math.round((interactive ? next.height : side) * ratio); if (cap && cap.width >= width && cap.height >= height) return; cap = { width: Math.max(width, cap?.width ?? 0), height: Math.max(height, cap?.height ?? 0) }; - connect(); + if (client) client.setFrameSize(cap); + else connect(); }; const observer = new ResizeObserver(() => { if (resizeTimer !== null) clearTimeout(resizeTimer); diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index 8087a25e6e559..b5fc088f8b0b2 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -1525,6 +1525,68 @@ it.effect("enforces provider ownership and explicit targets when a session has m ).pipe(Effect.provide(layer)), ); +it.live("changes a viewer's frame size without releasing control or resizing the page", () => + Effect.scoped( + Effect.gen(function* () { + const { browser, broker, tabId } = yield* ready; + const viewer = yield* browser.attachViewer(viewerInput(tabId, true)); + yield* viewer.input({ type: "takeControl" }); + const page = contexts[0]!.page; + const session = contexts[0]!.sessions.at(-1)!; + page.setViewportSize.mockClear(); + session.send.mockClear(); + yield* viewer.input({ type: "streamSize", width: 1800, height: 1200 }); + expect(session.send).toHaveBeenCalledWith("Page.startScreencast", { + format: "jpeg", + quality: 70, + maxWidth: 1800, + maxHeight: 1200, + }); + expect(page.setViewportSize).not.toHaveBeenCalled(); + yield* viewer.input({ type: "text", text: "still controlled" }); + expect(session.send).toHaveBeenCalledWith("Input.insertText", { text: "still controlled" }); + const status = yield* broker.invoke({ + scope, + tabId, + operation: "status", + input: {}, + }); + expect(status.control?.owner).toBe("human"); + yield* viewer.input({ type: "releaseControl" }); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("lets a read-only viewer change only its own frame size and bounds the dimensions", () => + Effect.scoped( + Effect.gen(function* () { + const { browser, tabId } = yield* ready; + const watcher = yield* browser.attachViewer(viewerInput(tabId, false)); + const session = contexts[0]!.sessions.at(-1)!; + session.send.mockClear(); + for (const [width, height] of [ + [0, 800], + [800, -1], + [NaN, 800], + [Infinity, 800], + ["900", 800], + ]) + yield* watcher.input({ type: "streamSize", width, height }); + expect(session.send).not.toHaveBeenCalled(); + yield* watcher.input({ type: "streamSize", width: 99999, height: 99999 }); + expect(session.send).toHaveBeenCalledWith("Page.startScreencast", { + format: "jpeg", + quality: 70, + maxWidth: 7680, + maxHeight: 4320, + }); + session.send.mockClear(); + yield* watcher.input({ type: "streamSize", width: 99999, height: 99999 }); + expect(session.send).not.toHaveBeenCalled(); + }), + ).pipe(Effect.provide(layer)), +); + it.effect("a first still that never returns does not stall the tab's later captures", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 59f70c887f146..9cb06b95c94eb 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -116,6 +116,8 @@ const DESKTOP_ATTACH_TIMEOUT = "10 seconds"; // A viewer still is best-effort; a capture Chromium never answers must not hold the capture lock. const VIEWER_STILL_TIMEOUT_MS = 2_000; const VIEWER_OUTPUT_LIMIT = 64; +export const VIEWER_MAX_FRAME_WIDTH = 7680; +export const VIEWER_MAX_FRAME_HEIGHT = 4320; const RECORDING_SCREENCAST = { format: "jpeg", quality: 90, everyNthFrame: 1 } as const; const RECORDING_BINARY_EXPORT_MIN_BYTES = 8 * 1024 * 1024; const RECORDING_DOWNLOAD_TIMEOUT_MS = 5_000; @@ -3181,6 +3183,8 @@ const make = Effect.gen(function* () { ), ); const quality = Math.min(100, Math.max(1, Math.round(input.quality))); + let maxWidth = input.maxWidth; + let maxHeight = input.maxHeight; // Chromium may drop the final frame during a burst; send a still when it settles. let framesInFlight = 0; let mayHaveDropped = false; @@ -3197,8 +3201,8 @@ const make = Effect.gen(function* () { .send("Page.startScreencast", { format: "jpeg", quality: screencastScale < 1 ? Math.min(quality, SCREENCAST_MOTION_QUALITY) : quality, - maxWidth: Math.max(1, Math.round(input.maxWidth * screencastScale)), - maxHeight: Math.max(1, Math.round(input.maxHeight * screencastScale)), + maxWidth: Math.max(1, Math.round(maxWidth * screencastScale)), + maxHeight: Math.max(1, Math.round(maxHeight * screencastScale)), }) .catch(constVoid); }); @@ -3338,9 +3342,30 @@ const make = Effect.gen(function* () { output, input: (raw: unknown) => Effect.promise(async () => { - if (!viewer.canOperate) return; const message = asRecord(raw); if (!message) return; + // Frame dimensions affect only this viewer, including read-only viewers. + if (message.type === "streamSize") { + const { width, height } = message; + if ( + typeof width !== "number" || + typeof height !== "number" || + !Number.isFinite(width) || + !Number.isFinite(height) || + width < 1 || + height < 1 + ) + return; + const nextWidth = Math.min(Math.round(width), VIEWER_MAX_FRAME_WIDTH); + const nextHeight = Math.min(Math.round(height), VIEWER_MAX_FRAME_HEIGHT); + if (maxWidth === nextWidth && maxHeight === nextHeight) return; + maxWidth = nextWidth; + maxHeight = nextHeight; + await startScreencast(screencastScale); + await pushStill(); + return; + } + if (!viewer.canOperate) return; try { if (message.type === "takeControl") { const taking = tab.control.take(viewer.id); diff --git a/apps/server/src/preview/ServerBrowserStream.test.ts b/apps/server/src/preview/ServerBrowserStream.test.ts index c0a9fbc923ff2..8a428cea54458 100644 --- a/apps/server/src/preview/ServerBrowserStream.test.ts +++ b/apps/server/src/preview/ServerBrowserStream.test.ts @@ -129,6 +129,7 @@ it.effect.each([ } frames.push(new Uint8Array(event.data as ArrayBuffer)); for (const message of mutations) socket.send(JSON.stringify(message)); + socket.send(JSON.stringify({ type: "streamSize", width: 1600, height: 1000 })); socket.send("malformed input"); socket.send(JSON.stringify({ type: "ack" })); received.resolve(); @@ -138,7 +139,11 @@ it.effect.each([ yield* Effect.promise(() => acked.promise); expect(frames).toEqual([frame]); expect(viewports).toEqual([{ type: "viewport", width: 1280, height: 800 }]); - expect(inputs).toEqual(canOperate ? [...mutations, null] : []); + expect(inputs).toEqual([ + ...(canOperate ? mutations : []), + { type: "streamSize", width: 1600, height: 1000 }, + ...(canOperate ? [null] : []), + ]); expect(attachments).toEqual([ { threadId: "thread", diff --git a/apps/server/src/preview/ServerBrowserStream.ts b/apps/server/src/preview/ServerBrowserStream.ts index e46a0c76adb53..eb68ceb510b06 100644 --- a/apps/server/src/preview/ServerBrowserStream.ts +++ b/apps/server/src/preview/ServerBrowserStream.ts @@ -40,8 +40,8 @@ const intParam = (params: URLSearchParams, name: string, fallback: number, max: type Frame = Extract; -const isAck = (message: unknown) => - typeof message === "object" && message !== null && "type" in message && message.type === "ack"; +const messageType = (message: unknown) => + typeof message === "object" && message !== null && "type" in message ? message.type : undefined; const parseMessage = (chunk: Uint8Array | string): unknown => { try { @@ -86,8 +86,8 @@ const makeHandler = Effect.fnUntraced(function* (browser: ServerBrowser.ServerBr threadId, tabId, canOperate, - maxWidth: intParam(params, "maxWidth", 1280, 7680), - maxHeight: intParam(params, "maxHeight", 800, 4320), + maxWidth: intParam(params, "maxWidth", 1280, ServerBrowser.VIEWER_MAX_FRAME_WIDTH), + maxHeight: intParam(params, "maxHeight", 800, ServerBrowser.VIEWER_MAX_FRAME_HEIGHT), quality: intParam(params, "quality", DEFAULT_QUALITY, 100), }) .pipe( @@ -181,8 +181,10 @@ const makeHandler = Effect.fnUntraced(function* (browser: ServerBrowser.ServerBr const message = parseMessage(chunk); // Ownership serializes actions in the service. Keep reading so dialog // replies and takeover can unblock an action already waiting on the page. - if (!isAck(message)) - return yield* canOperate + const type = messageType(message); + if (type !== "ack") + // Read-only viewers may still size their own frames. + return yield* canOperate || type === "streamSize" ? viewer.input(message).pipe(Effect.forkScoped, Effect.asVoid) : Effect.void; const ack = inFlight; diff --git a/apps/web/src/browser/ServerBrowserSurface.browser.test.tsx b/apps/web/src/browser/ServerBrowserSurface.browser.test.tsx index 47cfb623dc7f9..7857837245338 100644 --- a/apps/web/src/browser/ServerBrowserSurface.browser.test.tsx +++ b/apps/web/src/browser/ServerBrowserSurface.browser.test.tsx @@ -14,6 +14,7 @@ const mocks = vi.hoisted(() => ({ sent: [] as PreviewStreamInput[], access: { url: "ws://preview.test", token: "test" }, painted: null as (() => void) | null, + frameSizes: vi.fn(), })); vi.mock(import("@t3tools/client-runtime/preview/server-browser-stream"), async (original) => ({ @@ -35,9 +36,11 @@ import { ServerBrowserSurface } from "./ServerBrowserSurface"; function setup() { mocks.sent.length = 0; mocks.painted = null; + mocks.frameSizes.mockReset(); mocks.createClient.mockReset().mockImplementation(() => ({ send: (input: PreviewStreamInput) => mocks.sent.push(input), stop: () => undefined, + setFrameSize: mocks.frameSizes, })); } @@ -220,3 +223,27 @@ describe("ServerBrowserSurface history", () => { expect(mouseInputs()).not.toEqual([]); }); }); + +it("keeps control and the focused input when the streamed panel grows", async () => { + setup(); + const { screen, input, canvas } = await renderSurface("you"); + await using _screen = screen; + const panel = canvas.parentElement!; + panel.style.width = "400px"; + panel.style.height = "300px"; + await vi.waitFor(() => expect(panel.getBoundingClientRect().width).toBe(400)); + input.focus(); + const connections = mocks.createClient.mock.calls.length; + panel.style.width = "700px"; + panel.style.height = "500px"; + await vi.waitFor(() => + expect(mocks.frameSizes).toHaveBeenCalledWith({ + width: Math.round(700 * window.devicePixelRatio), + height: expect.any(Number), + }), + ); + expect(mocks.createClient).toHaveBeenCalledTimes(connections); + expect(document.activeElement).toBe(input); + await userEvent.keyboard("a"); + expect(keys()).toContainEqual(expect.objectContaining({ action: "down", key: "a" })); +}); diff --git a/apps/web/src/browser/ServerBrowserSurface.tsx b/apps/web/src/browser/ServerBrowserSurface.tsx index a881650f69bcf..267a31b4b0311 100644 --- a/apps/web/src/browser/ServerBrowserSurface.tsx +++ b/apps/web/src/browser/ServerBrowserSurface.tsx @@ -220,7 +220,7 @@ export function ServerBrowserSurface(props: { ); const controlChanged = useEffectEvent((next: PreviewStreamControl | null) => onControl?.(next)); const popupOpened = useEffectEvent((popupTabId: string) => onPopup?.(popupTabId)); - const [cap, setCap] = useState<{ width: number; height: number } | null>(null); + const capRef = useRef<{ width: number; height: number } | null>(null); // Keep clearInput stable: a new identity reconnects the stream. const send = (input: PreviewStreamInput) => { @@ -332,14 +332,14 @@ export function ServerBrowserSurface(props: { const ratio = window.devicePixelRatio || 1; const width = Math.round(size.width * ratio); const height = Math.round(size.height * ratio); - setCap((current) => - current !== null && current.width >= width && current.height >= height - ? current - : { - width: Math.max(width, current?.width ?? 0), - height: Math.max(height, current?.height ?? 0), - }, - ); + const current = capRef.current; + if (current && current.width >= width && current.height >= height) return; + const cap = { + width: Math.max(width, current?.width ?? 0), + height: Math.max(height, current?.height ?? 0), + }; + capRef.current = cap; + clientRef.current?.setFrameSize(cap); }; const measure = () => { timer = null; @@ -350,7 +350,7 @@ export function ServerBrowserSurface(props: { if (previous?.width === size.width && previous.height === size.height) return; sizeRef.current = size; if (followSize) clientRef.current?.send({ type: "resize", ...size }); - // Growing the cap reconnects the stream; wait until resizing settles. + // Wait until resizing settles before increasing the frame size. if (capTimer !== null) clearTimeout(capTimer); capTimer = setTimeout(growCap, previous === null ? 0 : RESIZE_THROTTLE_MS * 2); }; @@ -368,7 +368,13 @@ export function ServerBrowserSurface(props: { // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- opens the preview stream client and paints its frames onto the canvas while visible useEffect(() => { const canvas = canvasRef.current; - if (!visible || accessDenied || hostSetup || !access || !cap || !canvas) return; + if (!visible || accessDenied || hostSetup || !access || !canvas) return; + const ratio = window.devicePixelRatio || 1; + const cap = capRef.current ?? { + width: Math.max(1, Math.round(canvas.clientWidth * ratio)), + height: Math.max(1, Math.round(canvas.clientHeight * ratio)), + }; + capRef.current = cap; let refreshTimer: ReturnType | null = null; const painter = createPreviewFramePainter(canvas, () => { if (hasFrameRef.current) return; @@ -460,7 +466,6 @@ export function ServerBrowserSurface(props: { }, [ access, accessDenied, - cap, clearInput, environmentId, followSize, diff --git a/packages/client-runtime/src/preview/serverBrowserStream.test.ts b/packages/client-runtime/src/preview/serverBrowserStream.test.ts index 7b749acea4402..7746f30810b64 100644 --- a/packages/client-runtime/src/preview/serverBrowserStream.test.ts +++ b/packages/client-runtime/src/preview/serverBrowserStream.test.ts @@ -63,6 +63,30 @@ describe("preview stream control", () => { dialog: null, }; + it("changes frame size on the same socket while watching or controlling", () => { + const { client, socket } = connect(); + socket.control({ ...agent, canOperate: false }); + client.setFrameSize({ width: 1600, height: 1000 }); + expect(socket.sent).toEqual(['{"type":"streamSize","width":1600,"height":1000}']); + socket.control({ ...agent, controller: "you" }); + client.setFrameSize({ width: 1800, height: 1200 }); + expect(FakeSocket.current).toBe(socket); + expect(client.send({ type: "text", text: "still controlled" })).toBe(true); + expect(socket.sent.at(-1)).toBe('{"type":"text","text":"still controlled"}'); + client.stop(); + }); + + it("uses the latest frame size if it changes before the socket opens", () => { + const { client, socket } = connect(); + socket.readyState = 0; + client.setFrameSize({ width: 1800, height: 1200 }); + expect(socket.sent).toEqual([]); + socket.readyState = 1; + socket.dispatchEvent(new Event("open")); + expect(socket.sent).toEqual(['{"type":"streamSize","width":1800,"height":1200}']); + client.stop(); + }); + it("does not grant control from malformed dialog metadata", () => { const { client, socket, onControl } = connect(); socket.message( diff --git a/packages/client-runtime/src/preview/serverBrowserStream.ts b/packages/client-runtime/src/preview/serverBrowserStream.ts index 75b45c57e7217..a4bee1e77f13d 100644 --- a/packages/client-runtime/src/preview/serverBrowserStream.ts +++ b/packages/client-runtime/src/preview/serverBrowserStream.ts @@ -201,6 +201,7 @@ export interface PreviewStreamEvents { } export interface PreviewStreamClient { + readonly setFrameSize: (size: PreviewStreamViewport) => void; readonly send: (input: PreviewStreamInput) => boolean; readonly stop: () => void; } @@ -217,7 +218,7 @@ export function createPreviewStreamClient( maxHeight: String(Math.max(1, Math.round(target.maxHeight))), }); if (target.interactive === false) query.set("interactive", "false"); - const url = withDeviceHubQuery(`${target.access.wsBase}/ws?${query.toString()}`, target.access); + let frameSizeChanged = false; let stopped = false; let socket: WebSocket | null = null; let retryTimer: ReturnType | null = null; @@ -234,13 +235,16 @@ export function createPreviewStreamClient( const connect = () => { if (stopped) return; - const ws = new WebSocket(url); + const ws = new WebSocket( + withDeviceHubQuery(`${target.access.wsBase}/ws?${query.toString()}`, target.access), + ); ws.binaryType = "arraybuffer"; socket = ws; let opened = false; ws.addEventListener("open", () => { if (socket !== ws) return; opened = true; + if (frameSizeChanged) sendFrameSize(); events.onConnectedChange(true); }); ws.addEventListener("message", (event) => { @@ -378,9 +382,30 @@ export function createPreviewStreamClient( ws.addEventListener("error", () => ws.close()); }; + const sendFrameSize = () => { + if (socket?.readyState !== WebSocket.OPEN) return; + socket.send( + JSON.stringify({ + type: "streamSize", + width: Number(query.get("maxWidth")), + height: Number(query.get("maxHeight")), + }), + ); + }; + connect(); return { + setFrameSize: ({ width, height }) => { + if (stopped) return; + const maxWidth = String(Math.max(1, Math.round(width))); + const maxHeight = String(Math.max(1, Math.round(height))); + if (query.get("maxWidth") === maxWidth && query.get("maxHeight") === maxHeight) return; + query.set("maxWidth", maxWidth); + query.set("maxHeight", maxHeight); + frameSizeChanged = true; + sendFrameSize(); + }, send: (input) => { if (socket?.readyState !== WebSocket.OPEN) return false; if (!control?.canOperate) return false;