From 356b1afc71901c50c3c6e79160920fb2b207076b Mon Sep 17 00:00:00 2001 From: Brett Chien <1193046+brettchien@users.noreply.github.com> Date: Mon, 14 Sep 2026 00:42:53 +0800 Subject: [PATCH] feat(chat): render images in agent turns, paste-to-send from the input Closes #158. The ACP wire already supports image ContentBlocks; the gap was the bridge (only forwarded text chunks) and the console (no image render, no paste handling). Bridge: acp-tunnel classifies session/update content by type (text/image), remote.rs forwards image chunks as their own agent-update and threads PromptImage attachments through agent_prompt/session.prompt. Console: chat.ts renders images behind a mime allow-list + base64-shape check (agent content is less trusted than markdown text), chatPanel.ts adds a paste handler with a 5MB/image, 4-images/turn cap and a removable preview strip above the input. Co-Authored-By: Claude Sonnet 5 --- console/src/chat.test.ts | 69 +++++++++++++++++ console/src/chat.ts | 78 ++++++++++++++++++- console/src/chatPanel.ts | 141 +++++++++++++++++++++++++++++++++-- console/src/main.ts | 6 ++ console/src/source.ts | 7 +- console/src/styles.css | 49 ++++++++++++ crates/acp-tunnel/src/lib.rs | 139 +++++++++++++++++++++++++++++----- src-tauri/src/lib.rs | 3 +- src-tauri/src/remote.rs | 28 +++++-- 9 files changed, 481 insertions(+), 39 deletions(-) diff --git a/console/src/chat.test.ts b/console/src/chat.test.ts index f7db9cf..94f8403 100644 --- a/console/src/chat.test.ts +++ b/console/src/chat.test.ts @@ -6,10 +6,18 @@ import { appendUser, settlePending, appendChunk, + appendImage, endTurn, + isRenderableImage, type ChatTurn, + type ChatImage, } from "./chat"; +const PNG_PIXEL: ChatImage = { + data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", + mimeType: "image/png", +}; + function agent(partial: Partial): ChatTurn { return { id: 1, role: "agent", text: "", streaming: false, ...partial }; } @@ -93,6 +101,41 @@ describe("transcript reducers", () => { appendUser(start, 1, "x"); expect(start).toHaveLength(0); }); + + it("appendImage opens an agent turn on the first image, appends after", () => { + let turns = appendUser([], 1, "hi"); + turns = appendImage(turns, 2, PNG_PIXEL); + expect(turns).toHaveLength(2); + expect(turns[1]).toMatchObject({ role: "agent", streaming: true, id: 2, images: [PNG_PIXEL] }); + const second: ChatImage = { ...PNG_PIXEL, mimeType: "image/jpeg" }; + turns = appendImage(turns, 99, second); + // still one agent turn, id preserved from the opening image + expect(turns).toHaveLength(2); + expect(turns[1].images).toEqual([PNG_PIXEL, second]); + }); + + it("appendUser attaches images to the user's own turn", () => { + const turns = appendUser([], 1, "look", false, [PNG_PIXEL]); + expect(turns[0].images).toEqual([PNG_PIXEL]); + }); +}); + +describe("isRenderableImage", () => { + it("accepts an allow-listed mime type with base64-shaped data", () => { + expect(isRenderableImage(PNG_PIXEL)).toBe(true); + }); + + it("rejects a mime type outside the allow-list", () => { + expect(isRenderableImage({ ...PNG_PIXEL, mimeType: "image/svg+xml" })).toBe(false); + }); + + it("rejects data that isn't valid base64 shape (defends the src attribute)", () => { + expect(isRenderableImage({ ...PNG_PIXEL, data: '">' })).toBe(false); + }); + + it("rejects empty data", () => { + expect(isRenderableImage({ ...PNG_PIXEL, data: "" })).toBe(false); + }); }); describe("turnHtml / transcriptHtml", () => { @@ -142,4 +185,30 @@ describe("turnHtml / transcriptHtml", () => { it("renders an empty-state message for no turns", () => { expect(transcriptHtml([])).toContain("chat-empty"); }); + + it("renders a valid image as a data-URI ", () => { + const html = turnHtml(agent({ text: "look", streaming: false, images: [PNG_PIXEL] }), mdToHtml); + expect(html).toContain("chat-images"); + expect(html).toContain(`data:${PNG_PIXEL.mimeType};base64,${PNG_PIXEL.data}`); + }); + + it("drops an unrenderable image instead of emitting it", () => { + const bad: ChatImage = { data: "not base64!!", mimeType: "image/png" }; + const html = turnHtml(agent({ text: "look", streaming: false, images: [bad] }), mdToHtml); + expect(html).not.toContain("chat-images"); + expect(html).not.toContain(bad.data); + }); + + it("renders images on a streaming turn too", () => { + const html = turnHtml(agent({ text: "", streaming: true, images: [PNG_PIXEL] }), mdToHtml); + expect(html).toContain("chat-images"); + }); + + it("renders the operator's own attached image on a user turn", () => { + const html = turnHtml( + { id: 1, role: "user", text: "look", streaming: false, images: [PNG_PIXEL] }, + mdToHtml, + ); + expect(html).toContain("chat-images"); + }); }); diff --git a/console/src/chat.ts b/console/src/chat.ts index 814735f..421f97d 100644 --- a/console/src/chat.ts +++ b/console/src/chat.ts @@ -25,6 +25,61 @@ export interface ChatTurn { // a "queued" state so it isn't invisible until the queue drains. Cleared when // its turn actually starts. pending?: boolean; + // Inline images attached to this turn (issue #158) — pasted by the operator on + // a `user` turn, or streamed in from the agent's reply on an `agent` turn. + // Rendered after the turn's text; order relative to interleaved text chunks + // isn't preserved, the same simplification `appendChunk` already makes for + // multiple text chunks accumulating into one string. + images?: ChatImage[]; +} + +// One inline image — the wire shape is already `{ data, mimeType }` end to end +// (ACP `ImageContent`, `agent-update` events, the `agent_prompt` payload), so the +// transcript keeps it as-is rather than inventing another shape. +export interface ChatImage { + data: string; + mimeType: string; +} + +// Only these are rendered — image content is agent- or clipboard-sourced, less +// trusted than markdown text, so it gets an allow-list instead of open season on +// ``. +const ALLOWED_IMAGE_MIME = new Set(["image/png", "image/jpeg", "image/gif", "image/webp"]); + +// Base64's alphabet has no `<`/`>`/`"`/`'` — a `data` value that doesn't match +// this shape isn't real base64 and must not reach a `src` attribute. +const BASE64_RE = /^[A-Za-z0-9+/]+={0,2}$/; + +// Hard cap on one image's base64 payload: ~8MB decoded (≈10.9M base64 chars). +// Mirrors the ~5MB raw-file cap `chatPanel.ts` enforces before it ever base64s a +// pasted image — this is the render-side half of the same guard, so a malformed +// or oversized image (however it got here) is dropped rather than rendered. +const MAX_IMAGE_BASE64_CHARS = Math.ceil((8 * 1024 * 1024 * 4) / 3); + +// Exported so `chatPanel.ts` can apply the same allow-list/shape check to a +// freshly pasted image before it ever gets attached to an outgoing turn. +export function isRenderableImage(img: ChatImage): boolean { + return ( + ALLOWED_IMAGE_MIME.has(img.mimeType) && + img.data.length > 0 && + img.data.length <= MAX_IMAGE_BASE64_CHARS && + BASE64_RE.test(img.data) + ); +} + +// `mimeType` is allow-listed and `data` is base64-shape-checked above, so neither +// can carry a quote/angle-bracket — safe to interpolate directly into the `src` +// attribute without a further escape pass. +function imagesHtml(images: ChatImage[] | undefined): string { + const valid = (images ?? []).filter(isRenderableImage); + if (valid.length === 0) return ""; + return ( + `
` + + valid + .map((img) => `pasted image`) + .join("") + + `
` + ); } // Markdown → HTML for a finalized agent turn. `html: false` escapes any raw HTML @@ -63,6 +118,7 @@ export function turnHtml(turn: ChatTurn, renderMd: RenderMarkdown): string { return ( `
` + `
${renderMd(turn.text)}
` + + imagesHtml(turn.images) + queued + `
` ); @@ -74,6 +130,7 @@ export function turnHtml(turn: ChatTurn, renderMd: RenderMarkdown): string { `
` + `
${escapeHtml(turn.text)}` + `
` + + imagesHtml(turn.images) + `
` ); } @@ -84,6 +141,7 @@ export function turnHtml(turn: ChatTurn, renderMd: RenderMarkdown): string { return ( `
` + `
${renderMd(turn.text)}
` + + imagesHtml(turn.images) + `
` + `${reason}` + `
` + @@ -113,8 +171,9 @@ export function appendUser( id: number, text: string, pending = false, + images?: ChatImage[], ): ChatTurn[] { - return [...turns, { id, role: "user", text, streaming: false, pending }]; + return [...turns, { id, role: "user", text, streaming: false, pending, images }]; } // Clear the `pending` flag on the user turn `id` — its queued message is now the @@ -138,6 +197,23 @@ export function appendChunk( return [...turns, { id, role: "agent", text, streaming: true }]; } +// Append a streamed inline `image` (issue #158). Mirrors `appendChunk`: opens the +// turn on the first update of a reply (chunk or image), otherwise appends to the +// still-open turn's image list. +export function appendImage( + turns: ChatTurn[], + id: number, + image: ChatImage, +): ChatTurn[] { + const last = turns[turns.length - 1]; + if (last && last.role === "agent" && last.streaming) { + const images = [...(last.images ?? []), image]; + const updated: ChatTurn = { ...last, images }; + return [...turns.slice(0, -1), updated]; + } + return [...turns, { id, role: "agent", text: "", streaming: true, images: [image] }]; +} + // Close the open agent turn on `turn_end`. If a `turn_end` arrives with no open // turn (e.g. a cancel before any chunk), synthesize an empty finalized turn so // the stop reason is still visible. No-op shape otherwise stays pure. diff --git a/console/src/chatPanel.ts b/console/src/chatPanel.ts index 5ba9065..a30d466 100644 --- a/console/src/chatPanel.ts +++ b/console/src/chatPanel.ts @@ -20,8 +20,11 @@ import { appendUser, settlePending, appendChunk, + appendImage, endTurn, + isRenderableImage, type ChatTurn, + type ChatImage, } from "./chat"; import type { Source } from "./source"; @@ -63,6 +66,7 @@ export interface ChatPanelOptions { export interface ChatPanel { readonly agent?: string; onChunk(text: string): void; + onImage(image: ChatImage): void; onTurnEnd(stopReason: string): void; setConnected(connected: boolean): void; isConnected(): boolean; @@ -78,6 +82,28 @@ function renderMarkdownBody(text: string): string { return DOMPurify.sanitize(mdToHtml(text)); } +// Max raw (pre-base64) bytes for one pasted image, and max images per message. +// ACP ships images as base64-in-JSON over the same single WS/stdio pipe as +// turn-taking chat (issue #158's stated concern) — no downscale in this slice, +// an oversized paste is rejected with a note rather than silently shrunk. +const MAX_IMAGE_BYTES = 5 * 1024 * 1024; +const MAX_IMAGES_PER_TURN = 4; + +// `FileReader.readAsDataURL` yields `"data:;base64,"` — keep only +// the payload, the mime type is read separately off the `File`. +function fileToBase64(file: File): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => { + const result = reader.result as string; + const comma = result.indexOf(","); + resolve(comma === -1 ? result : result.slice(comma + 1)); + }; + reader.onerror = () => reject(reader.error ?? new Error("image read failed")); + reader.readAsDataURL(file); + }); +} + export function createChatPanel( els: ChatPanelElements, opts: ChatPanelOptions, @@ -88,14 +114,39 @@ export function createChatPanel( let turnActive = false; // Each queued prompt carries the id of its (already-rendered) pending turn, so // `flush` can settle that exact turn when it starts sending. - const queue: { id: number; text: string }[] = []; + const queue: { id: number; text: string; images: ChatImage[] }[] = []; let seq = 0; let connected = false; + // Images pasted into `els.text` since the last send (issue #158) — held here so + // `submit()` can bundle them into the next queued turn; cleared once queued. + let pendingImages: ChatImage[] = []; // DOM listeners are scoped to this controller so `dispose()` removes them all // at once — an agent console mounts a fresh panel each time it opens. const ac = new AbortController(); const { signal } = ac; + // A small preview strip above the textarea so the operator can see (and drop) + // an attached image before sending. Built here rather than threaded through + // `ChatPanelElements` (and both its callers) — this panel already owns + // `els.form`'s DOM, so growing it in place keeps the change panel-local. + const attachEl = document.createElement("div"); + attachEl.className = "chat-attachments"; + attachEl.hidden = true; + els.form.insertBefore(attachEl, els.text); + + function renderAttachments(): void { + attachEl.hidden = pendingImages.length === 0; + attachEl.innerHTML = pendingImages + .map( + (img, i) => + `` + + `pasted image` + + `` + + ``, + ) + .join(""); + } + function errText(e: unknown): string { return e instanceof Error ? e.message : String(e); } @@ -123,16 +174,21 @@ export function createChatPanel( } // Enqueue a prompt and try to release it. `flush` is the single choke point - // that enforces one-turn-at-a-time; typing mid-turn just grows the queue. + // that enforces one-turn-at-a-time; typing mid-turn just grows the queue. Text + // is optional when at least one image is attached (issue #158) — an image-only + // message is valid. function submit(text: string): void { const trimmed = text.trim(); - if (!trimmed) return; + const images = pendingImages; + if (!trimmed && images.length === 0) return; + pendingImages = []; + renderAttachments(); // Show the message immediately — as `pending` if a turn is in flight (so it // sits visibly queued), else it is settled the instant `flush` runs. seq += 1; const id = seq; - turns = appendUser(turns, id, trimmed, turnActive); - queue.push({ id, text: trimmed }); + turns = appendUser(turns, id, trimmed, turnActive, images.length ? images : undefined); + queue.push({ id, text: trimmed, images }); render(); void flush(); } @@ -146,7 +202,7 @@ export function createChatPanel( render(); updateControls(); try { - await opts.source.agentPrompt(next.text, opts.agent); + await opts.source.agentPrompt(next.text, opts.agent, next.images.length ? next.images : undefined); if (opts.mock) mockReply(next.text); // browser preview: synthesize the reply } catch (e) { // Send failed (not connected / socket just closed): close the turn with an @@ -171,6 +227,17 @@ export function createChatPanel( render(); } + // A streamed inline `image` (issue #158): same open-turn rule as `onChunk` — + // the image and text chunks of one reply share a turn regardless of arrival + // order. + function onImage(image: ChatImage): void { + const last = turns[turns.length - 1]; + const open = last?.role === "agent" && last.streaming; + const id = open ? (last as ChatTurn).id : (seq += 1); + turns = appendImage(turns, id, image); + render(); + } + // `turn_end`: finalize the open agent turn (markdown render), free the gate, // and release any queued prompt. function onTurnEnd(stopReason: string): void { @@ -245,6 +312,58 @@ export function createChatPanel( }, { signal }, ); + // Paste an image → attach it to the next send instead of dropping it (issue + // #158). A paste with no image items falls through to the default text paste. + els.text.addEventListener( + "paste", + (ev) => { + const items = ev.clipboardData?.items; + if (!items) return; + const files = Array.from(items) + .filter((it) => it.kind === "file" && it.type.startsWith("image/")) + .map((it) => it.getAsFile()) + .filter((f): f is File => f !== null); + if (files.length === 0) return; + ev.preventDefault(); + for (const file of files) { + if (pendingImages.length >= MAX_IMAGES_PER_TURN) { + opts.note("error", `chat: only ${MAX_IMAGES_PER_TURN} images per message`); + break; + } + if (file.size > MAX_IMAGE_BYTES) { + opts.note( + "error", + `chat: image too large (${(file.size / 1024 / 1024).toFixed(1)}MB, max ${MAX_IMAGE_BYTES / 1024 / 1024}MB)`, + ); + continue; + } + void fileToBase64(file).then((data) => { + const image: ChatImage = { data, mimeType: file.type }; + if (!isRenderableImage(image)) { + opts.note("error", `chat: unsupported image type — ${file.type}`); + return; + } + pendingImages = [...pendingImages, image]; + renderAttachments(); + }); + } + }, + { signal }, + ); + // Delegated remove: drop one pending attachment before it's sent. + attachEl.addEventListener( + "click", + (ev) => { + const btn = (ev.target as HTMLElement).closest( + "button.chat-attachment-remove", + ); + if (!btn) return; + const idx = Number(btn.dataset.index); + pendingImages = pendingImages.filter((_, i) => i !== idx); + renderAttachments(); + }, + { signal }, + ); els.stop.addEventListener("click", () => void stopTurn(), { signal }); // Delegated copy: copy the raw turn text (not the rendered HTML). els.log.addEventListener( @@ -275,10 +394,18 @@ export function createChatPanel( return { agent: opts.agent, onChunk, + onImage, onTurnEnd, setConnected, isConnected: () => connected, render, - dispose: () => ac.abort(), + // `attachEl` was inserted into the caller's static form DOM (not owned by + // `els`), so it must be pulled back out — an agent console re-binds a fresh + // panel to the same `
` each time it opens, and a leftover node would + // both linger visibly and accumulate a duplicate every re-open. + dispose: () => { + ac.abort(); + attachEl.remove(); + }, }; } diff --git a/console/src/main.ts b/console/src/main.ts index cada376..401339f 100644 --- a/console/src/main.ts +++ b/console/src/main.ts @@ -851,6 +851,9 @@ async function registerManagementPanel(): Promise { function routeChunk(agent: string, text: string): void { chatPanels.get(agent)?.onChunk(text); } +function routeImage(agent: string, data: string, mimeType: string): void { + chatPanels.get(agent)?.onImage({ data, mimeType }); +} function routeTurnEnd(agent: string, stopReason: string): void { chatPanels.get(agent)?.onTurnEnd(stopReason); } @@ -882,11 +885,14 @@ async function bindAgentUpdates(): Promise { agent?: string; kind?: string; text?: string; + data?: string; + mimeType?: string; stopReason?: string; }>("agent-update", (e) => { const p = e.payload; const agent = p.agent ?? managementName ?? "management"; if (p.kind === "chunk") routeChunk(agent, p.text ?? ""); + else if (p.kind === "image" && p.data && p.mimeType) routeImage(agent, p.data, p.mimeType); else if (p.kind === "turn_end") routeTurnEnd(agent, p.stopReason ?? "end_turn"); }); } diff --git a/console/src/source.ts b/console/src/source.ts index 1a40f47..d980511 100644 --- a/console/src/source.ts +++ b/console/src/source.ts @@ -9,6 +9,7 @@ import type { RemoteConfig, RuntimeContext, } from "./types"; +import type { ChatImage } from "./chat"; import { FIXTURE_AGENTS, FIXTURE_DEPLOYMENTS, @@ -70,7 +71,7 @@ export interface Source { // names the registry endpoint; omitted ⇒ the management endpoint. The reply // streams back as `agent-update` events, not through this call, so both // resolve immediately. - agentPrompt(text: string, agent?: string): Promise; + agentPrompt(text: string, agent?: string, images?: ChatImage[]): Promise; agentCancel(agent?: string): Promise; // The remote file editor's read path (Part D). fs is an MCP files server the // target agent exposes, reached Studio-brokered via the `oab` reverse-MCP tool @@ -212,8 +213,8 @@ export class TauriSource implements Source { ); return res.agents; } - async agentPrompt(text: string, agent?: string): Promise { - await this.invoke()("agent_prompt", { text, agent }); + async agentPrompt(text: string, agent?: string, images?: ChatImage[]): Promise { + await this.invoke()("agent_prompt", { text, agent, images }); } async agentCancel(agent?: string): Promise { await this.invoke()("agent_cancel", { agent }); diff --git a/console/src/styles.css b/console/src/styles.css index f224917..c7e1c45 100644 --- a/console/src/styles.css +++ b/console/src/styles.css @@ -1139,6 +1139,19 @@ button.act:disabled { font-size: 11px; color: var(--warn); } +/* Inline images (issue #158) — agent replies or the operator's own pasted turn. */ +.chat-images { + display: flex; + flex-wrap: wrap; + gap: 6px; + margin-top: 4px; +} +.chat-image { + max-width: 260px; + max-height: 260px; + border-radius: 6px; + border: 1px solid var(--border); +} .chat-status { font-size: 12px; color: var(--muted); @@ -1150,10 +1163,46 @@ button.act:disabled { } .chat-input { display: flex; + flex-wrap: wrap; gap: 8px; margin-top: 8px; align-items: flex-end; } +/* Pending pasted images (issue #158) — forced onto their own full-width row + above the textarea/actions via `order` + a 100% flex-basis. */ +.chat-attachments { + order: -1; + flex: 0 0 100%; + display: flex; + flex-wrap: wrap; + gap: 6px; +} +.chat-attachment { + position: relative; + display: inline-flex; +} +.chat-attachment img { + width: 48px; + height: 48px; + object-fit: cover; + border-radius: 5px; + border: 1px solid var(--border); +} +.chat-attachment-remove { + position: absolute; + top: -6px; + right: -6px; + width: 16px; + height: 16px; + line-height: 14px; + padding: 0; + cursor: pointer; + border: 1px solid var(--border); + border-radius: 50%; + background: var(--bg); + color: var(--text); + font-size: 11px; +} .chat-text { flex: 1 1 auto; resize: none; diff --git a/crates/acp-tunnel/src/lib.rs b/crates/acp-tunnel/src/lib.rs index e2da35f..e4d670e 100644 --- a/crates/acp-tunnel/src/lib.rs +++ b/crates/acp-tunnel/src/lib.rs @@ -137,6 +137,18 @@ pub enum Phase { SessionActive, } +/// One inline image the operator pasted into the chat input (issue #158). Carries +/// the already-base64-encoded payload as-is — this layer never decodes or +/// re-encodes it, only wraps it in an ACP `ContentBlock` (`type: "image"`). +/// `Deserialize` lets the Tauri `agent_prompt` command take it straight off the +/// frontend's `invoke` payload. +#[derive(Debug, Clone, PartialEq, serde::Deserialize)] +pub struct PromptImage { + pub data: String, + #[serde(rename = "mimeType")] + pub mime_type: String, +} + /// The client-side session state machine. It **produces** the outbound frames /// (`initialize`, `session/new`, `session/resume`) and tracks the phase + the /// declaration set; the transport owns the socket and feeds results back in. @@ -238,20 +250,28 @@ impl Session { Some((id, frame)) } - /// Build a `session/prompt` request carrying one text block — the operator's - /// chat turn (ADR *agent-chat-panel*, Part A). `None` until a session is - /// active. Returns `(id, frame)`; the transport correlates the eventual - /// `{ stopReason }` result by this `id` (prompt results are method-less, the - /// same shape as handshake acks, so id-correlation is mandatory). - pub fn prompt(&mut self, text: &str) -> Option<(u64, Value)> { + /// Build a `session/prompt` request carrying the operator's chat turn (ADR + /// *agent-chat-panel*, Part A) — a text block plus zero or more pasted image + /// blocks (issue #158). `None` until a session is active. Returns + /// `(id, frame)`; the transport correlates the eventual `{ stopReason }` + /// result by this `id` (prompt results are method-less, the same shape as + /// handshake acks, so id-correlation is mandatory). + pub fn prompt(&mut self, text: &str, images: &[PromptImage]) -> Option<(u64, Value)> { let session_id = self.session_id.clone()?; let id = self.alloc_id(); + let mut blocks: Vec = Vec::new(); + if !text.is_empty() { + blocks.push(json!({ "type": "text", "text": text })); + } + for img in images { + blocks.push(json!({ "type": "image", "data": img.data, "mimeType": img.mime_type })); + } let frame = request( id, "session/prompt", json!({ "sessionId": session_id, - "prompt": [{ "type": "text", "text": text }], + "prompt": blocks, }), ); Some((id, frame)) @@ -332,12 +352,18 @@ pub enum Inbound { /// `mcp/cancel` (notification) — abandon the in-flight request whose **outer** /// frame id is `request_id`. No reply is owed. Cancel { request_id: Value }, - /// `session/update` with `sessionUpdate == "agent_message_chunk"` — a piece of - /// the agent's chat reply (ADR *agent-chat-panel*). openab emits a single - /// terminal chunk today, but the transport accumulates so incremental chunks - /// need no rework. Other `session/update` kinds (thoughts, tool calls) are not - /// emitted by openab yet and stay [`Inbound::Other`]. + /// `session/update` with `sessionUpdate == "agent_message_chunk"` and content + /// `type == "text"` — a piece of the agent's chat reply (ADR + /// *agent-chat-panel*). openab emits a single terminal chunk today, but the + /// transport accumulates so incremental chunks need no rework. Other + /// `session/update` kinds (thoughts, tool calls) are not emitted by openab yet + /// and stay [`Inbound::Other`]. AgentChunk { text: String }, + /// `session/update` with `sessionUpdate == "agent_message_chunk"` and content + /// `type == "image"` — an inline image in the agent's chat reply (issue + /// #158). Unlike [`Inbound::AgentChunk`] this is not accumulated — each image + /// is forwarded to the panel as its own `agent-update`. + AgentImage { data: String, mime_type: String }, /// Not a tunnel frame (unknown method, or a `session/update` kind we don't /// surface) — ignored. Other, @@ -387,16 +413,32 @@ pub fn parse_inbound(frame: &Value) -> Inbound { }, "session/update" => { let update = params.get("update").cloned().unwrap_or(Value::Null); - match update.get("sessionUpdate").and_then(Value::as_str) { - Some("agent_message_chunk") => Inbound::AgentChunk { - text: update - .get("content") - .and_then(|c| c.get("text")) + if update.get("sessionUpdate").and_then(Value::as_str) != Some("agent_message_chunk") { + // thoughts / tool_call etc. — openab does not emit these yet. + return Inbound::Other; + } + let content = update.get("content").cloned().unwrap_or(Value::Null); + match content.get("type").and_then(Value::as_str) { + Some("text") => Inbound::AgentChunk { + text: content + .get("text") .and_then(Value::as_str) .unwrap_or_default() .to_string(), }, - // thoughts / tool_call etc. — openab does not emit these yet. + Some("image") => Inbound::AgentImage { + data: content + .get("data") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string(), + mime_type: content + .get("mimeType") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string(), + }, + // audio / resource / resource_link — not surfaced in chat yet. _ => Inbound::Other, } } @@ -673,12 +715,12 @@ mod tests { fn prompt_needs_a_session_then_builds_a_text_turn() { let mut fresh = Session::new(vec![oab_server("c")]); assert!( - fresh.prompt("hi").is_none(), + fresh.prompt("hi", &[]).is_none(), "no prompt before a session exists" ); let mut s = active_session(); - let (id, frame) = s.prompt("hello there").expect("session active"); + let (id, frame) = s.prompt("hello there", &[]).expect("session active"); assert_eq!(frame["jsonrpc"], "2.0"); assert_eq!(frame["id"], id); assert_eq!(frame["method"], "session/prompt"); @@ -689,6 +731,36 @@ mod tests { ); } + #[test] + fn prompt_appends_image_blocks_after_the_text_block() { + let mut s = active_session(); + let images = vec![PromptImage { + data: "YWJj".into(), + mime_type: "image/png".into(), + }]; + let (_, frame) = s.prompt("look at this", &images).expect("session active"); + let blocks = frame["params"]["prompt"].as_array().unwrap(); + assert_eq!(blocks.len(), 2); + assert_eq!(blocks[0], json!({ "type": "text", "text": "look at this" })); + assert_eq!( + blocks[1], + json!({ "type": "image", "data": "YWJj", "mimeType": "image/png" }) + ); + } + + #[test] + fn prompt_with_only_images_omits_the_text_block() { + let mut s = active_session(); + let images = vec![PromptImage { + data: "YWJj".into(), + mime_type: "image/png".into(), + }]; + let (_, frame) = s.prompt("", &images).expect("session active"); + let blocks = frame["params"]["prompt"].as_array().unwrap(); + assert_eq!(blocks.len(), 1); + assert_eq!(blocks[0]["type"], "image"); + } + #[test] fn cancel_is_an_idless_notification_for_the_active_session() { assert!(Session::new(vec![oab_server("c")]).cancel().is_none()); @@ -741,4 +813,31 @@ mod tests { "sessionUpdate": "agent_thought_chunk", "content": { "text": "hmm" } } } }); assert_eq!(parse_inbound(&thought), Inbound::Other); } + + #[test] + fn agent_message_chunk_with_image_content_is_classified() { + let chunk = json!({ + "jsonrpc": "2.0", "method": "session/update", + "params": { "sessionId": "s", "update": { + "sessionUpdate": "agent_message_chunk", + "content": { "type": "image", "data": "YWJj", "mimeType": "image/png" } + }} + }); + assert_eq!( + parse_inbound(&chunk), + Inbound::AgentImage { + data: "YWJj".into(), + mime_type: "image/png".into(), + } + ); + } + + #[test] + fn agent_message_chunk_with_unsupported_content_type_stays_other() { + // audio / resource / resource_link content blocks — not surfaced in chat yet. + let chunk = json!({ "method": "session/update", "params": { "update": { + "sessionUpdate": "agent_message_chunk", + "content": { "type": "audio", "data": "YWJj", "mimeType": "audio/wav" } } } }); + assert_eq!(parse_inbound(&chunk), Inbound::Other); + } } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index fe995dc..0f47aed 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -750,9 +750,10 @@ async fn agent_prompt( remote: tauri::State<'_, remote::Remote>, agent: Option, text: String, + images: Option>, ) -> Result<(), String> { let name = remote::resolve_name(agent.as_deref())?; - remote.send_prompt(&name, text).await + remote.send_prompt(&name, text, images.unwrap_or_default()).await } /// Abandon an agent's in-flight chat turn (`session/cancel`). Best-effort. diff --git a/src-tauri/src/remote.rs b/src-tauri/src/remote.rs index bb81c56..8bb64f4 100644 --- a/src-tauri/src/remote.rs +++ b/src-tauri/src/remote.rs @@ -18,7 +18,7 @@ use std::time::{Duration, Instant}; use acp_tunnel as acp; use acp_tunnel::config::{AgentEndpoint, AgentRegistry, RemoteConfig}; -use acp_tunnel::{DisconnectReason, Inbound, Session}; +use acp_tunnel::{DisconnectReason, Inbound, PromptImage, Session}; use futures_util::{Sink, SinkExt, StreamExt}; use serde_json::{json, Value}; use tauri::{AppHandle, Emitter, Manager, Runtime}; @@ -70,8 +70,9 @@ pub struct RemoteState { /// *agent-chat-panel*, Part B). Pushed by the `agent_prompt` / `agent_cancel` /// commands into the per-connection channel that `run_once` drains onto the WS. pub enum OutMsg { - /// Send a chat turn (`session/prompt`). - Prompt(String), + /// Send a chat turn (`session/prompt`): the typed text plus any pasted + /// images (issue #158). + Prompt(String, Vec), /// Abandon the in-flight turn (`session/cancel`). Cancel, /// User is disconnecting: abandon any in-flight turn and close the socket @@ -83,8 +84,13 @@ pub enum OutMsg { impl Remote { /// Send a chat turn to the named agent. Errors if that agent's session is not /// live (each agent console has its own connection, so the target is explicit). - pub async fn send_prompt(&self, agent: &str, text: String) -> Result<(), String> { - self.push(agent, OutMsg::Prompt(text)).await + pub async fn send_prompt( + &self, + agent: &str, + text: String, + images: Vec, + ) -> Result<(), String> { + self.push(agent, OutMsg::Prompt(text, images)).await } /// Cancel the named agent's in-flight turn (best-effort). @@ -604,7 +610,7 @@ async fn run_once( // whole run, so it stays open until teardown. Some(out) = out_rx.recv() => { match out { - OutMsg::Prompt(text) => { + OutMsg::Prompt(text, images) => { // In-flight guard (review #1): single-shot turn model. A // second prompt while one is pending would overwrite // `pending_prompt` and orphan the first turn's `turn_end` @@ -616,7 +622,7 @@ async fn run_once( "app-log", json!({ "level": "warn", "msg": "remote: prompt ignored — a turn is already in flight" }), ); - } else if let Some((id, frame)) = session.prompt(&text) { + } else if let Some((id, frame)) = session.prompt(&text, &images) { pending_prompt = Some(id); if let Err(e) = send(&mut write, &frame).await { outcome = Err(e); @@ -987,6 +993,14 @@ async fn run_once( Inbound::AgentChunk { text } => { let _ = app.emit("agent-update", json!({ "agent": agent, "kind": "chunk", "text": text })); } + // An inline image in the agent's chat reply (issue #158) — forwarded + // as its own update, not accumulated like a text chunk. + Inbound::AgentImage { data, mime_type } => { + let _ = app.emit( + "agent-update", + json!({ "agent": agent, "kind": "image", "data": data, "mimeType": mime_type }), + ); + } Inbound::Cancel { .. } | Inbound::Other => {} } }