(null);
const liveFrame = live.status === "live" ? (live.frame ?? null) : null;
- if (liveFrame !== null && liveFrame !== lastFrame) setLastFrame(liveFrame);
- const frame = liveFrame ?? lastFrame;
- const src = frame ? `data:${frame.mimeType};base64,${frame.dataBase64}` : null;
+ const painted = useComputerPreviewFrame(liveFrame);
+ const frame =
+ live.status === "unavailable" || (live.status === "live" && !liveFrame) ? null : painted.frame;
const sourceSize =
frame && frame.width > 0 && frame.height > 0
? { width: frame.width, height: frame.height }
@@ -286,12 +282,12 @@ function ComputerMiniPlayer({
true,
);
}, [viewportKey, sourceSize.width, sourceSize.height]);
- const label = frame?.appName ?? preview?.appName ?? gt("Computer");
+ const label = frame?.appName ?? gt("Computer");
const caption =
frame?.windowTitle && frame.windowTitle !== label ? `${label} · ${frame.windowTitle}` : label;
const appIcon = useAssetUrlState(
threadRef.environmentId,
- preview?.app ? { _tag: "native-app-icon", app: preview.app } : null,
+ frame?.app ? { _tag: "native-app-icon", app: frame.app } : null,
);
return (
@@ -307,21 +303,19 @@ function ComputerMiniPlayer({
className="pointer-events-auto absolute inset-0 overflow-hidden rounded-[inherit] bg-muted"
style={{ zIndex: PREVIEW_MINI_PLAYER_WEBVIEW_Z_INDEX }}
>
- {src ? (
-
- ) : (
+
+ {!frame ? (
{live.status === "unavailable"
? gt("Preview unavailable")
: gt("Waiting for the agent…")}
- )}
+ ) : null}
{live.status === "unavailable" && live.detail ? (
{live.detail}
diff --git a/apps/web/src/components/preview/useComputerPreviewFrame.browser.test.tsx b/apps/web/src/components/preview/useComputerPreviewFrame.browser.test.tsx
new file mode 100644
index 0000000000000..910612c070931
--- /dev/null
+++ b/apps/web/src/components/preview/useComputerPreviewFrame.browser.test.tsx
@@ -0,0 +1,84 @@
+import type { CuaWindowPreviewFrame } from "@t3tools/contracts";
+import { expect, it, vi } from "vite-plus/test";
+
+import { act, render } from "../../../test/render";
+import { useComputerPreviewFrame } from "./useComputerPreviewFrame";
+
+const images: DeferredImage[] = [];
+class DeferredImage {
+ src = "";
+ naturalWidth = 480;
+ naturalHeight = 800;
+ resolve!: () => void;
+ ready = new Promise((resolve) => {
+ this.resolve = resolve;
+ });
+ constructor() {
+ images.push(this);
+ }
+ decode() {
+ return this.ready;
+ }
+}
+
+const makeFrame = (data: string): CuaWindowPreviewFrame => ({
+ appName: data,
+ mimeType: "image/png",
+ dataBase64: data,
+ width: 480,
+ height: 800,
+ capturedAt: "2026-10-16T00:00:00Z",
+});
+
+function Harness({ frame }: { frame: CuaWindowPreviewFrame | null }) {
+ const { canvasRef, frame: painted } = useComputerPreviewFrame(frame);
+ return (
+ <>
+
+
+ >
+ );
+}
+
+// Images decode only when a test resolves them, and canvas drawing is recorded.
+function stubFrameDecoding() {
+ vi.stubGlobal("Image", DeferredImage);
+ images.length = 0;
+ const drawImage = vi
+ .spyOn(CanvasRenderingContext2D.prototype, "drawImage")
+ .mockImplementation(() => undefined);
+ return {
+ drawImage,
+ [Symbol.dispose]() {
+ drawImage.mockRestore();
+ vi.unstubAllGlobals();
+ },
+ };
+}
+
+it("keeps the previous pixels and caption until the replacement finishes decoding", async () => {
+ using stubs = stubFrameDecoding();
+ const { drawImage } = stubs;
+ await using screen = await render(, { router: false });
+ const caption = () => screen.container.querySelector("output")?.textContent;
+ await act(async () => images[0]!.resolve());
+ expect(caption()).toBe("first");
+
+ await screen.rerender();
+ expect(drawImage).toHaveBeenCalledTimes(1);
+ expect(caption()).toBe("first");
+ await act(async () => images[1]!.resolve());
+ expect(drawImage).toHaveBeenLastCalledWith(images[1], 0, 0);
+ expect(caption()).toBe("next");
+});
+
+it("ignores a decode that finishes after a newer frame", async () => {
+ using stubs = stubFrameDecoding();
+ const { drawImage } = stubs;
+ await using screen = await render(, { router: false });
+ await screen.rerender();
+ await act(async () => images[1]!.resolve());
+ await act(async () => images[0]!.resolve());
+ expect(drawImage).toHaveBeenCalledTimes(1);
+ expect(screen.container.querySelector("output")?.textContent).toBe("new");
+});
diff --git a/apps/web/src/components/preview/useComputerPreviewFrame.ts b/apps/web/src/components/preview/useComputerPreviewFrame.ts
new file mode 100644
index 0000000000000..94685886be6ce
--- /dev/null
+++ b/apps/web/src/components/preview/useComputerPreviewFrame.ts
@@ -0,0 +1,37 @@
+import type { CuaWindowPreviewFrame } from "@t3tools/contracts";
+import { useEffect, useRef, useState } from "react";
+
+export function useComputerPreviewFrame(nextFrame: CuaWindowPreviewFrame | null) {
+ const canvasRef = useRef(null);
+ const [frame, setFrame] = useState(null);
+
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- Decodes each streamed frame off-screen and paints it into the canvas only once it can replace the shown one.
+ useEffect(() => {
+ if (!nextFrame) return;
+ let cancelled = false;
+ const image = new Image();
+ image.src = `data:${nextFrame.mimeType};base64,${nextFrame.dataBase64}`;
+ void image
+ .decode()
+ .then(() => {
+ if (cancelled) return;
+ const canvas = canvasRef.current;
+ const context = canvas?.getContext("2d");
+ if (!canvas || !context) return;
+ // Changing an img src can blank its previous frame while decoding.
+ // Keep the canvas pixels until the replacement can be drawn in one task.
+ if (canvas.width !== image.naturalWidth || canvas.height !== image.naturalHeight) {
+ canvas.width = image.naturalWidth;
+ canvas.height = image.naturalHeight;
+ }
+ context.drawImage(image, 0, 0);
+ setFrame(nextFrame);
+ })
+ .catch(() => undefined);
+ return () => {
+ cancelled = true;
+ };
+ }, [nextFrame]);
+
+ return { canvasRef, frame };
+}
diff --git a/apps/web/src/composer-editor-mentions.test.ts b/apps/web/src/composer-editor-mentions.test.ts
index f1305b83fbb11..459f47ff1b6f6 100644
--- a/apps/web/src/composer-editor-mentions.test.ts
+++ b/apps/web/src/composer-editor-mentions.test.ts
@@ -1,5 +1,6 @@
import { EnvironmentId, MessageId, ThreadId, type AssistantCitation } from "@t3tools/contracts";
import { serializeAssistantCitation } from "@t3tools/shared/assistantCitations";
+import { formatNativeAppMention } from "@t3tools/shared/nativeAppMentions";
import { describe, expect, it } from "vite-plus/test";
import {
@@ -8,6 +9,18 @@ import {
} from "./composer-editor-mentions";
import { formatTerminalContextReference } from "./lib/terminalContext";
+it("keeps a native application reference as one composer token", () => {
+ const source = formatNativeAppMention(
+ { name: "Helium", launchPath: "C:\\Apps\\Helium\\chrome.exe" },
+ "windows",
+ );
+ expect(splitPromptIntoComposerSegments(`use ${source} please`)).toEqual([
+ { type: "text", text: "use " },
+ { type: "application", href: source.slice(source.indexOf("](") + 2, -1), source },
+ { type: "text", text: " please" },
+ ]);
+});
+
const terminalReference = formatTerminalContextReference({
id: "ctx-1",
terminalLabel: "Terminal 1",
diff --git a/apps/web/src/composer-editor-mentions.ts b/apps/web/src/composer-editor-mentions.ts
index 7f3e93b821488..cee7c4e04bd98 100644
--- a/apps/web/src/composer-editor-mentions.ts
+++ b/apps/web/src/composer-editor-mentions.ts
@@ -1,5 +1,6 @@
import { collectComposerModelMentions } from "@t3tools/client-runtime/composer-model-mentions";
import type { AssistantCitation } from "@t3tools/contracts";
+import { collectNativeAppMentions } from "@t3tools/shared/nativeAppMentions";
import { collectAssistantCitations } from "@t3tools/shared/assistantCitations";
import { collectComposerContextReferences } from "@t3tools/shared/composerContextReferences";
import {
@@ -9,6 +10,7 @@ import {
} from "@t3tools/shared/composerInlineTokens";
export type ComposerPromptSegment =
+ | { type: "application"; href: string; source: string }
| {
type: "text";
text: string;
@@ -106,10 +108,12 @@ export function collectComposerPromptInlineTokens(text: string) {
: [...inlineTokens, ...models].sort((left, right) => left.start - right.start);
const citations = collectAssistantCitations(text);
const references = collectComposerContextReferences(text);
- if (citations.length === 0 && references.length === 0) return tokens;
+ const applications = collectNativeAppMentions(text);
+ if (citations.length === 0 && references.length === 0 && applications.length === 0) return tokens;
// An unfinished @ mention can otherwise consume the start of a link label.
const links = [
+ ...applications.map((match) => ({ ...match, type: "application" as const })),
...citations.map((match) => ({ ...match, type: "citation" as const })),
...references.map((match) => ({ ...match, type: "context-reference" as const })),
];
@@ -138,7 +142,9 @@ function splitPromptTextIntoComposerSegments(text: string): ComposerPromptSegmen
pushTextSegment(segments, text.slice(cursor, match.start));
}
- if (match.type === "citation") {
+ if (match.type === "application") {
+ segments.push({ type: "application", href: match.href, source: match.source });
+ } else if (match.type === "citation") {
segments.push({ type: "citation", citation: match.citation, source: match.source });
} else if (match.type === "context-reference") {
segments.push({
diff --git a/apps/web/src/composer-logic.test.ts b/apps/web/src/composer-logic.test.ts
index 8a714a4c572db..a8ef982f55b49 100644
--- a/apps/web/src/composer-logic.test.ts
+++ b/apps/web/src/composer-logic.test.ts
@@ -12,6 +12,7 @@ import {
mergeWithDefaultKeybindings,
} from "@t3tools/shared/keybindings";
import { describe, expect, it } from "vite-plus/test";
+import { formatNativeAppMention } from "@t3tools/shared/nativeAppMentions";
import {
clampCollapsedComposerCursor,
@@ -29,6 +30,12 @@ import { slashCommandItemsForPromptPosition } from "./components/chat/composerSl
import { carryDisplacedCustomAnswerIntoPrompt } from "./pendingUserInput";
import { formatTerminalContextReference } from "./lib/terminalContext";
+it("treats an application chip as one cursor position", () => {
+ const source = formatNativeAppMention({ name: "Helium" }, "host");
+ expect(collapseExpandedComposerCursor(`${source} x`, source.length)).toBe(1);
+ expect(expandCollapsedComposerCursor(`${source} x`, 1)).toBe(source.length);
+});
+
const terminalReference = formatTerminalContextReference({
id: "ctx-1",
terminalLabel: "Terminal 1",
diff --git a/apps/web/src/composer-logic.ts b/apps/web/src/composer-logic.ts
index 0a6de3de73249..abad6e0787476 100644
--- a/apps/web/src/composer-logic.ts
+++ b/apps/web/src/composer-logic.ts
@@ -138,6 +138,7 @@ export function expandCollapsedComposerCursor(
for (const segment of segments) {
if (
segment.type === "mention" ||
+ segment.type === "application" ||
segment.type === "citation" ||
segment.type === "context-reference"
) {
@@ -221,6 +222,7 @@ export function collapseExpandedComposerCursor(
for (const segment of segments) {
if (
segment.type === "mention" ||
+ segment.type === "application" ||
segment.type === "citation" ||
segment.type === "context-reference"
) {
diff --git a/apps/web/src/composer-rich-text-doc.ts b/apps/web/src/composer-rich-text-doc.ts
index 3c2b353f59b76..51984e2a4d00d 100644
--- a/apps/web/src/composer-rich-text-doc.ts
+++ b/apps/web/src/composer-rich-text-doc.ts
@@ -806,6 +806,9 @@ function atomJsonForSegment(
segment: Exclude[number], { type: "text" }>,
chips: ChipResolver,
): AtomJson | null {
+ if (segment.type === "application") {
+ return { type: "composer-application", attrs: { href: segment.href, source: segment.source } };
+ }
if (segment.type === "mention") {
return {
type: "composer-mention",
@@ -1207,6 +1210,7 @@ function readAtomSource(node: ProseMirrorNode): string {
switch (node.type.name) {
case "composer-mention":
case "composer-model":
+ case "composer-application":
case "composer-citation":
case "composer-context-reference":
return typeof attrs.source === "string" ? attrs.source : "";
diff --git a/apps/web/src/i18n/translations/de.json b/apps/web/src/i18n/translations/de.json
index 857d6ab51bf12..e8a9f995d0a20 100644
--- a/apps/web/src/i18n/translations/de.json
+++ b/apps/web/src/i18n/translations/de.json
@@ -2128,6 +2128,7 @@
" verwaltet."
],
"d71a78d5a0b8045d": "Hilfe erhalten",
+ "cc10829e6a2d7661": "App: {name}",
"b9e49ec03efc8383": "Kein aktiver Thread",
"fc7b51a6f2a86236": "Wähle einen Thread, um weiterzumachen",
"7543807a82fa9ccd": "Wähle einen vorhandenen Thread oder erstelle einen neuen, um loszulegen.",
@@ -3180,7 +3181,7 @@
"3d3f30f1a1c4fc52": "Frag einfach...",
"305d6d72f1220d98": "Pull Request wird gesucht...",
"cf855794f7b8ddb5": "Skills im Arbeitsbereich werden durchsucht...",
- "e159e35b83a86404": "Dateien im Arbeitsbereich werden durchsucht...",
+ "5b31a853afbdd6bc": "Dateien, Threads und Apps werden durchsucht...",
"c347652bd889e70f": "Nachricht an {title}",
"9bf7fa475fe78de5": "Nachricht",
"86ba20bb0ac0e5d0": "Sende Feedback, um diese Anfrage abzulehnen",
@@ -3295,7 +3296,7 @@
"a4f8a7db663bbc79": "Codesprache: {language}",
"249192edde7f4932": "Sprachen durchsuchen",
"71d8b593668d2106": "Keine passenden Sprachen.",
- "326bc17df66a7f05": "Dateien und Ordner",
+ "1fcbf1a95bcb4002": "Dateien, Threads und Apps",
"0e83c3bf28f8958e": "Befehle",
"75d18dac06f4f037": "Skills",
"05340d7dfcc34edf": "App",
@@ -3311,7 +3312,7 @@
"6640341b04ff6825": "Anbieter",
"e9d791bbaaa61fac": "Anbieter-Skill",
"061c0c9f27104d23": "Keine Skills gefunden. Mit / kannst du Anbieterbefehle durchsuchen.",
- "0a6fbecab03f7ef3": "Keine passenden Dateien oder Ordner.",
+ "4fcabb6aca677012": "Keine passenden Dateien, Threads oder Apps.",
"0e7b21ab11721ca7": "Kein passender Befehl.",
"2029db2cc30beb6d": "Anbietereinstellungen öffnen",
"517cfb8f24123f5b": "Kein Anbieter verfügbar",
@@ -4580,6 +4581,7 @@
"49c80af1dd4826f1": "Der Download aus der Quelle ist fehlgeschlagen.",
"5ba22a660758e453": "Das Limit für Anhänge ist erreicht.",
"181933312b23cb0c": "Der Entwurf hat diesen Anhang abgelehnt (Duplikat oder Limit für Anhänge erreicht).",
+ "c8c58bab29faa2e9": "App",
"df02c1d51d158d32": "Antwortmodell für diesen Thread wechseln",
"929781439a221aae": "Eine Nebenfrage in einem neuen Nebenchat stellen",
"378c3404c430fc05": "Wie /side",
diff --git a/apps/web/src/i18n/translations/en.json b/apps/web/src/i18n/translations/en.json
index fe65f35ff14d3..d33e1662441d7 100644
--- a/apps/web/src/i18n/translations/en.json
+++ b/apps/web/src/i18n/translations/en.json
@@ -2128,6 +2128,7 @@
"."
],
"d71a78d5a0b8045d": "Get help",
+ "cc10829e6a2d7661": "Application: {name}",
"b9e49ec03efc8383": "No active thread",
"fc7b51a6f2a86236": "Pick a thread to continue",
"7543807a82fa9ccd": "Select an existing thread or create a new one to get started.",
@@ -3179,7 +3180,7 @@
"3d3f30f1a1c4fc52": "Ask anything...",
"305d6d72f1220d98": "Finding pull request...",
"cf855794f7b8ddb5": "Searching workspace skills...",
- "e159e35b83a86404": "Searching workspace files...",
+ "5b31a853afbdd6bc": "Searching files, threads, and applications...",
"c347652bd889e70f": "Message {title}",
"9bf7fa475fe78de5": "Message",
"86ba20bb0ac0e5d0": "Send feedback to decline this request",
@@ -3294,7 +3295,7 @@
"a4f8a7db663bbc79": "Code language: {language}",
"249192edde7f4932": "Search languages",
"71d8b593668d2106": "No matching languages.",
- "326bc17df66a7f05": "Files and folders",
+ "1fcbf1a95bcb4002": "Files, threads, and applications",
"0e83c3bf28f8958e": "Commands",
"75d18dac06f4f037": "Skills",
"05340d7dfcc34edf": "App",
@@ -3310,7 +3311,7 @@
"6640341b04ff6825": "Provider",
"e9d791bbaaa61fac": "Provider Skill",
"061c0c9f27104d23": "No skills found. Try / to browse provider commands.",
- "0a6fbecab03f7ef3": "No matching files or folders.",
+ "4fcabb6aca677012": "No matching files, threads, or applications.",
"0e7b21ab11721ca7": "No matching command.",
"2029db2cc30beb6d": "Open provider settings",
"517cfb8f24123f5b": "No provider available",
@@ -4579,6 +4580,7 @@
"49c80af1dd4826f1": "Downloading it from the source failed.",
"5ba22a660758e453": "The attachment limit has been reached.",
"181933312b23cb0c": "The draft rejected this attachment (duplicate or attachment limit reached).",
+ "c8c58bab29faa2e9": "Application",
"df02c1d51d158d32": "Switch response model for this thread",
"929781439a221aae": "Ask a side question in a new side chat",
"378c3404c430fc05": "Same as /side",
diff --git a/apps/web/src/i18n/translations/es.json b/apps/web/src/i18n/translations/es.json
index 9ea91d27d1815..203feb167a4fe 100644
--- a/apps/web/src/i18n/translations/es.json
+++ b/apps/web/src/i18n/translations/es.json
@@ -2128,6 +2128,7 @@
"."
],
"d71a78d5a0b8045d": "Obtener ayuda",
+ "cc10829e6a2d7661": "App: {name}",
"b9e49ec03efc8383": "No hay ningún hilo activo",
"fc7b51a6f2a86236": "Elige un hilo para continuar",
"7543807a82fa9ccd": "Selecciona un hilo existente o crea uno nuevo para empezar.",
@@ -3179,7 +3180,7 @@
"3d3f30f1a1c4fc52": "Pregunta lo que quieras...",
"305d6d72f1220d98": "Buscando la pull request...",
"cf855794f7b8ddb5": "Buscando skills del espacio de trabajo...",
- "e159e35b83a86404": "Buscando archivos del espacio de trabajo...",
+ "5b31a853afbdd6bc": "Buscando archivos, hilos y apps...",
"c347652bd889e70f": "Mensaje para {title}",
"9bf7fa475fe78de5": "Mensaje",
"86ba20bb0ac0e5d0": "Envía comentarios para rechazar esta solicitud",
@@ -3294,7 +3295,7 @@
"a4f8a7db663bbc79": "Lenguaje del código: {language}",
"249192edde7f4932": "Buscar lenguajes",
"71d8b593668d2106": "No hay lenguajes que coincidan.",
- "326bc17df66a7f05": "Archivos y carpetas",
+ "1fcbf1a95bcb4002": "Archivos, hilos y apps",
"0e83c3bf28f8958e": "Comandos",
"75d18dac06f4f037": "Skills",
"05340d7dfcc34edf": "App",
@@ -3310,7 +3311,7 @@
"6640341b04ff6825": "Proveedor",
"e9d791bbaaa61fac": "Skill del proveedor",
"061c0c9f27104d23": "No se encontraron skills. Prueba con / para explorar los comandos del proveedor.",
- "0a6fbecab03f7ef3": "No hay archivos ni carpetas que coincidan.",
+ "4fcabb6aca677012": "No hay archivos, hilos ni apps que coincidan.",
"0e7b21ab11721ca7": "Ningún comando coincide.",
"2029db2cc30beb6d": "Abrir ajustes del proveedor",
"517cfb8f24123f5b": "No hay ningún proveedor disponible",
@@ -4577,6 +4578,7 @@
"49c80af1dd4826f1": "Falló la descarga desde el origen.",
"5ba22a660758e453": "Se alcanzó el límite de adjuntos.",
"181933312b23cb0c": "El borrador rechazó este adjunto (duplicado o límite de adjuntos alcanzado).",
+ "c8c58bab29faa2e9": "App",
"df02c1d51d158d32": "Cambiar el modelo de respuesta de este hilo",
"929781439a221aae": "Hacer una pregunta aparte en un chat lateral nuevo",
"378c3404c430fc05": "Igual que /side",
diff --git a/apps/web/src/i18n/translations/fr.json b/apps/web/src/i18n/translations/fr.json
index 39a4fe0f3ea39..85e21be0fa062 100644
--- a/apps/web/src/i18n/translations/fr.json
+++ b/apps/web/src/i18n/translations/fr.json
@@ -2128,6 +2128,7 @@
"."
],
"d71a78d5a0b8045d": "Obtenir de l’aide",
+ "cc10829e6a2d7661": "App : {name}",
"b9e49ec03efc8383": "Aucun fil actif",
"fc7b51a6f2a86236": "Choisissez un fil pour continuer",
"7543807a82fa9ccd": "Sélectionnez un fil existant ou créez-en un pour commencer.",
@@ -3179,7 +3180,7 @@
"3d3f30f1a1c4fc52": "Demandez ce que vous voulez...",
"305d6d72f1220d98": "Recherche de la pull request...",
"cf855794f7b8ddb5": "Recherche des skills de l’espace de travail...",
- "e159e35b83a86404": "Recherche des fichiers de l’espace de travail...",
+ "5b31a853afbdd6bc": "Recherche des fichiers, fils et apps...",
"c347652bd889e70f": "Message à {title}",
"9bf7fa475fe78de5": "Message",
"86ba20bb0ac0e5d0": "Envoyez un commentaire pour refuser cette demande",
@@ -3294,7 +3295,7 @@
"a4f8a7db663bbc79": "Langage du code : {language}",
"249192edde7f4932": "Rechercher un langage",
"71d8b593668d2106": "Aucun langage correspondant.",
- "326bc17df66a7f05": "Fichiers et dossiers",
+ "1fcbf1a95bcb4002": "Fichiers, fils et apps",
"0e83c3bf28f8958e": "Commandes",
"75d18dac06f4f037": "Skills",
"05340d7dfcc34edf": "App",
@@ -3310,7 +3311,7 @@
"6640341b04ff6825": "Fournisseur",
"e9d791bbaaa61fac": "Skill du fournisseur",
"061c0c9f27104d23": "Aucun skill trouvé. Essayez / pour parcourir les commandes du fournisseur.",
- "0a6fbecab03f7ef3": "Aucun fichier ni dossier correspondant.",
+ "4fcabb6aca677012": "Aucun fichier, fil ni app correspondant.",
"0e7b21ab11721ca7": "Aucune commande correspondante.",
"2029db2cc30beb6d": "Ouvrir les paramètres des fournisseurs",
"517cfb8f24123f5b": "Aucun fournisseur disponible",
@@ -4578,6 +4579,7 @@
"49c80af1dd4826f1": "Le téléchargement depuis la source a échoué.",
"5ba22a660758e453": "La limite de pièces jointes est atteinte.",
"181933312b23cb0c": "Le brouillon a refusé cette pièce jointe (doublon ou limite de pièces jointes atteinte).",
+ "c8c58bab29faa2e9": "App",
"df02c1d51d158d32": "Changer le modèle de réponse de ce fil",
"929781439a221aae": "Poser une question annexe dans une nouvelle discussion annexe",
"378c3404c430fc05": "Identique à /side",
diff --git a/apps/web/src/i18n/translations/ja.json b/apps/web/src/i18n/translations/ja.json
index c971e6b0d107a..a9ca6fdf99103 100644
--- a/apps/web/src/i18n/translations/ja.json
+++ b/apps/web/src/i18n/translations/ja.json
@@ -2126,6 +2126,7 @@
" によって管理されています。"
],
"d71a78d5a0b8045d": "ヘルプを表示",
+ "cc10829e6a2d7661": "アプリ: {name}",
"b9e49ec03efc8383": "アクティブなスレッドがありません",
"fc7b51a6f2a86236": "続けるスレッドを選択",
"7543807a82fa9ccd": "既存のスレッドを選択するか、新しいスレッドを作成して始めましょう。",
@@ -3177,7 +3178,7 @@
"3d3f30f1a1c4fc52": "何でも聞いてください...",
"305d6d72f1220d98": "プルリクエストを検索しています...",
"cf855794f7b8ddb5": "ワークスペースのスキルを検索しています...",
- "e159e35b83a86404": "ワークスペースのファイルを検索しています...",
+ "5b31a853afbdd6bc": "ファイル、スレッド、アプリを検索しています...",
"c347652bd889e70f": "{title} にメッセージ",
"9bf7fa475fe78de5": "メッセージ",
"86ba20bb0ac0e5d0": "このリクエストを拒否するにはフィードバックを送信してください",
@@ -3290,7 +3291,7 @@
"a4f8a7db663bbc79": "コードの言語: {language}",
"249192edde7f4932": "言語を検索",
"71d8b593668d2106": "一致する言語はありません。",
- "326bc17df66a7f05": "ファイルとフォルダー",
+ "1fcbf1a95bcb4002": "ファイル、スレッド、アプリ",
"0e83c3bf28f8958e": "コマンド",
"75d18dac06f4f037": "スキル",
"05340d7dfcc34edf": "アプリ",
@@ -3306,7 +3307,7 @@
"6640341b04ff6825": "プロバイダー",
"e9d791bbaaa61fac": "プロバイダーのスキル",
"061c0c9f27104d23": "スキルが見つかりません。/ を入力してプロバイダーのコマンドを参照してください。",
- "0a6fbecab03f7ef3": "一致するファイルやフォルダーはありません。",
+ "4fcabb6aca677012": "一致するファイル、スレッド、アプリはありません。",
"0e7b21ab11721ca7": "一致するコマンドはありません。",
"2029db2cc30beb6d": "プロバイダー設定を開く",
"517cfb8f24123f5b": "利用できるプロバイダーがありません",
@@ -4567,6 +4568,7 @@
"49c80af1dd4826f1": "元の場所からのダウンロードに失敗しました。",
"5ba22a660758e453": "添付ファイルの上限に達しました。",
"181933312b23cb0c": "下書きがこの添付ファイルを拒否しました (重複しているか、添付ファイルの上限に達しています)。",
+ "c8c58bab29faa2e9": "アプリ",
"df02c1d51d158d32": "このスレッドの応答モデルを切り替え",
"929781439a221aae": "新しいサイドチャットで別の質問をする",
"378c3404c430fc05": "/side と同じ",
diff --git a/apps/web/src/i18n/translations/ko.json b/apps/web/src/i18n/translations/ko.json
index 462622eb1e2e9..a03ebb8f38747 100644
--- a/apps/web/src/i18n/translations/ko.json
+++ b/apps/web/src/i18n/translations/ko.json
@@ -2127,6 +2127,7 @@
"에서 관리합니다."
],
"d71a78d5a0b8045d": "도움말 보기",
+ "cc10829e6a2d7661": "앱: {name}",
"b9e49ec03efc8383": "활성 스레드 없음",
"fc7b51a6f2a86236": "계속할 스레드를 선택하세요",
"7543807a82fa9ccd": "기존 스레드를 선택하거나 새 스레드를 만들어 시작하세요.",
@@ -3178,7 +3179,7 @@
"3d3f30f1a1c4fc52": "무엇이든 물어보세요...",
"305d6d72f1220d98": "풀 리퀘스트를 찾는 중...",
"cf855794f7b8ddb5": "작업 영역 스킬을 검색하는 중...",
- "e159e35b83a86404": "작업 영역 파일을 검색하는 중...",
+ "5b31a853afbdd6bc": "파일, 스레드, 앱을 검색하는 중...",
"c347652bd889e70f": "{title}에 메시지 보내기",
"9bf7fa475fe78de5": "메시지",
"86ba20bb0ac0e5d0": "이 요청을 거부하려면 피드백을 보내세요",
@@ -3290,7 +3291,7 @@
"a4f8a7db663bbc79": "코드 언어: {language}",
"249192edde7f4932": "언어 검색",
"71d8b593668d2106": "일치하는 언어가 없습니다.",
- "326bc17df66a7f05": "파일 및 폴더",
+ "1fcbf1a95bcb4002": "파일, 스레드 및 앱",
"0e83c3bf28f8958e": "명령",
"75d18dac06f4f037": "스킬",
"05340d7dfcc34edf": "앱",
@@ -3306,7 +3307,7 @@
"6640341b04ff6825": "제공자",
"e9d791bbaaa61fac": "제공자 스킬",
"061c0c9f27104d23": "스킬을 찾을 수 없습니다. 제공자 명령을 둘러보려면 /를 입력하세요.",
- "0a6fbecab03f7ef3": "일치하는 파일이나 폴더가 없습니다.",
+ "4fcabb6aca677012": "일치하는 파일, 스레드 또는 앱이 없습니다.",
"0e7b21ab11721ca7": "일치하는 명령이 없습니다.",
"2029db2cc30beb6d": "제공자 설정 열기",
"517cfb8f24123f5b": "사용 가능한 제공자 없음",
@@ -4564,6 +4565,7 @@
"49c80af1dd4826f1": "원본에서 다운로드하지 못했습니다.",
"5ba22a660758e453": "첨부 파일 한도에 도달했습니다.",
"181933312b23cb0c": "초안에서 이 첨부 파일을 거부했습니다(중복 또는 첨부 파일 한도 도달).",
+ "c8c58bab29faa2e9": "앱",
"df02c1d51d158d32": "이 스레드의 응답 모델 전환",
"929781439a221aae": "새 사이드 채팅에서 곁가지 질문하기",
"378c3404c430fc05": "/side와 같음",
diff --git a/apps/web/src/i18n/translations/pt-BR.json b/apps/web/src/i18n/translations/pt-BR.json
index 2aee29dc38f9d..ed49f68fe4199 100644
--- a/apps/web/src/i18n/translations/pt-BR.json
+++ b/apps/web/src/i18n/translations/pt-BR.json
@@ -2128,6 +2128,7 @@
"."
],
"d71a78d5a0b8045d": "Obter ajuda",
+ "cc10829e6a2d7661": "App: {name}",
"b9e49ec03efc8383": "Nenhuma thread ativa",
"fc7b51a6f2a86236": "Escolha uma thread para continuar",
"7543807a82fa9ccd": "Selecione uma thread existente ou crie uma nova para começar.",
@@ -3179,7 +3180,7 @@
"3d3f30f1a1c4fc52": "Pergunte qualquer coisa...",
"305d6d72f1220d98": "Procurando o pull request...",
"cf855794f7b8ddb5": "Pesquisando skills do workspace...",
- "e159e35b83a86404": "Pesquisando arquivos do workspace...",
+ "5b31a853afbdd6bc": "Pesquisando arquivos, threads e apps...",
"c347652bd889e70f": "Mensagem para {title}",
"9bf7fa475fe78de5": "Mensagem",
"86ba20bb0ac0e5d0": "Envie um feedback para recusar esta solicitação",
@@ -3294,7 +3295,7 @@
"a4f8a7db663bbc79": "Linguagem do código: {language}",
"249192edde7f4932": "Pesquisar linguagens",
"71d8b593668d2106": "Nenhuma linguagem correspondente.",
- "326bc17df66a7f05": "Arquivos e pastas",
+ "1fcbf1a95bcb4002": "Arquivos, threads e apps",
"0e83c3bf28f8958e": "Comandos",
"75d18dac06f4f037": "Skills",
"05340d7dfcc34edf": "App",
@@ -3310,7 +3311,7 @@
"6640341b04ff6825": "Provedor",
"e9d791bbaaa61fac": "Skill do provedor",
"061c0c9f27104d23": "Nenhuma skill encontrada. Use / para ver os comandos do provedor.",
- "0a6fbecab03f7ef3": "Nenhum arquivo ou pasta correspondente.",
+ "4fcabb6aca677012": "Nenhum arquivo, thread ou app correspondente.",
"0e7b21ab11721ca7": "Nenhum comando correspondente.",
"2029db2cc30beb6d": "Abrir configurações do provedor",
"517cfb8f24123f5b": "Nenhum provedor disponível",
@@ -4577,6 +4578,7 @@
"49c80af1dd4826f1": "Falha ao baixá-lo da origem.",
"5ba22a660758e453": "O limite de anexos foi atingido.",
"181933312b23cb0c": "O rascunho rejeitou este anexo (duplicado ou limite de anexos atingido).",
+ "c8c58bab29faa2e9": "App",
"df02c1d51d158d32": "Trocar o modelo de resposta desta thread",
"929781439a221aae": "Fazer uma pergunta paralela em um novo chat paralelo",
"378c3404c430fc05": "Igual a /side",
diff --git a/apps/web/src/i18n/translations/ru.json b/apps/web/src/i18n/translations/ru.json
index b7bcae501d913..f37ad87b97d2f 100644
--- a/apps/web/src/i18n/translations/ru.json
+++ b/apps/web/src/i18n/translations/ru.json
@@ -2128,6 +2128,7 @@
"."
],
"d71a78d5a0b8045d": "Получить помощь",
+ "cc10829e6a2d7661": "Приложение: {name}",
"b9e49ec03efc8383": "Нет активного треда",
"fc7b51a6f2a86236": "Выберите тред, чтобы продолжить",
"7543807a82fa9ccd": "Выберите существующий тред или создайте новый, чтобы начать.",
@@ -3179,7 +3180,7 @@
"3d3f30f1a1c4fc52": "Спросите что угодно...",
"305d6d72f1220d98": "Поиск пул-реквеста...",
"cf855794f7b8ddb5": "Поиск навыков рабочей области...",
- "e159e35b83a86404": "Поиск файлов рабочей области...",
+ "5b31a853afbdd6bc": "Поиск файлов, тредов и приложений...",
"c347652bd889e70f": "Сообщение для {title}",
"9bf7fa475fe78de5": "Сообщение",
"86ba20bb0ac0e5d0": "Отправьте отзыв, чтобы отклонить этот запрос",
@@ -3294,7 +3295,7 @@
"a4f8a7db663bbc79": "Язык кода: {language}",
"249192edde7f4932": "Поиск языков",
"71d8b593668d2106": "Нет подходящих языков.",
- "326bc17df66a7f05": "Файлы и папки",
+ "1fcbf1a95bcb4002": "Файлы, треды и приложения",
"0e83c3bf28f8958e": "Команды",
"75d18dac06f4f037": "Навыки",
"05340d7dfcc34edf": "Приложение",
@@ -3310,7 +3311,7 @@
"6640341b04ff6825": "Провайдер",
"e9d791bbaaa61fac": "Навык провайдера",
"061c0c9f27104d23": "Навыки не найдены. Введите /, чтобы просмотреть команды провайдера.",
- "0a6fbecab03f7ef3": "Нет подходящих файлов или папок.",
+ "4fcabb6aca677012": "Нет подходящих файлов, тредов или приложений.",
"0e7b21ab11721ca7": "Нет подходящей команды.",
"2029db2cc30beb6d": "Открыть настройки провайдера",
"517cfb8f24123f5b": "Нет доступного провайдера",
@@ -4577,6 +4578,7 @@
"49c80af1dd4826f1": "Не удалось скачать его из источника.",
"5ba22a660758e453": "Достигнут лимит вложений.",
"181933312b23cb0c": "Черновик отклонил это вложение (дубликат или достигнут лимит вложений).",
+ "c8c58bab29faa2e9": "Приложение",
"df02c1d51d158d32": "Сменить модель ответа для этого треда",
"929781439a221aae": "Задать побочный вопрос в новом боковом чате",
"378c3404c430fc05": "То же, что /side",
diff --git a/apps/web/src/i18n/translations/sv.json b/apps/web/src/i18n/translations/sv.json
index 867c1f3d6481d..0ec033a447a03 100644
--- a/apps/web/src/i18n/translations/sv.json
+++ b/apps/web/src/i18n/translations/sv.json
@@ -2128,6 +2128,7 @@
"."
],
"d71a78d5a0b8045d": "Få hjälp",
+ "cc10829e6a2d7661": "App: {name}",
"b9e49ec03efc8383": "Ingen aktiv tråd",
"fc7b51a6f2a86236": "Välj en tråd att fortsätta i",
"7543807a82fa9ccd": "Välj en befintlig tråd eller skapa en ny för att komma igång.",
@@ -3179,7 +3180,7 @@
"3d3f30f1a1c4fc52": "Fråga vad som helst...",
"305d6d72f1220d98": "Hittar pull request...",
"cf855794f7b8ddb5": "Söker färdigheter i arbetsytan...",
- "e159e35b83a86404": "Söker filer i arbetsytan...",
+ "5b31a853afbdd6bc": "Söker filer, trådar och appar...",
"c347652bd889e70f": "Meddelande till {title}",
"9bf7fa475fe78de5": "Meddelande",
"86ba20bb0ac0e5d0": "Skicka feedback för att avböja den här begäran",
@@ -3294,7 +3295,7 @@
"a4f8a7db663bbc79": "Kodspråk: {language}",
"249192edde7f4932": "Sök språk",
"71d8b593668d2106": "Inga matchande språk.",
- "326bc17df66a7f05": "Filer och mappar",
+ "1fcbf1a95bcb4002": "Filer, trådar och appar",
"0e83c3bf28f8958e": "Kommandon",
"75d18dac06f4f037": "Färdigheter",
"05340d7dfcc34edf": "App",
@@ -3310,7 +3311,7 @@
"6640341b04ff6825": "Leverantör",
"e9d791bbaaa61fac": "Leverantörsfärdighet",
"061c0c9f27104d23": "Inga färdigheter hittades. Prova / för att bläddra bland leverantörens kommandon.",
- "0a6fbecab03f7ef3": "Inga matchande filer eller mappar.",
+ "4fcabb6aca677012": "Inga matchande filer, trådar eller appar.",
"0e7b21ab11721ca7": "Inget matchande kommando.",
"2029db2cc30beb6d": "Öppna leverantörsinställningar",
"517cfb8f24123f5b": "Ingen leverantör tillgänglig",
@@ -4578,6 +4579,7 @@
"49c80af1dd4826f1": "Det gick inte att ladda ned den från källan.",
"5ba22a660758e453": "Gränsen för bilagor har nåtts.",
"181933312b23cb0c": "Utkastet avvisade den här bilagan (dubblett eller gränsen för bilagor nådd).",
+ "c8c58bab29faa2e9": "App",
"df02c1d51d158d32": "Byt svarsmodell för den här tråden",
"929781439a221aae": "Ställ en sidofråga i en ny sidochatt",
"378c3404c430fc05": "Samma som /side",
diff --git a/apps/web/src/i18n/translations/zh.json b/apps/web/src/i18n/translations/zh.json
index 9bf50aa1f0fc8..9889d0b97720b 100644
--- a/apps/web/src/i18n/translations/zh.json
+++ b/apps/web/src/i18n/translations/zh.json
@@ -2132,6 +2132,7 @@
" 管理。"
],
"d71a78d5a0b8045d": "获取帮助",
+ "cc10829e6a2d7661": "应用:{name}",
"b9e49ec03efc8383": "没有打开的会话",
"fc7b51a6f2a86236": "选择一个会话以继续",
"7543807a82fa9ccd": "选择已有会话,或新建一个会话开始。",
@@ -3183,7 +3184,7 @@
"3d3f30f1a1c4fc52": "随便问点什么…",
"305d6d72f1220d98": "正在查找 pull request…",
"cf855794f7b8ddb5": "正在搜索工作区技能…",
- "e159e35b83a86404": "正在搜索工作区文件…",
+ "5b31a853afbdd6bc": "正在搜索文件、会话和应用…",
"c347652bd889e70f": "给 {title} 发消息",
"9bf7fa475fe78de5": "消息",
"86ba20bb0ac0e5d0": "发送反馈以拒绝这个请求",
@@ -3299,7 +3300,7 @@
"a4f8a7db663bbc79": "代码语言:{language}",
"249192edde7f4932": "搜索语言",
"71d8b593668d2106": "没有匹配的语言。",
- "326bc17df66a7f05": "文件和文件夹",
+ "1fcbf1a95bcb4002": "文件、会话和应用",
"0e83c3bf28f8958e": "命令",
"75d18dac06f4f037": "技能",
"05340d7dfcc34edf": "应用",
@@ -3315,7 +3316,7 @@
"6640341b04ff6825": "提供方",
"e9d791bbaaa61fac": "提供方技能",
"061c0c9f27104d23": "未找到技能。试试输入 / 浏览提供方命令。",
- "0a6fbecab03f7ef3": "没有匹配的文件或文件夹。",
+ "4fcabb6aca677012": "没有匹配的文件、会话或应用。",
"0e7b21ab11721ca7": "没有匹配的命令。",
"2029db2cc30beb6d": "打开提供方设置",
"517cfb8f24123f5b": "没有可用的提供方",
@@ -4584,6 +4585,7 @@
"49c80af1dd4826f1": "从来源下载失败。",
"5ba22a660758e453": "已达到附件数量上限。",
"181933312b23cb0c": "草稿拒绝了这个附件(重复或已达到附件上限)。",
+ "c8c58bab29faa2e9": "应用",
"df02c1d51d158d32": "切换这个会话使用的回复模型",
"929781439a221aae": "在新的侧聊中问一个旁支问题",
"378c3404c430fc05": "与 /side 相同",
diff --git a/apps/web/src/state/cua.ts b/apps/web/src/state/cua.ts
index 7ebe5dbd6b519..0c4f073432166 100644
--- a/apps/web/src/state/cua.ts
+++ b/apps/web/src/state/cua.ts
@@ -1,5 +1,5 @@
import { createCuaEnvironmentAtoms } from "@t3tools/client-runtime/state/cua";
-import type { CuaWindowPreviewState, ScopedThreadRef } from "@t3tools/contracts";
+import type { CuaWindowPreviewState, ScopedThreadRef, EnvironmentId } from "@t3tools/contracts";
import { connectionAtomRuntime } from "../connection/runtime";
import { useEnvironmentQuery } from "./query";
@@ -8,6 +8,12 @@ const cuaEnvironment = createCuaEnvironmentAtoms(connectionAtomRuntime);
const IDLE: CuaWindowPreviewState = { status: "idle" };
+export function useCuaApplications(environmentId: EnvironmentId | null) {
+ return useEnvironmentQuery(
+ environmentId === null ? null : cuaEnvironment.applications({ environmentId, input: {} }),
+ );
+}
+
/** Subscribes while mounted; the server captures only for mounted subscribers. */
export function useCuaWindowPreview(threadRef: ScopedThreadRef | null): CuaWindowPreviewState {
const query = useEnvironmentQuery(
diff --git a/docs/user/computer-use.md b/docs/user/computer-use.md
index 44e3cc87a67a7..6c19fd1fb0de9 100644
--- a/docs/user/computer-use.md
+++ b/docs/user/computer-use.md
@@ -28,7 +28,7 @@ What the host needs depends on its OS:
at the computer may need to approve a screen capture prompt.
The desktop app includes Cua Driver. A server started with `npx t3` downloads
-the same version into T3 Code's home the first time an agent session needs it.
+the same version into T3 Code's home the first time computer use needs it.
To use your own build instead, set `T3CODE_CUA_DRIVER_PATH` to its executable
before starting the server.
@@ -37,6 +37,11 @@ keeps yours and T3 Code does not add a second one.
## Watch the agent work
+In the web or desktop composer, type `@` followed by an app name, such as
+`@Helium`, and select it to tell the agent which app to use. With Computer use
+enabled, suggestions include apps on the selected environment's computer,
+including when you connect remotely. Selecting an app does not launch it.
+
Each action an agent takes shows in the thread with the app's icon and a plain
title, such as **Clicked in Safari**.
diff --git a/packages/client-runtime/src/state/cua.ts b/packages/client-runtime/src/state/cua.ts
index 8ac30275f63ac..52b19d78671cc 100644
--- a/packages/client-runtime/src/state/cua.ts
+++ b/packages/client-runtime/src/state/cua.ts
@@ -2,12 +2,21 @@ import { WS_METHODS } from "@t3tools/contracts";
import { Atom } from "effect/reactivity";
import type { EnvironmentRegistry } from "../connection/registry.ts";
-import { createEnvironmentRpcSubscriptionAtomFamily } from "./runtime.ts";
+import {
+ createEnvironmentRpcSubscriptionAtomFamily,
+ createEnvironmentRpcQueryAtomFamily,
+} from "./runtime.ts";
export function createCuaEnvironmentAtoms(
runtime: Atom.AtomRuntime,
) {
return {
+ applications: createEnvironmentRpcQueryAtomFamily(runtime, {
+ label: "environment-data:cua:applications",
+ tag: WS_METHODS.serverListApplications,
+ staleTimeMs: 30_000,
+ idleTtlMs: 30_000,
+ }),
/** Live frames of the window the agent is driving; the server captures only while subscribed. */
windowPreview: createEnvironmentRpcSubscriptionAtomFamily(runtime, {
label: "environment-data:cua:window-preview",
diff --git a/packages/contracts/src/cua.ts b/packages/contracts/src/cua.ts
index 9651e897f3c2d..ee73f5e9c7c6f 100644
--- a/packages/contracts/src/cua.ts
+++ b/packages/contracts/src/cua.ts
@@ -1,6 +1,24 @@
import * as Schema from "effect/Schema";
import { ThreadId, TrimmedNonEmptyString } from "./baseSchemas.ts";
+import { ToolActivityNativeAppReference } from "./providerRuntime.ts";
+
+export const CuaApplication = Schema.Struct({
+ name: TrimmedNonEmptyString.check(Schema.isMaxLength(160)),
+ bundleId: Schema.optionalKey(TrimmedNonEmptyString.check(Schema.isMaxLength(512))),
+ launchPath: Schema.optionalKey(TrimmedNonEmptyString.check(Schema.isMaxLength(2048))),
+ pid: Schema.optionalKey(Schema.Int.check(Schema.isGreaterThan(0))),
+});
+export type CuaApplication = typeof CuaApplication.Type;
+
+export class CuaApplicationsError extends Schema.TaggedError()(
+ "CuaApplicationsError",
+ { cause: Schema.Defect() },
+) {
+ override get message() {
+ return "Could not list applications on this computer.";
+ }
+}
/** Native host descriptor, confined to the desktop/server control pipe. */
export const CuaDriverMcpConfiguration = Schema.Struct({
@@ -14,6 +32,7 @@ export type CuaDriverMcpConfiguration = typeof CuaDriverMcpConfiguration.Type;
/** A window the agent is driving, refreshed for the user while a client watches. */
export const CuaWindowPreviewFrame = Schema.Struct({
+ app: Schema.optionalKey(ToolActivityNativeAppReference),
appName: Schema.optionalKey(Schema.String),
windowTitle: Schema.optionalKey(Schema.String),
width: Schema.Number,
diff --git a/packages/contracts/src/providerRuntime.ts b/packages/contracts/src/providerRuntime.ts
index 4769bfd7955c7..b933a68b41155 100644
--- a/packages/contracts/src/providerRuntime.ts
+++ b/packages/contracts/src/providerRuntime.ts
@@ -92,7 +92,7 @@ export const ToolActivityNativeAppReference = Schema.Union([
Schema.TaggedStruct("app-id", {
appId: TrimmedNonEmptyStringSchema.check(
Schema.isMaxLength(512),
- Schema.isPattern(/^[A-Za-z0-9._-]+$/u),
+ Schema.isPattern(/^[A-Za-z0-9._-]+(?:![A-Za-z0-9._-]+)?$/u),
),
}),
Schema.TaggedStruct("display-name", {
diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts
index 61298138b5bf1..4e20cfbb520c5 100644
--- a/packages/contracts/src/rpc.ts
+++ b/packages/contracts/src/rpc.ts
@@ -342,6 +342,8 @@ import {
BrowserTabsError,
BrowserTabsStatus,
CuaHostStatus,
+ CuaApplication,
+ CuaApplicationsError,
CuaWindowPreviewState,
CuaWindowPreviewSubscribeInput,
} from "./cua.ts";
@@ -541,6 +543,7 @@ export const WS_METHODS = {
serverGetConfig: "server.getConfig",
serverGetNpmRelease: "server.getNpmRelease",
serverCheckComputerUse: "server.checkComputerUse",
+ serverListApplications: "server.listApplications",
serverCheckBrowserTabs: "server.checkBrowserTabs",
serverRunBrowserTabsAction: "server.runBrowserTabsAction",
serverRefreshProviders: "server.refreshProviders",
@@ -706,6 +709,12 @@ const WsServerCheckComputerUseRpc = Rpc.make(WS_METHODS.serverCheckComputerUse,
error: EnvironmentAuthorizationError,
});
+const WsServerListApplicationsRpc = Rpc.make(WS_METHODS.serverListApplications, {
+ payload: Schema.Struct({}),
+ success: Schema.Array(CuaApplication),
+ error: Schema.Union([CuaApplicationsError, EnvironmentAuthorizationError]),
+});
+
const WsServerCheckBrowserTabsRpc = Rpc.make(WS_METHODS.serverCheckBrowserTabs, {
payload: Schema.Struct({}),
success: BrowserTabsStatus,
@@ -2300,6 +2309,7 @@ export const WsRpcGroup = RpcGroup.make(
WsServerProbeRpc,
WsServerGetConfigRpc,
WsServerCheckComputerUseRpc,
+ WsServerListApplicationsRpc,
WsServerCheckBrowserTabsRpc,
WsServerRunBrowserTabsActionRpc,
WsServerRefreshProvidersRpc,
diff --git a/packages/provider-core/src/server/cuaToolPresentation.test.ts b/packages/provider-core/src/server/cuaToolPresentation.test.ts
index f91110bb01ca7..b080cd2528728 100644
--- a/packages/provider-core/src/server/cuaToolPresentation.test.ts
+++ b/packages/provider-core/src/server/cuaToolPresentation.test.ts
@@ -37,6 +37,100 @@ describe("parseCuaToolName", () => {
});
describe("cuaToolPresentation", () => {
+ it("retains a discovered app's display name and icon identity in window listings", () => {
+ clearCuaToolContext(threadId);
+ rememberCuaToolResult(
+ threadId,
+ "list_apps",
+ {},
+ { apps: [{ pid: 12, name: "Helium", bundle_id: "net.imput.helium" }] },
+ );
+ rememberCuaToolResult(
+ threadId,
+ "list_windows",
+ {},
+ { windows: [{ pid: 12, window_id: 34, app_name: "chrome.exe", title: "Releases - Helium" }] },
+ );
+ const presentation = cuaToolPresentation({
+ threadId,
+ rawToolName: "cua-driver/get_window_state",
+ args: { pid: 12, window_id: 34 },
+ status: "completed",
+ });
+ NodeAssert.equal(presentation?.title, "Looked at Helium");
+ NodeAssert.deepEqual(presentation?.toolIcon, {
+ _tag: "native-app",
+ app: { _tag: "app-id", appId: "net.imput.helium" },
+ });
+ });
+
+ it("keeps identities for Windows apps sharing a host PID separate", () => {
+ clearCuaToolContext(threadId);
+ rememberCuaToolResult(
+ threadId,
+ "launch_app",
+ { name: "Calculator" },
+ {
+ pid: 11804,
+ name: "Microsoft.WindowsCalculator_8wekyb3d8bbwe!App",
+ bundle_id: "Microsoft.WindowsCalculator_8wekyb3d8bbwe!App",
+ windows: [{ window_id: 10, title: "Calculator" }],
+ },
+ );
+ rememberCuaToolResult(
+ threadId,
+ "list_windows",
+ {},
+ {
+ windows: [
+ { pid: 11804, window_id: 10, app_name: "ApplicationFrameHost.exe", title: "Calculator" },
+ { pid: 11804, window_id: 20, app_name: "ApplicationFrameHost.exe", title: "Settings" },
+ ],
+ },
+ );
+ for (const [windowId, expected] of [
+ [10, "Calculator"],
+ [20, "Settings"],
+ [10, "Calculator"],
+ ] as const) {
+ rememberCuaToolResult(
+ threadId,
+ "get_window_state",
+ { pid: 11804, window_id: windowId },
+ {
+ pid: 11804,
+ window_id: windowId,
+ app_name: "ApplicationFrameHost.exe",
+ window_title: expected,
+ },
+ );
+ const viewed = cuaToolPresentation({
+ threadId,
+ rawToolName: "cua-driver/get_window_state",
+ args:
+ windowId === 20
+ ? { target: { kind: "window", pid: 11804, window_id: windowId } }
+ : { pid: 11804, window_id: windowId },
+ status: "completed",
+ });
+ NodeAssert.equal(viewed?.title, `Looked at ${expected}`);
+ const clicked = cuaToolPresentation({
+ threadId,
+ rawToolName: "cua-driver/click",
+ args: { pid: 11804, element_token: "snapshot:1" },
+ status: "completed",
+ });
+ NodeAssert.equal(clicked?.title, `Clicked in ${expected}`);
+ NodeAssert.deepEqual(clicked?.toolIcon, {
+ _tag: "native-app",
+ app:
+ expected === "Calculator"
+ ? { _tag: "app-id", appId: "Microsoft.WindowsCalculator_8wekyb3d8bbwe!App" }
+ : { _tag: "display-name", displayName: "Settings" },
+ });
+ }
+ });
+
it("resolves apps by pid from earlier discovery results", () => {
clearCuaToolContext(threadId);
rememberCuaToolResult(
@@ -165,6 +259,15 @@ describe("cuaToolPresentation", () => {
})?.title,
"Typed text in Safari",
);
+ NodeAssert.equal(
+ cuaToolPresentation({
+ threadId,
+ rawToolName: "mcp__cua-driver__launch_app",
+ args: { name: "/Applications/Visual Studio Code.app" },
+ status: "inProgress",
+ })?.title,
+ "Launching Visual Studio Code",
+ );
});
it("returns nothing for tools outside Cua", () => {
@@ -252,6 +355,48 @@ const CUA_DRIVER_TOOLS = [
];
describe("readCuaWindowTarget", () => {
+ it("keeps the capture target when restoring focus or a later tool fails", () => {
+ clearCuaToolContext(threadId);
+ cuaToolPresentation({
+ threadId,
+ rawToolName: "cua-driver/get_window_state",
+ args: { pid: 42, window_id: 7 },
+ status: "completed",
+ });
+ for (const status of ["inProgress", "completed"] as const) {
+ cuaToolPresentation({
+ threadId,
+ rawToolName: "cua-driver/bring_to_front",
+ args: { pid: 99, window_id: 8 },
+ status,
+ });
+ NodeAssert.deepEqual(readCuaWindowTarget(threadId), { pid: 42, windowId: 7n });
+ }
+ cuaToolPresentation({
+ threadId,
+ rawToolName: "cua-driver/get_window_state",
+ args: { pid: 99, window_id: 8 },
+ status: "failed",
+ });
+ NodeAssert.deepEqual(readCuaWindowTarget(threadId), { pid: 42, windowId: 7n });
+ clearCuaToolContext(threadId);
+ });
+
+ it("keeps display-only inspections from redirecting the preview", () => {
+ clearCuaToolContext(threadId);
+ for (const args of [
+ { pid: 42, window_id: 7 },
+ { pid: 99, window_id: 8, display_only: true },
+ ]) {
+ cuaToolPresentation({
+ threadId,
+ rawToolName: "cua-driver/get_window_state",
+ args,
+ status: "completed",
+ });
+ }
+ NodeAssert.deepEqual(readCuaWindowTarget(threadId), { pid: 42, windowId: 7n });
+ });
it("remembers the last pid and window id a tool call addressed", () => {
clearCuaToolContext(threadId);
NodeAssert.equal(readCuaWindowTarget(threadId), undefined);
diff --git a/packages/provider-core/src/server/cuaToolPresentation.ts b/packages/provider-core/src/server/cuaToolPresentation.ts
index 22530ef0f47d1..7576b50902a51 100644
--- a/packages/provider-core/src/server/cuaToolPresentation.ts
+++ b/packages/provider-core/src/server/cuaToolPresentation.ts
@@ -10,8 +10,8 @@ import { CUA_MCP_SERVER_NAME } from "./mcpSession.ts";
/**
* Shared presentation for Cua Driver tool calls, so every provider renders
* them like Codex computer use: a "computer" surface, the target app's icon,
- * and a human title. Cua addresses apps by pid, so the directory learns
- * pid-to-app mappings from discovery results and resolves later calls.
+ * and a human title. The directory learns app and window identities from
+ * discovery results; Windows can host several apps inside one process.
*/
export interface CuaToolPresentation {
readonly title: string;
@@ -51,10 +51,12 @@ const MAX_APPS_PER_THREAD = 128;
export interface CuaWindowTarget {
readonly pid: number;
readonly windowId: bigint;
+ readonly app?: CuaApp;
}
interface ThreadDirectory {
readonly byPid: Map;
+ readonly byWindow: Map;
lastApp: CuaApp | undefined;
lastWindow: CuaWindowTarget | undefined;
}
@@ -74,7 +76,12 @@ function directoryFor(threadId: ThreadId): ThreadDirectory {
if (oldest === undefined) break;
directories.delete(oldest);
}
- const created: ThreadDirectory = { byPid: new Map(), lastApp: undefined, lastWindow: undefined };
+ const created: ThreadDirectory = {
+ byPid: new Map(),
+ byWindow: new Map(),
+ lastApp: undefined,
+ lastWindow: undefined,
+ };
directories.set(threadId, created);
return created;
}
@@ -83,9 +90,19 @@ export function clearCuaToolContext(threadId: ThreadId): void {
directories.delete(threadId);
}
+/** End the turn's capture target without forgetting learned app identities. */
+export function clearCuaWindowTarget(threadId: ThreadId): void {
+ const directory = directories.get(threadId);
+ if (directory) directory.lastWindow = undefined;
+}
+
/** Last window the agent targeted in this thread, if any tool call named one. */
export function readCuaWindowTarget(threadId: ThreadId): CuaWindowTarget | undefined {
- return directories.get(threadId)?.lastWindow;
+ const directory = directories.get(threadId);
+ const target = directory?.lastWindow;
+ if (!target) return undefined;
+ const app = directory.byWindow.get(`${target.pid}:${target.windowId}`);
+ return { ...target, ...(app ? { app } : {}) };
}
function asWindowId(value: unknown): bigint | undefined {
@@ -118,7 +135,7 @@ function asText(value: unknown, maxLength: number): string | undefined {
const asAppName = (value: unknown) => asText(value, 160);
const asBundleId = (value: unknown) => {
const id = asText(value, 512);
- return id && /^[A-Za-z0-9._-]+$/u.test(id) ? id : undefined;
+ return id && /^[A-Za-z0-9._-]+(?:![A-Za-z0-9._-]+)?$/u.test(id) ? id : undefined;
};
/** Accepts tool result text, MCP content arrays, or already-parsed objects. */
@@ -161,13 +178,39 @@ function rememberApp(directory: ThreadDirectory, pid: number | undefined, app: C
}
function learnFromRecord(directory: ThreadDirectory, record: Record): void {
- const name = asAppName(record.app_name) ?? asAppName(record.name);
+ const pid = asPid(record.pid);
+ const reportedName = asAppName(record.app_name) ?? asAppName(record.name);
+ const sharedHost = /^(?:ApplicationFrameHost|WWAHost)(?:\.exe)?$/iu.test(reportedName ?? "");
+ const knownApp = !sharedHost && pid !== undefined ? directory.byPid.get(pid) : undefined;
+ const name = sharedHost
+ ? (asAppName(record.window_title) ?? asAppName(record.title))
+ : /\.exe$/iu.test(reportedName ?? "") && knownApp?.name && !/\.exe$/iu.test(knownApp.name)
+ ? knownApp.name
+ : reportedName;
const bundleId = asBundleId(record.bundle_id);
+ if (sharedHost && pid !== undefined) {
+ rememberApp(directory, pid, {});
+ directory.byPid.set(pid, {});
+ }
if (!name && !bundleId) return;
- rememberApp(directory, asPid(record.pid), {
+ const app = {
+ ...knownApp,
...(name ? { name } : {}),
...(bundleId ? { bundleId } : {}),
- });
+ };
+ const windowId = asWindowId(record.window_id);
+ if (pid !== undefined && windowId !== undefined) {
+ const key = `${pid}:${windowId}`;
+ const known = directory.byWindow.get(key);
+ directory.byWindow.delete(key);
+ directory.byWindow.set(key, { ...known, ...app });
+ if (directory.byWindow.size > MAX_APPS_PER_THREAD) {
+ const oldest = directory.byWindow.keys().next().value;
+ if (oldest !== undefined) directory.byWindow.delete(oldest);
+ }
+ }
+ // A Windows host PID can own Calculator and Settings at the same time.
+ if (!sharedHost) rememberApp(directory, pid, app);
}
/**
@@ -200,6 +243,30 @@ export function rememberCuaToolResult(
if (requestedBundleId && pid !== undefined && toolName === "launch_app") {
rememberApp(directory, pid, { bundleId: requestedBundleId });
}
+ if (toolName === "launch_app") {
+ const name = appNameFromLaunchTarget(asRecord(args)?.name);
+ if (name) {
+ learnFromRecord(directory, { ...parsed, name });
+ if (Array.isArray(parsed.windows)) {
+ for (const window of parsed.windows.slice(0, MAX_APPS_PER_THREAD)) {
+ const record = asRecord(window);
+ if (record) learnFromRecord(directory, { ...parsed, ...record, name });
+ }
+ }
+ }
+ }
+}
+
+// `launch_app` also accepts a `.app` bundle or executable path as its name.
+function appNameFromLaunchTarget(value: unknown): string | undefined {
+ const name = asAppName(value);
+ if (!name || !/[\\/]/u.test(name)) return name;
+ return asAppName(
+ name
+ .split(/[\\/]/u)
+ .findLast((part) => part.length > 0)
+ ?.replace(/\.(?:app|exe)$/iu, ""),
+ );
}
function humanizeBundleId(bundleId: string): string | undefined {
@@ -210,14 +277,26 @@ function humanizeBundleId(bundleId: string): string | undefined {
function resolveApp(
threadId: ThreadId,
args: Record | undefined,
+ trackWindow: boolean,
): CuaApp | undefined {
const directory = directories.get(threadId);
- const pid = asPid(args?.pid);
+ const target = asRecord(args?.target);
+ const pid = asPid(args?.pid) ?? asPid(target?.pid);
const bundleId = asBundleId(args?.bundle_id);
- const name = asAppName(args?.app_name) ?? asAppName(args?.app);
- const known = pid !== undefined ? directory?.byPid.get(pid) : undefined;
- const windowId = asWindowId(args?.window_id);
- if (pid !== undefined && windowId !== undefined) {
+ const name =
+ asAppName(args?.app_name) ?? asAppName(args?.app) ?? appNameFromLaunchTarget(args?.name);
+ const windowId = asWindowId(args?.window_id) ?? asWindowId(target?.window_id);
+ const targetedWindow =
+ windowId ??
+ (pid !== undefined && directory?.lastWindow?.pid === pid
+ ? directory.lastWindow.windowId
+ : undefined);
+ const windowApp =
+ pid !== undefined && targetedWindow !== undefined
+ ? directory?.byWindow.get(`${pid}:${targetedWindow}`)
+ : undefined;
+ const known = windowApp ?? (pid !== undefined ? directory?.byPid.get(pid) : undefined);
+ if (trackWindow && pid !== undefined && windowId !== undefined && args?.display_only !== true) {
directoryFor(threadId).lastWindow = { pid, windowId };
}
const app: CuaApp = {
@@ -227,7 +306,7 @@ function resolveApp(
};
if (app.name || app.bundleId) {
if (directory) {
- if (pid !== undefined) rememberApp(directory, pid, app);
+ if (pid !== undefined && !windowApp) rememberApp(directory, pid, app);
directory.lastApp = app;
}
return app;
@@ -465,7 +544,14 @@ export function cuaToolPresentation(input: {
if (!tool) return undefined;
const args = asRecord(input.args);
const targetArgs = tool === "run_steps" ? runStepsTarget(args) : args;
- const app = TOOLS_WITHOUT_APP.has(tool) ? undefined : resolveApp(input.threadId, targetArgs);
+ // Restoring the user's foreground window must not redirect the live preview.
+ const app = TOOLS_WITHOUT_APP.has(tool)
+ ? undefined
+ : resolveApp(
+ input.threadId,
+ targetArgs,
+ tool !== "bring_to_front" && input.status !== "failed",
+ );
const appName = app?.name ?? (app?.bundleId ? humanizeBundleId(app.bundleId) : undefined);
const reference = app ? nativeApp(app) : undefined;
const icon = reference ? ({ _tag: "native-app", app: reference } as const) : undefined;
diff --git a/packages/shared/package.json b/packages/shared/package.json
index 38d187377bf83..d6cd3c9916459 100644
--- a/packages/shared/package.json
+++ b/packages/shared/package.json
@@ -15,6 +15,10 @@
"types": "./src/changedFileKind.ts",
"import": "./src/changedFileKind.ts"
},
+ "./nativeAppMentions": {
+ "types": "./src/nativeAppMentions.ts",
+ "import": "./src/nativeAppMentions.ts"
+ },
"./authScopeOptions": {
"types": "./src/authScopeOptions.ts",
"import": "./src/authScopeOptions.ts"
@@ -631,6 +635,10 @@
"types": "./src/paletteSearchCopy.ts",
"import": "./src/paletteSearchCopy.ts"
},
+ "./cuaDriverHostOptions": {
+ "types": "./src/cuaDriverHostOptions.ts",
+ "import": "./src/cuaDriverHostOptions.ts"
+ },
"./cuaDriverRelease": {
"types": "./src/cuaDriverRelease.ts",
"import": "./src/cuaDriverRelease.ts"
diff --git a/packages/shared/src/composerContextReferences.ts b/packages/shared/src/composerContextReferences.ts
index a6c2fc914b666..fef51c8e027e8 100644
--- a/packages/shared/src/composerContextReferences.ts
+++ b/packages/shared/src/composerContextReferences.ts
@@ -9,6 +9,7 @@ import {
type ScopedThreadRef,
type ThreadContextRecord,
} from "@t3tools/contracts";
+import { projectNativeAppMentions } from "./nativeAppMentions.ts";
import { expandAssistantCitationsForProvider } from "./assistantCitations.ts";
import { replaceThreadLinks, threadLinkCandidates } from "./threadLinks.ts";
@@ -395,14 +396,15 @@ export function projectComposerContextForProvider(input: {
text: string;
records: ReadonlyArray;
}): string {
- const occurrences = collectComposerContextReferences(input.text);
- if (occurrences.length === 0) return expandAssistantCitationsForProvider(input.text);
+ const text = projectNativeAppMentions(input.text);
+ const occurrences = collectComposerContextReferences(text);
+ if (occurrences.length === 0) return expandAssistantCitationsForProvider(text);
const recordsById = new Map();
for (const record of input.records) {
recordsById.set(record.contextId, recordsById.has(record.contextId) ? undefined : record);
}
const body = expandAssistantCitationsForProvider(
- replaceComposerContextReferences(input.text, (occurrence) =>
+ replaceComposerContextReferences(text, (occurrence) =>
formatComposerContextProviderMarker(
recordsById.get(occurrence.contextId)?.kind ?? occurrence.kind,
occurrence.label,
diff --git a/packages/shared/src/cuaDriverHostOptions.test.ts b/packages/shared/src/cuaDriverHostOptions.test.ts
new file mode 100644
index 0000000000000..6107e666970c2
--- /dev/null
+++ b/packages/shared/src/cuaDriverHostOptions.test.ts
@@ -0,0 +1,33 @@
+import { describe, expect, it } from "vite-plus/test";
+
+import { cuaDriverHostOptions } from "./cuaDriverHostOptions.ts";
+
+describe("Cua host backend selection", () => {
+ it("uses Wayland even when the compositor also exposes XWayland", () => {
+ const options = cuaDriverHostOptions("/driver", "t3", "linux", {
+ WAYLAND_DISPLAY: "wayland-0",
+ DISPLAY: ":0",
+ });
+ expect(options.environment).toEqual([{ name: "CUA_DRIVER_RS_ENABLE_WAYLAND", value: "1" }]);
+ });
+
+ it.each(["0", "false", ""])("preserves an explicit opt-out (%j)", (value) => {
+ const environment = { WAYLAND_DISPLAY: "wayland-0", CUA_DRIVER_RS_ENABLE_WAYLAND: value };
+ expect(cuaDriverHostOptions("/driver", "t3", "linux", environment).environment).toEqual([
+ { name: "CUA_DRIVER_RS_ENABLE_WAYLAND", value },
+ ]);
+ });
+
+ it("does not enable Wayland for X11, headless, or non-Linux hosts", () => {
+ expect(cuaDriverHostOptions("/driver", "t3", "linux", { DISPLAY: ":0" }).environment).toEqual(
+ [],
+ );
+ expect(cuaDriverHostOptions("/driver", "t3", "linux", {}).environment).toEqual([]);
+ for (const platform of ["darwin", "win32"] as const) {
+ expect(
+ cuaDriverHostOptions("/driver", "t3", platform, { WAYLAND_DISPLAY: "wayland-0" })
+ .environment,
+ ).toEqual([]);
+ }
+ });
+});
diff --git a/packages/shared/src/cuaDriverHostOptions.ts b/packages/shared/src/cuaDriverHostOptions.ts
new file mode 100644
index 0000000000000..ca1dd8a4621b7
--- /dev/null
+++ b/packages/shared/src/cuaDriverHostOptions.ts
@@ -0,0 +1,26 @@
+/** Select the native Wayland backend for graphical Linux hosts, preserving explicit opt-outs. */
+export function cuaDriverHostOptions(
+ binaryPath: string,
+ hostBundleId: string,
+ platform: NodeJS.Platform,
+ environment: NodeJS.ProcessEnv,
+) {
+ const wayland = platform === "linux" && Boolean(environment.WAYLAND_DISPLAY?.trim());
+ return {
+ binaryPath,
+ hostBundleId,
+ approveCapabilityManifest: false,
+ approveSessionPolicy: false,
+ dangerouslyBypassApprovals: false,
+ inheritStderr: true,
+ noOverlay: false,
+ environment: wayland
+ ? [
+ {
+ name: "CUA_DRIVER_RS_ENABLE_WAYLAND",
+ value: environment.CUA_DRIVER_RS_ENABLE_WAYLAND ?? "1",
+ },
+ ]
+ : [],
+ };
+}
diff --git a/packages/shared/src/nativeAppMentions.test.ts b/packages/shared/src/nativeAppMentions.test.ts
new file mode 100644
index 0000000000000..80a04eaaf0ae6
--- /dev/null
+++ b/packages/shared/src/nativeAppMentions.test.ts
@@ -0,0 +1,43 @@
+import { describe, expect, it } from "vite-plus/test";
+import {
+ formatNativeAppMention,
+ parseNativeAppMention,
+ collectNativeAppMentions,
+ projectNativeAppMentions,
+} from "./nativeAppMentions.ts";
+import { projectComposerContextForProvider } from "./composerContextReferences.ts";
+
+describe("native application mentions", () => {
+ it.each([
+ { name: "Helium", bundleId: "net.imput.helium", launchPath: "/Applications/Helium.app" },
+ {
+ name: "Calculator",
+ bundleId: "Microsoft.WindowsCalculator_8wekyb3d8bbwe!App",
+ launchPath: "shell:AppsFolder\\Microsoft.WindowsCalculator_8wekyb3d8bbwe!App",
+ },
+ {
+ name: "Helium Browser",
+ bundleId: "helium.desktop",
+ launchPath: "/usr/share/applications/helium.desktop",
+ },
+ ])("round-trips $name with its host identity", (app) => {
+ const text = formatNativeAppMention(app, "desktop");
+ const mentions = collectNativeAppMentions(`Use ${text} please`);
+ expect(mentions).toHaveLength(1);
+ expect(mentions[0]).toMatchObject({ app, environmentId: "desktop" });
+ expect(parseNativeAppMention(mentions[0]!.href)?.app).toEqual(app);
+ const projected = projectComposerContextForProvider({ text, records: [] });
+ expect(projected).toContain(`bundle_id: ${app.bundleId}`);
+ expect(projected).toContain("environment: desktop");
+ expect(projected).not.toContain("t3-app://");
+ });
+ it("keeps malformed and unrelated links unchanged", () => {
+ expect(parseNativeAppMention("https://example.com")).toBeNull();
+ expect(parseNativeAppMention("t3-app://v1?name=Helium")).toBeNull();
+ expect(projectNativeAppMentions("[bad](t3-app://v1?name=)")).toBe("[bad](t3-app://v1?name=)");
+ });
+ it("preserves paths containing spaces and Markdown punctuation", () => {
+ const app = { name: "Example", launchPath: "C:\\Apps\\Example (Dev)\\app.exe" };
+ expect(collectNativeAppMentions(formatNativeAppMention(app, "host"))[0]?.app).toEqual(app);
+ });
+});
diff --git a/packages/shared/src/nativeAppMentions.ts b/packages/shared/src/nativeAppMentions.ts
new file mode 100644
index 0000000000000..3f98920464657
--- /dev/null
+++ b/packages/shared/src/nativeAppMentions.ts
@@ -0,0 +1,63 @@
+import { CuaApplication } from "@t3tools/contracts";
+import * as Schema from "effect/Schema";
+
+const PREFIX = "t3-app://v1?";
+const isApplication = Schema.is(CuaApplication);
+const LINKS = /\[([^\]\n]{1,200})\]\((t3-app:\/\/v1\?[^\s)]{1,8192})\)/gu;
+
+export function formatNativeAppMention(app: CuaApplication, environmentId: string): string {
+ const params = new URLSearchParams({ name: app.name, environment: environmentId });
+ if (app.bundleId) params.set("bundleId", app.bundleId);
+ if (app.launchPath) params.set("launchPath", app.launchPath);
+ const label = app.name.replace(/[[\]\\\r\n]/gu, " ");
+ return `[${label}](${PREFIX}${params})`;
+}
+
+export function parseNativeAppMention(href: string) {
+ if (!href.startsWith(PREFIX) || href.length > 8192) return null;
+ const params = new URLSearchParams(href.slice(PREFIX.length));
+ const app = {
+ name: params.get("name"),
+ ...(params.has("bundleId") ? { bundleId: params.get("bundleId") } : {}),
+ ...(params.has("launchPath") ? { launchPath: params.get("launchPath") } : {}),
+ };
+ const environmentId = params.get("environment");
+ return isApplication(app) && environmentId && environmentId.length <= 200
+ ? { app, environmentId }
+ : null;
+}
+
+export function collectNativeAppMentions(text: string) {
+ return Array.from(text.matchAll(LINKS)).flatMap((match) => {
+ const parsed = parseNativeAppMention(match[2]!);
+ return parsed
+ ? [
+ {
+ ...parsed,
+ source: match[0],
+ href: match[2]!,
+ start: match.index,
+ end: match.index + match[0].length,
+ },
+ ]
+ : [];
+ });
+}
+
+export function projectNativeAppMentions(text: string): string {
+ const mentions = collectNativeAppMentions(text);
+ if (!mentions.length) return text;
+ let result = text;
+ for (let index = mentions.length - 1; index >= 0; index--) {
+ const mention = mentions[index]!;
+ const fields = [
+ `Application: ${mention.app.name}`,
+ `environment: ${mention.environmentId}`,
+ ...(mention.app.bundleId ? [`bundle_id: ${mention.app.bundleId}`] : []),
+ ...(mention.app.launchPath ? [`launch_path: ${mention.app.launchPath}`] : []),
+ "Use the computer-use tools on this environment; resolve the app's current window before interacting.",
+ ];
+ result = result.slice(0, mention.start) + fields.join("\n") + result.slice(mention.end);
+ }
+ return result;
+}