From e9f96e8880c2c209a1ad598bbf3a2529ae41c619 Mon Sep 17 00:00:00 2001
From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com>
Date: Wed, 7 Oct 2026 15:02:17 -0700
Subject: [PATCH 1/8] fix(cua): stabilize Linux window previews and turn
cleanup
---
apps/desktop/src/cua/DesktopCuaDriver.test.ts | 3 +
apps/desktop/src/cua/DesktopCuaDriver.ts | 41 +++-
apps/server/src/cua/CuaDriver.test.ts | 6 +
apps/server/src/cua/CuaDriver.ts | 22 +-
apps/server/src/cua/CuaWindowPreview.test.ts | 226 +++++++++++++++++-
apps/server/src/cua/CuaWindowPreview.ts | 125 +++++-----
apps/web/src/components/ChatView.tsx | 23 +-
.../preview/ThreadPreviewMiniPlayer.tsx | 40 +++-
.../src/server/cuaToolPresentation.test.ts | 27 +++
.../src/server/cuaToolPresentation.ts | 14 +-
packages/shared/package.json | 4 +
.../shared/src/cuaDriverHostOptions.test.ts | 33 +++
packages/shared/src/cuaDriverHostOptions.ts | 26 ++
13 files changed, 491 insertions(+), 99 deletions(-)
create mode 100644 packages/shared/src/cuaDriverHostOptions.test.ts
create mode 100644 packages/shared/src/cuaDriverHostOptions.ts
diff --git a/apps/desktop/src/cua/DesktopCuaDriver.test.ts b/apps/desktop/src/cua/DesktopCuaDriver.test.ts
index adb9f1ab77733..9b6258b0afde6 100644
--- a/apps/desktop/src/cua/DesktopCuaDriver.test.ts
+++ b/apps/desktop/src/cua/DesktopCuaDriver.test.ts
@@ -64,6 +64,9 @@ const fixture = Effect.fn(function* (
}),
loadEmbedded: async () => ({
EmbeddedCuaDriverHost: class {
+ static withOptions() {
+ return new this();
+ }
async start() {
starts++;
Queue.offerUnsafe(starting, undefined);
diff --git a/apps/desktop/src/cua/DesktopCuaDriver.ts b/apps/desktop/src/cua/DesktopCuaDriver.ts
index 479e91219f4c2..8972c83936ed0 100644
--- a/apps/desktop/src/cua/DesktopCuaDriver.ts
+++ b/apps/desktop/src/cua/DesktopCuaDriver.ts
@@ -1,5 +1,10 @@
import { CuaDriverMcpConfiguration, type DesktopCuaDriverRequest } from "@t3tools/contracts";
-import type { EmbeddedCuaDriverHost, EmbeddedDriverConnection } from "@trycua/cua-driver/embedded";
+import { cuaDriverHostOptions } from "@t3tools/shared/cuaDriverHostOptions";
+import type {
+ EmbeddedCuaDriverHost,
+ EmbeddedDriverConnection,
+ EmbeddedDriverHostOptions,
+} from "@trycua/cua-driver/embedded";
import * as Cause from "effect/Cause";
import * as Exit from "effect/Exit";
import * as Context from "effect/Context";
@@ -16,10 +21,11 @@ import * as DesktopTelemetryPublisher from "../telemetry/DesktopTelemetryPublish
export interface DesktopCuaDriverDependencies {
readonly loadEmbedded: () => Promise<{
- readonly EmbeddedCuaDriverHost: new (
- path: string,
- hostBundleId: string,
- ) => Pick;
+ readonly EmbeddedCuaDriverHost: {
+ readonly withOptions: (
+ options: EmbeddedDriverHostOptions,
+ ) => Pick;
+ };
}>;
readonly loadElectron: () => Promise<
Pick<
@@ -41,7 +47,21 @@ const hostStep = (step: string) => (cause: unknown) =>
// The SDK loads its native libraries from the archive's .unpacked sibling
// through our @ubjs/node patch, so a packaged app imports it in place.
const defaultDependencies: DesktopCuaDriverDependencies = {
- loadEmbedded: () => import("@trycua/cua-driver/embedded"),
+ loadEmbedded: async () => {
+ const module = await import("@trycua/cua-driver/embedded");
+ return {
+ EmbeddedCuaDriverHost: {
+ withOptions: (options) => {
+ const host = module.EmbeddedCuaDriverHost.withOptions(options);
+ // The generated factory's interface omits the native object's disposal method.
+ if (!(host instanceof module.EmbeddedCuaDriverHost)) {
+ throw new Error("Cua Driver returned an unexpected host object.");
+ }
+ return host;
+ },
+ },
+ };
+ },
loadElectron: () => import("@trycua/cua-driver/electron"),
};
@@ -232,7 +252,14 @@ export const make = Effect.fn("desktop.cuaDriver.make")(function* (
const owned = yield* Effect.try({
try: () => {
const owned: OwnedHost = {
- host: new module.EmbeddedCuaDriverHost(binaryPath.value, environment.appUserModelId),
+ host: module.EmbeddedCuaDriverHost.withOptions(
+ cuaDriverHostOptions(
+ binaryPath.value,
+ environment.appUserModelId,
+ environment.platform,
+ process.env,
+ ),
+ ),
abort: new AbortController(),
};
hosts.add(owned);
diff --git a/apps/server/src/cua/CuaDriver.test.ts b/apps/server/src/cua/CuaDriver.test.ts
index ba423c3569472..ea3e1252c4aba 100644
--- a/apps/server/src/cua/CuaDriver.test.ts
+++ b/apps/server/src/cua/CuaDriver.test.ts
@@ -56,6 +56,9 @@ const standaloneFixture = Effect.fn(function* (failFirstDestroy = false) {
Effect.succeed("/driver"),
Effect.succeed({
EmbeddedCuaDriverHost: class {
+ static withOptions() {
+ return new this();
+ }
constructor() {
creates++;
}
@@ -127,6 +130,9 @@ describe("Cua Driver failure boundaries", () => {
Effect.succeed("/driver"),
Effect.succeed({
EmbeddedCuaDriverHost: class {
+ static withOptions() {
+ return new this();
+ }
constructor() {
throw cause;
}
diff --git a/apps/server/src/cua/CuaDriver.ts b/apps/server/src/cua/CuaDriver.ts
index a69ae11225a69..b7d43d511e5be 100644
--- a/apps/server/src/cua/CuaDriver.ts
+++ b/apps/server/src/cua/CuaDriver.ts
@@ -6,7 +6,12 @@ import type {
DesktopCuaDriverReport,
} from "@t3tools/contracts";
import * as HostProcess from "@t3tools/shared/HostProcess";
-import type { EmbeddedCuaDriverHost, EmbeddedDriverConnection } from "@trycua/cua-driver/embedded";
+import { cuaDriverHostOptions } from "@t3tools/shared/cuaDriverHostOptions";
+import type {
+ EmbeddedCuaDriverHost,
+ EmbeddedDriverConnection,
+ EmbeddedDriverHostOptions,
+} from "@trycua/cua-driver/embedded";
import * as Config from "effect/Config";
import * as Context from "effect/Context";
import * as SecureRandom from "effect/crypto/SecureRandom";
@@ -315,10 +320,11 @@ export const make = Effect.fn("CuaDriver.make")(function* (
});
type StandaloneHostModule = {
- readonly EmbeddedCuaDriverHost: new (
- path: string,
- hostBundleId: string,
- ) => Pick;
+ readonly EmbeddedCuaDriverHost: {
+ readonly withOptions: (
+ options: EmbeddedDriverHostOptions,
+ ) => Pick;
+ };
};
// A Node single-executable cannot `import()` the external SDK from disk, and the SDK exposes
@@ -352,6 +358,7 @@ export const makeStandaloneHostFactory = Effect.fn("CuaDriver.standaloneHostFact
resolveBinary: Effect.Effect,
loadEmbedded: Effect.Effect,
) {
+ const platform = yield* HostProcess.Platform;
const mutex = yield* Semaphore.make(1);
let occupied = false;
// Return a lazy acquisition effect so one factory owns all subsequent host attempts.
@@ -363,7 +370,10 @@ export const makeStandaloneHostFactory = Effect.fn("CuaDriver.standaloneHostFact
const binaryPath = yield* resolveBinary;
const { EmbeddedCuaDriverHost } = yield* loadEmbedded;
const host = yield* Effect.try({
- try: () => new EmbeddedCuaDriverHost(binaryPath, "com.t3tools.t3code.server"),
+ try: () =>
+ EmbeddedCuaDriverHost.withOptions(
+ cuaDriverHostOptions(binaryPath, "com.t3tools.t3code.server", platform, process.env),
+ ),
catch: (cause) => new CuaDriverHostCreateError({ binaryPath, cause }),
});
occupied = true;
diff --git a/apps/server/src/cua/CuaWindowPreview.test.ts b/apps/server/src/cua/CuaWindowPreview.test.ts
index 1c1db464e3ef4..149fe513d630e 100644
--- a/apps/server/src/cua/CuaWindowPreview.test.ts
+++ b/apps/server/src/cua/CuaWindowPreview.test.ts
@@ -9,11 +9,15 @@ import * as Fiber from "effect/Fiber";
import * as Layer from "effect/Layer";
import * as PubSub from "effect/PubSub";
import * as Stream from "effect/Stream";
+import * as TestClock from "effect/testing/TestClock";
import { describe, expect } from "vite-plus/test";
+import {
+ clearCuaToolContext,
+ cuaToolPresentation,
+} from "@t3tools/provider-core/server/cuaToolPresentation";
import type { McpProviderSessionConfig } from "@t3tools/provider-core/server/mcpSession";
import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
-import { cuaToolPresentation } from "@t3tools/provider-core/server/cuaToolPresentation";
import * as Orchestrator from "../orchestration-v2/Orchestrator.ts";
import * as CuaWindowPreview from "./CuaWindowPreview.ts";
@@ -27,22 +31,31 @@ const turnEvent = (type: "turn.started" | "turn.completed") =>
payload: { status: type === "turn.started" ? "running" : "completed" },
}) as unknown as OrchestrationV2DomainEvent;
-const makeHarness = Effect.fn(function* () {
+const makeHarness = Effect.fn(function* (
+ windowFailure?: "missing" | "throw" | "transient",
+ onWindowCapture?: () => void | Promise,
+) {
const events = yield* PubSub.unbounded();
const captures: string[] = [];
+ const windows: Array<{ pid: number; windowId: bigint }> = [];
let clients = 0;
let destroyed = 0;
const fakeClient = {
- getWindowState: (input: { pid: number; windowId: bigint }) =>
- Promise.resolve({
+ getWindowState: async (input: { pid: number; windowId: bigint }) => {
+ windows.push(input);
+ await onWindowCapture?.();
+ if (windowFailure === "throw" || (windowFailure === "transient" && windows.length < 3))
+ return Promise.reject(new Error("Window capture interrupted"));
+ return Promise.resolve({
pid: input.pid,
windowId: input.windowId,
appName: "Calendar",
windowTitle: "December 2026",
screenshotWidth: 640,
screenshotHeight: 400,
- images: [{ mimeType: "image/png", dataBase64: png }],
- }),
+ images: windowFailure === "missing" ? [] : [{ mimeType: "image/png", dataBase64: png }],
+ });
+ },
getDesktopState: () => {
captures.push("desktop");
return Promise.resolve({
@@ -80,6 +93,7 @@ const makeHarness = Effect.fn(function* () {
service,
emit: (event: OrchestrationV2DomainEvent) => PubSub.publish(events, event),
captures,
+ windows,
counts: () => ({ clients, destroyed }),
};
});
@@ -97,10 +111,61 @@ const setCuaSession = Effect.flatMap(McpProviderSessions.McpProviderSessions, (s
);
describe("CuaWindowPreview", () => {
- it.live("captures only while a client subscribes during a Cua turn", () =>
+ it.effect("never overlaps captures and backs off after a slow frame", () =>
Effect.gen(function* () {
+ clearCuaToolContext(threadId);
yield* setCuaSession;
+ yield* Effect.addFinalizer(() => Effect.sync(() => clearCuaToolContext(threadId)));
+ cuaToolPresentation({
+ threadId,
+ rawToolName: "cua-driver/get_window_state",
+ args: { pid: 42, window_id: 7 },
+ status: "completed",
+ });
+ const gate = Promise.withResolvers();
+ const firstStarted = yield* Deferred.make();
+ const secondStarted = yield* Deferred.make();
+ const firstPublished = yield* Deferred.make();
+ let calls = 0;
+ const harness = yield* makeHarness(undefined, () => {
+ calls += 1;
+ Deferred.doneUnsafe(calls === 1 ? firstStarted : secondStarted, Effect.void);
+ return calls === 1 ? gate.promise : undefined;
+ });
+ yield* harness.service.stream(threadId).pipe(
+ Stream.runForEach((state) =>
+ state.frame ? Deferred.succeed(firstPublished, undefined) : Effect.void,
+ ),
+ Effect.forkScoped,
+ );
+ yield* Effect.yieldNow;
+ yield* harness.emit(turnEvent("turn.started"));
+ yield* Deferred.await(firstStarted);
+ yield* TestClock.adjust(40);
+ expect(calls).toBe(1);
+ gate.resolve();
+ yield* Deferred.await(firstPublished);
+ yield* TestClock.adjust(39);
+ expect(calls).toBe(1);
+ yield* TestClock.adjust(1);
+ yield* Deferred.await(secondStarted);
+ expect(calls).toBe(2);
+ }).pipe(Effect.scoped, Effect.provide(layer)),
+ );
+
+ it.live("stops capture at turn end and waits for a fresh window in the next turn", () =>
+ Effect.gen(function* () {
+ yield* setCuaSession;
+ clearCuaToolContext(threadId);
+ cuaToolPresentation({
+ threadId,
+ rawToolName: "cua-driver/get_window_state",
+ args: { pid: 42, window_id: 7 },
+ status: "completed",
+ });
const harness = yield* makeHarness();
+ const waiting = yield* Deferred.make();
+ const resumed = yield* Deferred.make();
const live = yield* Deferred.make();
const idle = yield* Deferred.make();
const states: string[] = [];
@@ -108,7 +173,13 @@ describe("CuaWindowPreview", () => {
Stream.tap((state) =>
Effect.gen(function* () {
states.push(state.status);
- if (state.status === "live") yield* Deferred.succeed(live, undefined);
+ if (state.status === "live" && state.frame) {
+ yield* Deferred.succeed(live, undefined);
+ if (harness.windows.at(-1)?.windowId === 8n)
+ yield* Deferred.succeed(resumed, undefined);
+ }
+ if (state.status === "live" && !state.frame)
+ yield* Deferred.succeed(waiting, undefined);
if (state.status === "idle" && states.length > 1)
yield* Deferred.succeed(idle, undefined);
}),
@@ -123,11 +194,25 @@ describe("CuaWindowPreview", () => {
yield* harness.emit(turnEvent("turn.started"));
yield* Deferred.await(live);
expect(harness.counts().clients).toBe(1);
- expect(harness.captures.length).toBeGreaterThan(0);
+ expect(harness.windows.length).toBeGreaterThan(0);
yield* harness.emit(turnEvent("turn.completed"));
yield* Deferred.await(idle);
expect(harness.counts().destroyed).toBe(1);
+ const capturesAtIdle = harness.windows.length;
+ yield* harness.emit(turnEvent("turn.started"));
+ yield* Deferred.await(waiting);
+ expect(harness.windows.length).toBe(capturesAtIdle);
+ expect(harness.captures).toEqual([]);
+ cuaToolPresentation({
+ threadId,
+ rawToolName: "cua-driver/get_window_state",
+ args: { pid: 43, window_id: 8 },
+ status: "completed",
+ });
+ yield* Deferred.await(resumed);
+ expect(harness.windows.at(-1)).toMatchObject({ pid: 43, windowId: 8n });
+ expect(states).not.toContain("unavailable");
yield* Fiber.interrupt(subscriber);
}).pipe(Effect.scoped, Effect.provide(layer)),
);
@@ -172,4 +257,127 @@ describe("CuaWindowPreview", () => {
expect(harness.counts().clients).toBe(0);
}).pipe(Effect.scoped, Effect.provide(layer)),
);
+
+ it.live("follows window selection and switching after capture has started", () =>
+ Effect.gen(function* () {
+ clearCuaToolContext(threadId);
+ yield* setCuaSession;
+ yield* Effect.addFinalizer(() => Effect.sync(() => clearCuaToolContext(threadId)));
+ const harness = yield* makeHarness();
+ const waiting = yield* Deferred.make();
+ const firstWindow = yield* Deferred.make();
+ const secondWindow = yield* Deferred.make();
+ yield* harness.service.stream(threadId).pipe(
+ Stream.runForEach((state) =>
+ Effect.gen(function* () {
+ if (state.status !== "live") return;
+ const window = harness.windows.at(-1);
+ if (!window) yield* Deferred.succeed(waiting, undefined);
+ else if (window.windowId === 7n) yield* Deferred.succeed(firstWindow, undefined);
+ else if (window.windowId === 8n) yield* Deferred.succeed(secondWindow, undefined);
+ }),
+ ),
+ Effect.forkScoped,
+ );
+ yield* Effect.yieldNow;
+ yield* harness.emit(turnEvent("turn.started"));
+ yield* Deferred.await(waiting);
+ expect(harness.captures).toEqual([]);
+ cuaToolPresentation({
+ threadId,
+ rawToolName: "cua-driver/get_window_state",
+ args: { pid: 42, window_id: 7 },
+ status: "completed",
+ });
+ yield* Deferred.await(firstWindow);
+ cuaToolPresentation({
+ threadId,
+ rawToolName: "cua-driver/get_window_state",
+ args: { pid: 43, window_id: 8 },
+ status: "completed",
+ });
+ yield* Deferred.await(secondWindow);
+ expect(harness.windows.at(-1)).toMatchObject({ pid: 43, windowId: 8n });
+ expect(harness.counts().clients).toBe(1);
+ }).pipe(Effect.scoped, Effect.provide(layer)),
+ );
+
+ it.live("discards a completed capture after the agent switches windows", () =>
+ Effect.gen(function* () {
+ yield* setCuaSession;
+ yield* Effect.addFinalizer(() => Effect.sync(() => clearCuaToolContext(threadId)));
+ const selectWindow = (windowId: number) =>
+ cuaToolPresentation({
+ threadId,
+ rawToolName: "cua-driver/get_window_state",
+ args: { pid: 42, window_id: windowId },
+ status: "completed",
+ });
+ selectWindow(7);
+ const harness = yield* makeHarness(undefined, () => {
+ selectWindow(8);
+ });
+ const first = yield* harness.service.stream(threadId).pipe(
+ Stream.filter((state) => state.status !== "idle"),
+ Stream.runHead,
+ Effect.forkScoped,
+ );
+ yield* Effect.yieldNow;
+ yield* harness.emit(turnEvent("turn.started"));
+ const state = yield* Fiber.join(first);
+ expect(state._tag === "Some" ? state.value.status : null).toBe("live");
+ expect(harness.windows.map((window) => window.windowId)).toEqual([7n, 8n]);
+ }).pipe(Effect.scoped, Effect.provide(layer)),
+ );
+
+ it.live("retries transient capture failures without publishing unavailable", () =>
+ Effect.gen(function* () {
+ yield* setCuaSession;
+ yield* Effect.addFinalizer(() => Effect.sync(() => clearCuaToolContext(threadId)));
+ cuaToolPresentation({
+ threadId,
+ rawToolName: "cua-driver/get_window_state",
+ args: { pid: 42, window_id: 7 },
+ status: "completed",
+ });
+ const harness = yield* makeHarness("transient");
+ const first = yield* harness.service.stream(threadId).pipe(
+ Stream.filter((state) => state.status !== "idle"),
+ Stream.runHead,
+ Effect.forkScoped,
+ );
+ yield* Effect.yieldNow;
+ yield* harness.emit(turnEvent("turn.started"));
+ const state = yield* Fiber.join(first);
+ expect(state._tag === "Some" ? state.value.status : null).toBe("live");
+ expect(harness.windows).toHaveLength(3);
+ expect(harness.captures).toEqual([]);
+ }).pipe(Effect.scoped, Effect.provide(layer)),
+ );
+
+ it.live.each(["missing", "throw"] as const)(
+ "does not substitute desktop pixels when window capture is %s",
+ (failure) =>
+ Effect.gen(function* () {
+ yield* setCuaSession;
+ cuaToolPresentation({
+ threadId,
+ rawToolName: "cua-driver/click",
+ args: { pid: 42, window_id: "7", x: 1, y: 1 },
+ status: "completed",
+ });
+ const harness = yield* makeHarness(failure);
+ const unavailable = yield* harness.service.stream(threadId).pipe(
+ Stream.filter((state) => state.status !== "idle"),
+ Stream.runHead,
+ Effect.forkScoped,
+ );
+ yield* Effect.yieldNow;
+ yield* harness.emit(turnEvent("turn.started"));
+ const state = yield* Fiber.join(unavailable);
+ expect(state._tag === "Some" ? state.value.status : null).toBe("unavailable");
+ expect(state._tag === "Some" ? state.value.frame : null).toBeUndefined();
+ expect(harness.captures).toEqual([]);
+ }).pipe(Effect.scoped, Effect.provide(layer)),
+ );
});
diff --git a/apps/server/src/cua/CuaWindowPreview.ts b/apps/server/src/cua/CuaWindowPreview.ts
index 987b3ac23eede..af3b042365b73 100644
--- a/apps/server/src/cua/CuaWindowPreview.ts
+++ b/apps/server/src/cua/CuaWindowPreview.ts
@@ -7,6 +7,7 @@ import type {
ThreadId,
} from "@t3tools/contracts";
import type * as CuaSdk from "@trycua/cua-driver";
+import * as Clock from "effect/Clock";
import * as Context from "effect/Context";
import * as DateTime from "effect/DateTime";
import * as Duration from "effect/Duration";
@@ -14,7 +15,6 @@ import * as Effect from "effect/Effect";
import * as Exit from "effect/Exit";
import * as FileSystem from "effect/FileSystem";
import * as Layer from "effect/Layer";
-import * as Option from "effect/Option";
import * as Path from "effect/Path";
import * as PubSub from "effect/PubSub";
import * as Schedule from "effect/Schedule";
@@ -26,6 +26,7 @@ import * as Stream from "effect/Stream";
import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
import * as Orchestrator from "../orchestration-v2/Orchestrator.ts";
import {
+ clearCuaWindowTarget,
readCuaWindowTarget,
type CuaWindowTarget,
} from "@t3tools/provider-core/server/cuaToolPresentation";
@@ -56,18 +57,12 @@ export class CuaWindowPreview extends Context.Service<
}
>()("t3/cua/CuaWindowPreview") {}
-export type SdkModule = Pick<
- typeof CuaSdk,
- "CuaDriver" | "GetWindowStateInput" | "GetDesktopStateInput"
->;
+export type SdkModule = Pick;
// `connect` is typed as the interface, but every implementation is the
// generated class with the native handle to release.
-type SdkClient = Pick<
- CuaSdk.CuaDriver,
- "getWindowState" | "getDesktopState" | "shutdown" | "uniffiDestroy"
->;
+type SdkClient = Pick;
-const REFRESH_INTERVAL = Duration.seconds(1.5);
+const REFRESH_INTERVAL = Duration.millis(250);
/** Long edge of the preview frame; small enough for a floating card over the app WS. */
const MAX_DIMENSION = 800;
const IDLE_STATE: CuaWindowPreviewState = { status: "idle" };
@@ -94,58 +89,38 @@ const loadSdk = Effect.fn("CuaWindowPreview.loadSdk")(function* () {
return yield* new CuaWindowPreviewSdkError({ cause: "@trycua/cua-driver is not installed." });
});
-/**
- * Captures the window the agent last targeted, or the whole desktop before it
- * has named one. A window that disappeared falls back to the desktop rather
- * than failing, so the card keeps showing something while the agent moves on.
- */
+/** Captures only the selected window; desktop pixels cannot stand in for it. */
const captureFrame = Effect.fn("CuaWindowPreview.capture")(function* (
sdk: SdkModule,
client: SdkClient,
- target: CuaWindowTarget | undefined,
+ target: CuaWindowTarget,
) {
const capturedAt = DateTime.formatIso(yield* DateTime.now);
- if (target) {
- const window = yield* Effect.tryPromise({
- try: (signal) =>
- client.getWindowState(
- sdk.GetWindowStateInput.new({
- pid: target.pid,
- windowId: target.windowId,
- includeAccessibilityTree: false,
- includeScreenshot: true,
- maxDimension: MAX_DIMENSION,
- }),
- { signal },
- ),
- catch: (cause) => new CuaWindowPreviewCaptureError({ cause }),
- }).pipe(Effect.option);
- const image = Option.isSome(window) ? window.value.images[0] : undefined;
- if (Option.isSome(window) && image) {
- return {
- ...(window.value.appName ? { appName: window.value.appName } : {}),
- ...(window.value.windowTitle ? { windowTitle: window.value.windowTitle } : {}),
- width: window.value.screenshotWidth ?? 0,
- height: window.value.screenshotHeight ?? 0,
- mimeType: image.mimeType,
- dataBase64: image.dataBase64,
- capturedAt,
- } satisfies CuaWindowPreviewFrame;
- }
- }
- const desktop = yield* Effect.tryPromise({
- try: (signal) => client.getDesktopState(sdk.GetDesktopStateInput.new({}), { signal }),
+ const window = yield* Effect.tryPromise({
+ try: (signal) =>
+ client.getWindowState(
+ sdk.GetWindowStateInput.new({
+ pid: target.pid,
+ windowId: target.windowId,
+ includeAccessibilityTree: false,
+ includeScreenshot: true,
+ maxDimension: MAX_DIMENSION,
+ }),
+ { signal },
+ ),
catch: (cause) => new CuaWindowPreviewCaptureError({ cause }),
});
- const image = desktop.images[0];
- if (desktop.isError || !image) {
+ const image = window.images[0];
+ if (!image) {
return yield* new CuaWindowPreviewCaptureError({
- cause: desktop.errorCode ?? desktop.text ?? "no image",
+ cause: "Window capture is unavailable.",
});
}
return {
- width: 0,
- height: 0,
+ ...(window.appName ? { appName: window.appName } : {}),
+ ...(window.windowTitle ? { windowTitle: window.windowTitle } : {}),
+ width: window.screenshotWidth ?? 0,
+ height: window.screenshotHeight ?? 0,
mimeType: image.mimeType,
dataBase64: image.dataBase64,
capturedAt,
@@ -226,17 +201,42 @@ export const make = Effect.fn("CuaWindowPreview.make")(function* (
Effect.andThen(Effect.sync(() => client.uniffiDestroy())),
),
);
- yield* captureFrame(module, client, readCuaWindowTarget(threadId)).pipe(
- Effect.flatMap((frame) => publish(preview, { status: "live", frame })),
- Effect.catchTags({
- CuaWindowPreviewCaptureError: (error) =>
- publish(preview, {
- status: "unavailable",
- detail: typeof error.cause === "string" ? error.cause : error.message,
- }),
- }),
- Effect.repeat(Schedule.spaced(refreshInterval)),
- );
+ const capture = Effect.suspend(() => {
+ const target = readCuaWindowTarget(threadId);
+ if (!target) return publish(preview, { status: "live" });
+ const isCurrent = () => {
+ const current = readCuaWindowTarget(threadId);
+ return current?.pid === target?.pid && current?.windowId === target?.windowId;
+ };
+ return captureFrame(module, client, target).pipe(
+ // Compositor transitions can briefly fail a frame. Keep the current
+ // frame during bounded retries, but report persistent capture loss.
+ Effect.retry({ times: 2, schedule: Schedule.spaced("150 millis"), while: isCurrent }),
+ Effect.flatMap((frame) =>
+ isCurrent() ? publish(preview, { status: "live", frame }) : Effect.void,
+ ),
+ Effect.catchTags({
+ CuaWindowPreviewCaptureError: (error) =>
+ isCurrent()
+ ? publish(preview, {
+ status: "unavailable",
+ detail: typeof error.cause === "string" ? error.cause : error.message,
+ })
+ : Effect.void,
+ }),
+ );
+ });
+ return yield* Effect.gen(function* () {
+ const startedAt = yield* Clock.currentTimeMillis;
+ yield* capture;
+ const elapsed = Math.max(0, (yield* Clock.currentTimeMillis) - startedAt);
+ // Cap at 4 FPS, with at least as much idle time as capture time.
+ // Slow captures lower the rate instead of filling a queue or busy-looping.
+ const minimumInterval = readCuaWindowTarget(threadId)
+ ? Duration.toMillis(refreshInterval)
+ : 1500;
+ yield* Effect.sleep(Math.max(minimumInterval - elapsed, elapsed));
+ }).pipe(Effect.forever);
},
(effect, _threadId, preview) =>
effect.pipe(
@@ -287,6 +287,7 @@ export const make = Effect.fn("CuaWindowPreview.make")(function* (
preview.live = (yield* mcpSessions.read(event.threadId))?.cuaDriver?.socketPath !== undefined;
yield* reconcile(event.threadId);
} else if (status !== "pending") {
+ clearCuaWindowTarget(event.threadId);
const preview = threads.get(event.threadId);
if (!preview) return;
preview.live = false;
diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx
index a4011cf0678c8..957f1bd920702 100644
--- a/apps/web/src/components/ChatView.tsx
+++ b/apps/web/src/components/ChatView.tsx
@@ -2026,17 +2026,30 @@ export default function ChatView(props: ChatViewProps) {
// Each new call reopens a card the user closed; a browser or device already floating stays.
const computerUse = selectLatestComputerUse(serverProjection?.turnItems ?? []);
const lastShownComputerUseItemId = useRef(null);
- // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- Floats the computer card when the shown thread's projection reports a new computer use call.
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- Floats the computer card for a new computer use call in the shown thread's projection and dismisses it when the run settles.
useEffect(() => {
- if (!activeThreadRef || !computerUse?.inProgress || shouldUsePlanSidebarSheet) return;
+ if (!activeThreadRef) return;
+ const current = activePreviewMiniPlayer;
+ if (latestRunSettled) {
+ if (current?.source.kind === "computer") {
+ usePreviewMiniPlayerStore.getState().close(activeThreadRef);
+ }
+ return;
+ }
+ if (!computerUse?.inProgress || shouldUsePlanSidebarSheet) return;
if (lastShownComputerUseItemId.current === computerUse.itemId) return;
lastShownComputerUseItemId.current = computerUse.itemId;
if (!getClientSettings().browserAutoShowFloatingPreview) return;
- const current =
- usePreviewMiniPlayerStore.getState().byThreadKey[scopedThreadKey(activeThreadRef)];
if (current && current.source.kind !== "computer") return;
usePreviewMiniPlayerStore.getState().open(activeThreadRef, COMPUTER_MINI_PLAYER_SOURCE);
- }, [activeThreadRef, computerUse, shouldUsePlanSidebarSheet]);
+ }, [
+ activePreviewMiniPlayer,
+ activeThreadRef,
+ computerUse,
+ lastShownComputerUseItemId,
+ latestRunSettled,
+ shouldUsePlanSidebarSheet,
+ ]);
// The thread's own change request, placed against the project it belongs to. Without a
// project there is nothing to resolve it against, so the caller falls back to the browser.
const activeProjectRepository = sourceControlClients.repositorySelector(
diff --git a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx
index 960411e7cf96b..e6b508761908a 100644
--- a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx
+++ b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx
@@ -253,9 +253,9 @@ export function ThreadPreviewMiniPlayer({
/**
* The window an agent drives on the host computer. While a computer use turn
- * runs, the server captures that window every second and a half and streams
- * it here; the capture runs only while this card is mounted. Between turns the
- * card keeps the last frame instead of going blank.
+ * runs, the server captures that window at an adaptive rate and streams
+ * it here; the capture runs only while this card is mounted. Chat dismisses
+ * the card when the turn settles.
*/
function ComputerMiniPlayer({
threadRef,
@@ -264,11 +264,35 @@ function ComputerMiniPlayer({
}: Props & { readonly preview: ComputerUsePreview | null }) {
const gt = useGT();
const live = useCuaWindowPreview(threadRef);
- const [lastFrame, setLastFrame] = useState(null);
+ const [decoded, setDecoded] = useState<{
+ frame: CuaWindowPreviewFrame;
+ src: string;
+ } | null>(null);
const liveFrame = live.status === "live" ? (live.frame ?? null) : null;
- if (liveFrame !== null && liveFrame !== lastFrame) setLastFrame(liveFrame);
- const frame = liveFrame ?? lastFrame;
- const src = frame ? `data:${frame.mimeType};base64,${frame.dataBase64}` : null;
+ if (live.status === "live" && !liveFrame && decoded !== null) setDecoded(null);
+ useEffect(() => {
+ if (!liveFrame) return;
+ let cancelled = false;
+ const image = new Image();
+ const src = `data:${liveFrame.mimeType};base64,${liveFrame.dataBase64}`;
+ image.src = src;
+ // Keep the visible frame until the replacement is ready to paint.
+ void image.decode().then(
+ () => {
+ if (!cancelled) setDecoded({ frame: liveFrame, src });
+ },
+ () => {
+ // A malformed frame must not erase the last successfully decoded one.
+ },
+ );
+ return () => {
+ cancelled = true;
+ };
+ }, [liveFrame]);
+ const visible =
+ live.status === "unavailable" || (live.status === "live" && !liveFrame) ? null : decoded;
+ const frame = visible?.frame ?? null;
+ const src = visible?.src ?? null;
const sourceSize =
frame && frame.width > 0 && frame.height > 0
? { width: frame.width, height: frame.height }
@@ -311,7 +335,7 @@ function ComputerMiniPlayer({
diff --git a/packages/provider-core/src/server/cuaToolPresentation.test.ts b/packages/provider-core/src/server/cuaToolPresentation.test.ts
index f91110bb01ca7..2fad60dc2fdd3 100644
--- a/packages/provider-core/src/server/cuaToolPresentation.test.ts
+++ b/packages/provider-core/src/server/cuaToolPresentation.test.ts
@@ -252,6 +252,33 @@ const CUA_DRIVER_TOOLS = [
];
describe("readCuaWindowTarget", () => {
+ it("keeps the capture target when restoring focus or a later tool fails", () => {
+ clearCuaToolContext(threadId);
+ cuaToolPresentation({
+ threadId,
+ rawToolName: "cua-driver/get_window_state",
+ args: { pid: 42, window_id: 7 },
+ status: "completed",
+ });
+ for (const status of ["inProgress", "completed"] as const) {
+ cuaToolPresentation({
+ threadId,
+ rawToolName: "cua-driver/bring_to_front",
+ args: { pid: 99, window_id: 8 },
+ status,
+ });
+ NodeAssert.deepEqual(readCuaWindowTarget(threadId), { pid: 42, windowId: 7n });
+ }
+ cuaToolPresentation({
+ threadId,
+ rawToolName: "cua-driver/get_window_state",
+ args: { pid: 99, window_id: 8 },
+ status: "failed",
+ });
+ NodeAssert.deepEqual(readCuaWindowTarget(threadId), { pid: 42, windowId: 7n });
+ clearCuaToolContext(threadId);
+ });
+
it("remembers the last pid and window id a tool call addressed", () => {
clearCuaToolContext(threadId);
NodeAssert.equal(readCuaWindowTarget(threadId), undefined);
diff --git a/packages/provider-core/src/server/cuaToolPresentation.ts b/packages/provider-core/src/server/cuaToolPresentation.ts
index 22530ef0f47d1..7b5f1c25fab27 100644
--- a/packages/provider-core/src/server/cuaToolPresentation.ts
+++ b/packages/provider-core/src/server/cuaToolPresentation.ts
@@ -83,6 +83,12 @@ export function clearCuaToolContext(threadId: ThreadId): void {
directories.delete(threadId);
}
+/** End the turn's capture target without forgetting learned app identities. */
+export function clearCuaWindowTarget(threadId: ThreadId): void {
+ const directory = directories.get(threadId);
+ if (directory) directory.lastWindow = undefined;
+}
+
/** Last window the agent targeted in this thread, if any tool call named one. */
export function readCuaWindowTarget(threadId: ThreadId): CuaWindowTarget | undefined {
return directories.get(threadId)?.lastWindow;
@@ -210,6 +216,7 @@ function humanizeBundleId(bundleId: string): string | undefined {
function resolveApp(
threadId: ThreadId,
args: Record | undefined,
+ trackWindow: boolean,
): CuaApp | undefined {
const directory = directories.get(threadId);
const pid = asPid(args?.pid);
@@ -217,7 +224,7 @@ function resolveApp(
const name = asAppName(args?.app_name) ?? asAppName(args?.app);
const known = pid !== undefined ? directory?.byPid.get(pid) : undefined;
const windowId = asWindowId(args?.window_id);
- if (pid !== undefined && windowId !== undefined) {
+ if (trackWindow && pid !== undefined && windowId !== undefined) {
directoryFor(threadId).lastWindow = { pid, windowId };
}
const app: CuaApp = {
@@ -465,7 +472,10 @@ export function cuaToolPresentation(input: {
if (!tool) return undefined;
const args = asRecord(input.args);
const targetArgs = tool === "run_steps" ? runStepsTarget(args) : args;
- const app = TOOLS_WITHOUT_APP.has(tool) ? undefined : resolveApp(input.threadId, targetArgs);
+ // Restoring the user's foreground window must not redirect the live preview.
+ const app = TOOLS_WITHOUT_APP.has(tool)
+ ? undefined
+ : resolveApp(input.threadId, targetArgs, tool !== "bring_to_front" && input.status !== "failed");
const appName = app?.name ?? (app?.bundleId ? humanizeBundleId(app.bundleId) : undefined);
const reference = app ? nativeApp(app) : undefined;
const icon = reference ? ({ _tag: "native-app", app: reference } as const) : undefined;
diff --git a/packages/shared/package.json b/packages/shared/package.json
index 38d187377bf83..7f5ad87d7b196 100644
--- a/packages/shared/package.json
+++ b/packages/shared/package.json
@@ -631,6 +631,10 @@
"types": "./src/paletteSearchCopy.ts",
"import": "./src/paletteSearchCopy.ts"
},
+ "./cuaDriverHostOptions": {
+ "types": "./src/cuaDriverHostOptions.ts",
+ "import": "./src/cuaDriverHostOptions.ts"
+ },
"./cuaDriverRelease": {
"types": "./src/cuaDriverRelease.ts",
"import": "./src/cuaDriverRelease.ts"
diff --git a/packages/shared/src/cuaDriverHostOptions.test.ts b/packages/shared/src/cuaDriverHostOptions.test.ts
new file mode 100644
index 0000000000000..6107e666970c2
--- /dev/null
+++ b/packages/shared/src/cuaDriverHostOptions.test.ts
@@ -0,0 +1,33 @@
+import { describe, expect, it } from "vite-plus/test";
+
+import { cuaDriverHostOptions } from "./cuaDriverHostOptions.ts";
+
+describe("Cua host backend selection", () => {
+ it("uses Wayland even when the compositor also exposes XWayland", () => {
+ const options = cuaDriverHostOptions("/driver", "t3", "linux", {
+ WAYLAND_DISPLAY: "wayland-0",
+ DISPLAY: ":0",
+ });
+ expect(options.environment).toEqual([{ name: "CUA_DRIVER_RS_ENABLE_WAYLAND", value: "1" }]);
+ });
+
+ it.each(["0", "false", ""])("preserves an explicit opt-out (%j)", (value) => {
+ const environment = { WAYLAND_DISPLAY: "wayland-0", CUA_DRIVER_RS_ENABLE_WAYLAND: value };
+ expect(cuaDriverHostOptions("/driver", "t3", "linux", environment).environment).toEqual([
+ { name: "CUA_DRIVER_RS_ENABLE_WAYLAND", value },
+ ]);
+ });
+
+ it("does not enable Wayland for X11, headless, or non-Linux hosts", () => {
+ expect(cuaDriverHostOptions("/driver", "t3", "linux", { DISPLAY: ":0" }).environment).toEqual(
+ [],
+ );
+ expect(cuaDriverHostOptions("/driver", "t3", "linux", {}).environment).toEqual([]);
+ for (const platform of ["darwin", "win32"] as const) {
+ expect(
+ cuaDriverHostOptions("/driver", "t3", platform, { WAYLAND_DISPLAY: "wayland-0" })
+ .environment,
+ ).toEqual([]);
+ }
+ });
+});
diff --git a/packages/shared/src/cuaDriverHostOptions.ts b/packages/shared/src/cuaDriverHostOptions.ts
new file mode 100644
index 0000000000000..ca1dd8a4621b7
--- /dev/null
+++ b/packages/shared/src/cuaDriverHostOptions.ts
@@ -0,0 +1,26 @@
+/** Select the native Wayland backend for graphical Linux hosts, preserving explicit opt-outs. */
+export function cuaDriverHostOptions(
+ binaryPath: string,
+ hostBundleId: string,
+ platform: NodeJS.Platform,
+ environment: NodeJS.ProcessEnv,
+) {
+ const wayland = platform === "linux" && Boolean(environment.WAYLAND_DISPLAY?.trim());
+ return {
+ binaryPath,
+ hostBundleId,
+ approveCapabilityManifest: false,
+ approveSessionPolicy: false,
+ dangerouslyBypassApprovals: false,
+ inheritStderr: true,
+ noOverlay: false,
+ environment: wayland
+ ? [
+ {
+ name: "CUA_DRIVER_RS_ENABLE_WAYLAND",
+ value: environment.CUA_DRIVER_RS_ENABLE_WAYLAND ?? "1",
+ },
+ ]
+ : [],
+ };
+}
From e908498f213d16eb89579eb8e14ebea19731c422 Mon Sep 17 00:00:00 2001
From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com>
Date: Wed, 7 Oct 2026 16:03:37 -0700
Subject: [PATCH 2/8] fix(cua): preserve native Linux preview transport patch
---
patches/cua-driver-linux/0.34.0.patch | 997 ++++++++++++++++++++++++++
patches/cua-driver-linux/README.md | 33 +
2 files changed, 1030 insertions(+)
create mode 100644 patches/cua-driver-linux/0.34.0.patch
create mode 100644 patches/cua-driver-linux/README.md
diff --git a/patches/cua-driver-linux/0.34.0.patch b/patches/cua-driver-linux/0.34.0.patch
new file mode 100644
index 0000000000000..4ed81c76d7e1f
--- /dev/null
+++ b/patches/cua-driver-linux/0.34.0.patch
@@ -0,0 +1,997 @@
+diff --git a/libs/cua-driver/rust/crates/cua-driver-sdk/src/embedded.rs b/libs/cua-driver/rust/crates/cua-driver-sdk/src/embedded.rs
+index 1883e39..d599610 100644
+--- a/libs/cua-driver/rust/crates/cua-driver-sdk/src/embedded.rs
++++ b/libs/cua-driver/rust/crates/cua-driver-sdk/src/embedded.rs
+@@ -860,6 +860,7 @@ pub(crate) fn allowed_environment_name(name: &str) -> bool {
+ | "PROGRAMDATA"
+ | "DISPLAY"
+ | "WAYLAND_DISPLAY"
++ | "CUA_DRIVER_RS_ENABLE_WAYLAND"
+ | "XDG_RUNTIME_DIR"
+ | "XDG_SESSION_TYPE"
+ | "DBUS_SESSION_BUS_ADDRESS"
+@@ -1306,6 +1307,41 @@ mod tests {
+ .any(|variable| variable.value == "forged-lock"));
+ }
+
++ #[test]
++ fn wayland_backend_opt_in_reaches_embedded_driver() {
++ let name = "CUA_DRIVER_RS_ENABLE_WAYLAND";
++ assert!(allowed_environment_name(name));
++ let inherited = [(name.to_owned(), "1".to_owned())];
++ let values = merge_safe_environment(inherited.clone(), &[]);
++ assert!(values
++ .iter()
++ .any(|variable| variable.name == name && variable.value == "1"));
++
++ // Hosts can opt in without changing their own process environment.
++ let enabled = EmbeddedEnvironmentVariable {
++ name: name.into(),
++ value: "1".into(),
++ };
++ let values = merge_safe_environment(std::iter::empty(), &[enabled]);
++ assert!(values
++ .iter()
++ .any(|variable| variable.name == name && variable.value == "1"));
++
++ // An explicit opt-out still wins over the inherited opt-in.
++ let disabled = EmbeddedEnvironmentVariable {
++ name: name.into(),
++ value: "0".into(),
++ };
++ let values = merge_safe_environment(inherited, &[disabled]);
++ assert!(values
++ .iter()
++ .any(|variable| variable.name == name && variable.value == "0"));
++ assert!(!allowed_environment_name("LD_PRELOAD"));
++ assert!(!allowed_environment_name(
++ "CUA_DRIVER_DANGEROUSLY_BYPASS_APPROVALS"
++ ));
++ }
++
+ #[test]
+ fn interactive_linux_session_environment_is_inherited() {
+ let values = merge_safe_environment(
+diff --git a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs
+index 1d35cd5..06035e6 100644
+--- a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs
++++ b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs
+@@ -483,12 +483,51 @@ fn start_shell_helper_idle_watch() {
+ });
+ }
+
++// GNOME's helper cursor is a stage actor, not a child of the target window.
++// Until the helper can bind it to a surface, only desktop actions may draw it.
++#[cfg(target_os = "linux")]
++static SHELL_HELPER_DESKTOP_ACTIONS: Mutex
@@ -244,7 +254,9 @@ const ComposerCommandMenuItem = memo(function ComposerCommandMenuItem(props: {
{props.item.type === "thread" ? (
) : null}
- {props.item.type === "environment" ? (
+ {props.item.type === "application" ? (
+
+ ) : props.item.type === "environment" ? (
= {
- path: msg("Files and folders"),
+ path: msg("Files, threads, and applications"),
"pull-request": msg("Pull requests"),
"slash-command": msg("Commands"),
skill: msg("Skills"),
diff --git a/apps/web/src/components/chat/useComposerMenuItems.ts b/apps/web/src/components/chat/useComposerMenuItems.ts
index 330e3125d6855..81f6bd3f61284 100644
--- a/apps/web/src/components/chat/useComposerMenuItems.ts
+++ b/apps/web/src/components/chat/useComposerMenuItems.ts
@@ -34,6 +34,7 @@ import {
getProviderSlashCommandsForSlashMenu,
} from "@t3tools/client-runtime/providerSkills";
import { searchProviderSkills } from "../../providerSkillSearch";
+import { useCuaApplications } from "~/state/cua";
import { useVisibleThreadShells } from "~/state/entities";
import { useEnvironmentIdentities, useEnvironmentMachines } from "~/state/environments";
import { offersLinkedEnvironmentsAtom } from "~/state/peerLinks";
@@ -125,6 +126,7 @@ export function useComposerMenuItems({
const settledPullRequestTextQuery =
pullRequestTextQuery === debouncedPullRequestTextQuery ? pullRequestTextQuery : null;
const isPathTrigger = composerTriggerKind === "path";
+ const applications = useCuaApplications(isPathTrigger ? environmentId : null);
const environmentThreadShells = useVisibleThreadShells(isPathTrigger);
// Mounts older settled threads matching the mention, here and on linked machines, which shells
// may leave out.
@@ -211,7 +213,19 @@ export function useComposerMenuItems({
const items = ((): ComposerCommandItem[] => {
if (!composerTrigger) return [];
if (composerTrigger.kind === "path") {
+ const query = composerTrigger.query.toLocaleLowerCase();
return [
+ ...(applications.data ?? [])
+ .filter((app) => app.name.toLocaleLowerCase().includes(query))
+ .slice(0, 8)
+ .map((application) => ({
+ id: `application:${application.bundleId ?? application.launchPath ?? application.name}`,
+ type: "application" as const,
+ application,
+ environmentId,
+ label: application.name,
+ description: gt("Application"),
+ })),
...matchComposerThreadItems(
{
shells: environmentThreadShells,
@@ -413,6 +427,7 @@ export function useComposerMenuItems({
return [];
})();
const isLoading =
+ (isPathTrigger && applications.isPending) ||
(composerTriggerKind === "path" && workspaceEntries.isPending) ||
((composerTriggerKind === "skill" || composerTriggerKind === "slash-command") &&
workspaceSkillsScanning) ||
@@ -442,7 +457,7 @@ export function useComposerMenuItems({
: gt("No pull requests found in this repository.");
}
return composerTriggerKind === "path"
- ? gt("No matching files or folders.")
+ ? gt("No matching files, threads, or applications.")
: gt("No matching command.");
})();
diff --git a/apps/web/src/composer-editor-mentions.test.ts b/apps/web/src/composer-editor-mentions.test.ts
index f1305b83fbb11..459f47ff1b6f6 100644
--- a/apps/web/src/composer-editor-mentions.test.ts
+++ b/apps/web/src/composer-editor-mentions.test.ts
@@ -1,5 +1,6 @@
import { EnvironmentId, MessageId, ThreadId, type AssistantCitation } from "@t3tools/contracts";
import { serializeAssistantCitation } from "@t3tools/shared/assistantCitations";
+import { formatNativeAppMention } from "@t3tools/shared/nativeAppMentions";
import { describe, expect, it } from "vite-plus/test";
import {
@@ -8,6 +9,18 @@ import {
} from "./composer-editor-mentions";
import { formatTerminalContextReference } from "./lib/terminalContext";
+it("keeps a native application reference as one composer token", () => {
+ const source = formatNativeAppMention(
+ { name: "Helium", launchPath: "C:\\Apps\\Helium\\chrome.exe" },
+ "windows",
+ );
+ expect(splitPromptIntoComposerSegments(`use ${source} please`)).toEqual([
+ { type: "text", text: "use " },
+ { type: "application", href: source.slice(source.indexOf("](") + 2, -1), source },
+ { type: "text", text: " please" },
+ ]);
+});
+
const terminalReference = formatTerminalContextReference({
id: "ctx-1",
terminalLabel: "Terminal 1",
diff --git a/apps/web/src/composer-editor-mentions.ts b/apps/web/src/composer-editor-mentions.ts
index 7f3e93b821488..cee7c4e04bd98 100644
--- a/apps/web/src/composer-editor-mentions.ts
+++ b/apps/web/src/composer-editor-mentions.ts
@@ -1,5 +1,6 @@
import { collectComposerModelMentions } from "@t3tools/client-runtime/composer-model-mentions";
import type { AssistantCitation } from "@t3tools/contracts";
+import { collectNativeAppMentions } from "@t3tools/shared/nativeAppMentions";
import { collectAssistantCitations } from "@t3tools/shared/assistantCitations";
import { collectComposerContextReferences } from "@t3tools/shared/composerContextReferences";
import {
@@ -9,6 +10,7 @@ import {
} from "@t3tools/shared/composerInlineTokens";
export type ComposerPromptSegment =
+ | { type: "application"; href: string; source: string }
| {
type: "text";
text: string;
@@ -106,10 +108,12 @@ export function collectComposerPromptInlineTokens(text: string) {
: [...inlineTokens, ...models].sort((left, right) => left.start - right.start);
const citations = collectAssistantCitations(text);
const references = collectComposerContextReferences(text);
- if (citations.length === 0 && references.length === 0) return tokens;
+ const applications = collectNativeAppMentions(text);
+ if (citations.length === 0 && references.length === 0 && applications.length === 0) return tokens;
// An unfinished @ mention can otherwise consume the start of a link label.
const links = [
+ ...applications.map((match) => ({ ...match, type: "application" as const })),
...citations.map((match) => ({ ...match, type: "citation" as const })),
...references.map((match) => ({ ...match, type: "context-reference" as const })),
];
@@ -138,7 +142,9 @@ function splitPromptTextIntoComposerSegments(text: string): ComposerPromptSegmen
pushTextSegment(segments, text.slice(cursor, match.start));
}
- if (match.type === "citation") {
+ if (match.type === "application") {
+ segments.push({ type: "application", href: match.href, source: match.source });
+ } else if (match.type === "citation") {
segments.push({ type: "citation", citation: match.citation, source: match.source });
} else if (match.type === "context-reference") {
segments.push({
diff --git a/apps/web/src/composer-logic.test.ts b/apps/web/src/composer-logic.test.ts
index 8a714a4c572db..a8ef982f55b49 100644
--- a/apps/web/src/composer-logic.test.ts
+++ b/apps/web/src/composer-logic.test.ts
@@ -12,6 +12,7 @@ import {
mergeWithDefaultKeybindings,
} from "@t3tools/shared/keybindings";
import { describe, expect, it } from "vite-plus/test";
+import { formatNativeAppMention } from "@t3tools/shared/nativeAppMentions";
import {
clampCollapsedComposerCursor,
@@ -29,6 +30,12 @@ import { slashCommandItemsForPromptPosition } from "./components/chat/composerSl
import { carryDisplacedCustomAnswerIntoPrompt } from "./pendingUserInput";
import { formatTerminalContextReference } from "./lib/terminalContext";
+it("treats an application chip as one cursor position", () => {
+ const source = formatNativeAppMention({ name: "Helium" }, "host");
+ expect(collapseExpandedComposerCursor(`${source} x`, source.length)).toBe(1);
+ expect(expandCollapsedComposerCursor(`${source} x`, 1)).toBe(source.length);
+});
+
const terminalReference = formatTerminalContextReference({
id: "ctx-1",
terminalLabel: "Terminal 1",
diff --git a/apps/web/src/composer-logic.ts b/apps/web/src/composer-logic.ts
index 0a6de3de73249..abad6e0787476 100644
--- a/apps/web/src/composer-logic.ts
+++ b/apps/web/src/composer-logic.ts
@@ -138,6 +138,7 @@ export function expandCollapsedComposerCursor(
for (const segment of segments) {
if (
segment.type === "mention" ||
+ segment.type === "application" ||
segment.type === "citation" ||
segment.type === "context-reference"
) {
@@ -221,6 +222,7 @@ export function collapseExpandedComposerCursor(
for (const segment of segments) {
if (
segment.type === "mention" ||
+ segment.type === "application" ||
segment.type === "citation" ||
segment.type === "context-reference"
) {
diff --git a/apps/web/src/composer-rich-text-doc.ts b/apps/web/src/composer-rich-text-doc.ts
index 3c2b353f59b76..51984e2a4d00d 100644
--- a/apps/web/src/composer-rich-text-doc.ts
+++ b/apps/web/src/composer-rich-text-doc.ts
@@ -806,6 +806,9 @@ function atomJsonForSegment(
segment: Exclude[number], { type: "text" }>,
chips: ChipResolver,
): AtomJson | null {
+ if (segment.type === "application") {
+ return { type: "composer-application", attrs: { href: segment.href, source: segment.source } };
+ }
if (segment.type === "mention") {
return {
type: "composer-mention",
@@ -1207,6 +1210,7 @@ function readAtomSource(node: ProseMirrorNode): string {
switch (node.type.name) {
case "composer-mention":
case "composer-model":
+ case "composer-application":
case "composer-citation":
case "composer-context-reference":
return typeof attrs.source === "string" ? attrs.source : "";
diff --git a/apps/web/src/i18n/translations/de.json b/apps/web/src/i18n/translations/de.json
index 857d6ab51bf12..e8a9f995d0a20 100644
--- a/apps/web/src/i18n/translations/de.json
+++ b/apps/web/src/i18n/translations/de.json
@@ -2128,6 +2128,7 @@
" verwaltet."
],
"d71a78d5a0b8045d": "Hilfe erhalten",
+ "cc10829e6a2d7661": "App: {name}",
"b9e49ec03efc8383": "Kein aktiver Thread",
"fc7b51a6f2a86236": "Wähle einen Thread, um weiterzumachen",
"7543807a82fa9ccd": "Wähle einen vorhandenen Thread oder erstelle einen neuen, um loszulegen.",
@@ -3180,7 +3181,7 @@
"3d3f30f1a1c4fc52": "Frag einfach...",
"305d6d72f1220d98": "Pull Request wird gesucht...",
"cf855794f7b8ddb5": "Skills im Arbeitsbereich werden durchsucht...",
- "e159e35b83a86404": "Dateien im Arbeitsbereich werden durchsucht...",
+ "5b31a853afbdd6bc": "Dateien, Threads und Apps werden durchsucht...",
"c347652bd889e70f": "Nachricht an {title}",
"9bf7fa475fe78de5": "Nachricht",
"86ba20bb0ac0e5d0": "Sende Feedback, um diese Anfrage abzulehnen",
@@ -3295,7 +3296,7 @@
"a4f8a7db663bbc79": "Codesprache: {language}",
"249192edde7f4932": "Sprachen durchsuchen",
"71d8b593668d2106": "Keine passenden Sprachen.",
- "326bc17df66a7f05": "Dateien und Ordner",
+ "1fcbf1a95bcb4002": "Dateien, Threads und Apps",
"0e83c3bf28f8958e": "Befehle",
"75d18dac06f4f037": "Skills",
"05340d7dfcc34edf": "App",
@@ -3311,7 +3312,7 @@
"6640341b04ff6825": "Anbieter",
"e9d791bbaaa61fac": "Anbieter-Skill",
"061c0c9f27104d23": "Keine Skills gefunden. Mit / kannst du Anbieterbefehle durchsuchen.",
- "0a6fbecab03f7ef3": "Keine passenden Dateien oder Ordner.",
+ "4fcabb6aca677012": "Keine passenden Dateien, Threads oder Apps.",
"0e7b21ab11721ca7": "Kein passender Befehl.",
"2029db2cc30beb6d": "Anbietereinstellungen öffnen",
"517cfb8f24123f5b": "Kein Anbieter verfügbar",
@@ -4580,6 +4581,7 @@
"49c80af1dd4826f1": "Der Download aus der Quelle ist fehlgeschlagen.",
"5ba22a660758e453": "Das Limit für Anhänge ist erreicht.",
"181933312b23cb0c": "Der Entwurf hat diesen Anhang abgelehnt (Duplikat oder Limit für Anhänge erreicht).",
+ "c8c58bab29faa2e9": "App",
"df02c1d51d158d32": "Antwortmodell für diesen Thread wechseln",
"929781439a221aae": "Eine Nebenfrage in einem neuen Nebenchat stellen",
"378c3404c430fc05": "Wie /side",
diff --git a/apps/web/src/i18n/translations/en.json b/apps/web/src/i18n/translations/en.json
index fe65f35ff14d3..d33e1662441d7 100644
--- a/apps/web/src/i18n/translations/en.json
+++ b/apps/web/src/i18n/translations/en.json
@@ -2128,6 +2128,7 @@
"."
],
"d71a78d5a0b8045d": "Get help",
+ "cc10829e6a2d7661": "Application: {name}",
"b9e49ec03efc8383": "No active thread",
"fc7b51a6f2a86236": "Pick a thread to continue",
"7543807a82fa9ccd": "Select an existing thread or create a new one to get started.",
@@ -3179,7 +3180,7 @@
"3d3f30f1a1c4fc52": "Ask anything...",
"305d6d72f1220d98": "Finding pull request...",
"cf855794f7b8ddb5": "Searching workspace skills...",
- "e159e35b83a86404": "Searching workspace files...",
+ "5b31a853afbdd6bc": "Searching files, threads, and applications...",
"c347652bd889e70f": "Message {title}",
"9bf7fa475fe78de5": "Message",
"86ba20bb0ac0e5d0": "Send feedback to decline this request",
@@ -3294,7 +3295,7 @@
"a4f8a7db663bbc79": "Code language: {language}",
"249192edde7f4932": "Search languages",
"71d8b593668d2106": "No matching languages.",
- "326bc17df66a7f05": "Files and folders",
+ "1fcbf1a95bcb4002": "Files, threads, and applications",
"0e83c3bf28f8958e": "Commands",
"75d18dac06f4f037": "Skills",
"05340d7dfcc34edf": "App",
@@ -3310,7 +3311,7 @@
"6640341b04ff6825": "Provider",
"e9d791bbaaa61fac": "Provider Skill",
"061c0c9f27104d23": "No skills found. Try / to browse provider commands.",
- "0a6fbecab03f7ef3": "No matching files or folders.",
+ "4fcabb6aca677012": "No matching files, threads, or applications.",
"0e7b21ab11721ca7": "No matching command.",
"2029db2cc30beb6d": "Open provider settings",
"517cfb8f24123f5b": "No provider available",
@@ -4579,6 +4580,7 @@
"49c80af1dd4826f1": "Downloading it from the source failed.",
"5ba22a660758e453": "The attachment limit has been reached.",
"181933312b23cb0c": "The draft rejected this attachment (duplicate or attachment limit reached).",
+ "c8c58bab29faa2e9": "Application",
"df02c1d51d158d32": "Switch response model for this thread",
"929781439a221aae": "Ask a side question in a new side chat",
"378c3404c430fc05": "Same as /side",
diff --git a/apps/web/src/i18n/translations/es.json b/apps/web/src/i18n/translations/es.json
index 9ea91d27d1815..203feb167a4fe 100644
--- a/apps/web/src/i18n/translations/es.json
+++ b/apps/web/src/i18n/translations/es.json
@@ -2128,6 +2128,7 @@
"."
],
"d71a78d5a0b8045d": "Obtener ayuda",
+ "cc10829e6a2d7661": "App: {name}",
"b9e49ec03efc8383": "No hay ningún hilo activo",
"fc7b51a6f2a86236": "Elige un hilo para continuar",
"7543807a82fa9ccd": "Selecciona un hilo existente o crea uno nuevo para empezar.",
@@ -3179,7 +3180,7 @@
"3d3f30f1a1c4fc52": "Pregunta lo que quieras...",
"305d6d72f1220d98": "Buscando la pull request...",
"cf855794f7b8ddb5": "Buscando skills del espacio de trabajo...",
- "e159e35b83a86404": "Buscando archivos del espacio de trabajo...",
+ "5b31a853afbdd6bc": "Buscando archivos, hilos y apps...",
"c347652bd889e70f": "Mensaje para {title}",
"9bf7fa475fe78de5": "Mensaje",
"86ba20bb0ac0e5d0": "Envía comentarios para rechazar esta solicitud",
@@ -3294,7 +3295,7 @@
"a4f8a7db663bbc79": "Lenguaje del código: {language}",
"249192edde7f4932": "Buscar lenguajes",
"71d8b593668d2106": "No hay lenguajes que coincidan.",
- "326bc17df66a7f05": "Archivos y carpetas",
+ "1fcbf1a95bcb4002": "Archivos, hilos y apps",
"0e83c3bf28f8958e": "Comandos",
"75d18dac06f4f037": "Skills",
"05340d7dfcc34edf": "App",
@@ -3310,7 +3311,7 @@
"6640341b04ff6825": "Proveedor",
"e9d791bbaaa61fac": "Skill del proveedor",
"061c0c9f27104d23": "No se encontraron skills. Prueba con / para explorar los comandos del proveedor.",
- "0a6fbecab03f7ef3": "No hay archivos ni carpetas que coincidan.",
+ "4fcabb6aca677012": "No hay archivos, hilos ni apps que coincidan.",
"0e7b21ab11721ca7": "Ningún comando coincide.",
"2029db2cc30beb6d": "Abrir ajustes del proveedor",
"517cfb8f24123f5b": "No hay ningún proveedor disponible",
@@ -4577,6 +4578,7 @@
"49c80af1dd4826f1": "Falló la descarga desde el origen.",
"5ba22a660758e453": "Se alcanzó el límite de adjuntos.",
"181933312b23cb0c": "El borrador rechazó este adjunto (duplicado o límite de adjuntos alcanzado).",
+ "c8c58bab29faa2e9": "App",
"df02c1d51d158d32": "Cambiar el modelo de respuesta de este hilo",
"929781439a221aae": "Hacer una pregunta aparte en un chat lateral nuevo",
"378c3404c430fc05": "Igual que /side",
diff --git a/apps/web/src/i18n/translations/fr.json b/apps/web/src/i18n/translations/fr.json
index 39a4fe0f3ea39..85e21be0fa062 100644
--- a/apps/web/src/i18n/translations/fr.json
+++ b/apps/web/src/i18n/translations/fr.json
@@ -2128,6 +2128,7 @@
"."
],
"d71a78d5a0b8045d": "Obtenir de l’aide",
+ "cc10829e6a2d7661": "App : {name}",
"b9e49ec03efc8383": "Aucun fil actif",
"fc7b51a6f2a86236": "Choisissez un fil pour continuer",
"7543807a82fa9ccd": "Sélectionnez un fil existant ou créez-en un pour commencer.",
@@ -3179,7 +3180,7 @@
"3d3f30f1a1c4fc52": "Demandez ce que vous voulez...",
"305d6d72f1220d98": "Recherche de la pull request...",
"cf855794f7b8ddb5": "Recherche des skills de l’espace de travail...",
- "e159e35b83a86404": "Recherche des fichiers de l’espace de travail...",
+ "5b31a853afbdd6bc": "Recherche des fichiers, fils et apps...",
"c347652bd889e70f": "Message à {title}",
"9bf7fa475fe78de5": "Message",
"86ba20bb0ac0e5d0": "Envoyez un commentaire pour refuser cette demande",
@@ -3294,7 +3295,7 @@
"a4f8a7db663bbc79": "Langage du code : {language}",
"249192edde7f4932": "Rechercher un langage",
"71d8b593668d2106": "Aucun langage correspondant.",
- "326bc17df66a7f05": "Fichiers et dossiers",
+ "1fcbf1a95bcb4002": "Fichiers, fils et apps",
"0e83c3bf28f8958e": "Commandes",
"75d18dac06f4f037": "Skills",
"05340d7dfcc34edf": "App",
@@ -3310,7 +3311,7 @@
"6640341b04ff6825": "Fournisseur",
"e9d791bbaaa61fac": "Skill du fournisseur",
"061c0c9f27104d23": "Aucun skill trouvé. Essayez / pour parcourir les commandes du fournisseur.",
- "0a6fbecab03f7ef3": "Aucun fichier ni dossier correspondant.",
+ "4fcabb6aca677012": "Aucun fichier, fil ni app correspondant.",
"0e7b21ab11721ca7": "Aucune commande correspondante.",
"2029db2cc30beb6d": "Ouvrir les paramètres des fournisseurs",
"517cfb8f24123f5b": "Aucun fournisseur disponible",
@@ -4578,6 +4579,7 @@
"49c80af1dd4826f1": "Le téléchargement depuis la source a échoué.",
"5ba22a660758e453": "La limite de pièces jointes est atteinte.",
"181933312b23cb0c": "Le brouillon a refusé cette pièce jointe (doublon ou limite de pièces jointes atteinte).",
+ "c8c58bab29faa2e9": "App",
"df02c1d51d158d32": "Changer le modèle de réponse de ce fil",
"929781439a221aae": "Poser une question annexe dans une nouvelle discussion annexe",
"378c3404c430fc05": "Identique à /side",
diff --git a/apps/web/src/i18n/translations/ja.json b/apps/web/src/i18n/translations/ja.json
index c971e6b0d107a..a9ca6fdf99103 100644
--- a/apps/web/src/i18n/translations/ja.json
+++ b/apps/web/src/i18n/translations/ja.json
@@ -2126,6 +2126,7 @@
" によって管理されています。"
],
"d71a78d5a0b8045d": "ヘルプを表示",
+ "cc10829e6a2d7661": "アプリ: {name}",
"b9e49ec03efc8383": "アクティブなスレッドがありません",
"fc7b51a6f2a86236": "続けるスレッドを選択",
"7543807a82fa9ccd": "既存のスレッドを選択するか、新しいスレッドを作成して始めましょう。",
@@ -3177,7 +3178,7 @@
"3d3f30f1a1c4fc52": "何でも聞いてください...",
"305d6d72f1220d98": "プルリクエストを検索しています...",
"cf855794f7b8ddb5": "ワークスペースのスキルを検索しています...",
- "e159e35b83a86404": "ワークスペースのファイルを検索しています...",
+ "5b31a853afbdd6bc": "ファイル、スレッド、アプリを検索しています...",
"c347652bd889e70f": "{title} にメッセージ",
"9bf7fa475fe78de5": "メッセージ",
"86ba20bb0ac0e5d0": "このリクエストを拒否するにはフィードバックを送信してください",
@@ -3290,7 +3291,7 @@
"a4f8a7db663bbc79": "コードの言語: {language}",
"249192edde7f4932": "言語を検索",
"71d8b593668d2106": "一致する言語はありません。",
- "326bc17df66a7f05": "ファイルとフォルダー",
+ "1fcbf1a95bcb4002": "ファイル、スレッド、アプリ",
"0e83c3bf28f8958e": "コマンド",
"75d18dac06f4f037": "スキル",
"05340d7dfcc34edf": "アプリ",
@@ -3306,7 +3307,7 @@
"6640341b04ff6825": "プロバイダー",
"e9d791bbaaa61fac": "プロバイダーのスキル",
"061c0c9f27104d23": "スキルが見つかりません。/ を入力してプロバイダーのコマンドを参照してください。",
- "0a6fbecab03f7ef3": "一致するファイルやフォルダーはありません。",
+ "4fcabb6aca677012": "一致するファイル、スレッド、アプリはありません。",
"0e7b21ab11721ca7": "一致するコマンドはありません。",
"2029db2cc30beb6d": "プロバイダー設定を開く",
"517cfb8f24123f5b": "利用できるプロバイダーがありません",
@@ -4567,6 +4568,7 @@
"49c80af1dd4826f1": "元の場所からのダウンロードに失敗しました。",
"5ba22a660758e453": "添付ファイルの上限に達しました。",
"181933312b23cb0c": "下書きがこの添付ファイルを拒否しました (重複しているか、添付ファイルの上限に達しています)。",
+ "c8c58bab29faa2e9": "アプリ",
"df02c1d51d158d32": "このスレッドの応答モデルを切り替え",
"929781439a221aae": "新しいサイドチャットで別の質問をする",
"378c3404c430fc05": "/side と同じ",
diff --git a/apps/web/src/i18n/translations/ko.json b/apps/web/src/i18n/translations/ko.json
index 462622eb1e2e9..a03ebb8f38747 100644
--- a/apps/web/src/i18n/translations/ko.json
+++ b/apps/web/src/i18n/translations/ko.json
@@ -2127,6 +2127,7 @@
"에서 관리합니다."
],
"d71a78d5a0b8045d": "도움말 보기",
+ "cc10829e6a2d7661": "앱: {name}",
"b9e49ec03efc8383": "활성 스레드 없음",
"fc7b51a6f2a86236": "계속할 스레드를 선택하세요",
"7543807a82fa9ccd": "기존 스레드를 선택하거나 새 스레드를 만들어 시작하세요.",
@@ -3178,7 +3179,7 @@
"3d3f30f1a1c4fc52": "무엇이든 물어보세요...",
"305d6d72f1220d98": "풀 리퀘스트를 찾는 중...",
"cf855794f7b8ddb5": "작업 영역 스킬을 검색하는 중...",
- "e159e35b83a86404": "작업 영역 파일을 검색하는 중...",
+ "5b31a853afbdd6bc": "파일, 스레드, 앱을 검색하는 중...",
"c347652bd889e70f": "{title}에 메시지 보내기",
"9bf7fa475fe78de5": "메시지",
"86ba20bb0ac0e5d0": "이 요청을 거부하려면 피드백을 보내세요",
@@ -3290,7 +3291,7 @@
"a4f8a7db663bbc79": "코드 언어: {language}",
"249192edde7f4932": "언어 검색",
"71d8b593668d2106": "일치하는 언어가 없습니다.",
- "326bc17df66a7f05": "파일 및 폴더",
+ "1fcbf1a95bcb4002": "파일, 스레드 및 앱",
"0e83c3bf28f8958e": "명령",
"75d18dac06f4f037": "스킬",
"05340d7dfcc34edf": "앱",
@@ -3306,7 +3307,7 @@
"6640341b04ff6825": "제공자",
"e9d791bbaaa61fac": "제공자 스킬",
"061c0c9f27104d23": "스킬을 찾을 수 없습니다. 제공자 명령을 둘러보려면 /를 입력하세요.",
- "0a6fbecab03f7ef3": "일치하는 파일이나 폴더가 없습니다.",
+ "4fcabb6aca677012": "일치하는 파일, 스레드 또는 앱이 없습니다.",
"0e7b21ab11721ca7": "일치하는 명령이 없습니다.",
"2029db2cc30beb6d": "제공자 설정 열기",
"517cfb8f24123f5b": "사용 가능한 제공자 없음",
@@ -4564,6 +4565,7 @@
"49c80af1dd4826f1": "원본에서 다운로드하지 못했습니다.",
"5ba22a660758e453": "첨부 파일 한도에 도달했습니다.",
"181933312b23cb0c": "초안에서 이 첨부 파일을 거부했습니다(중복 또는 첨부 파일 한도 도달).",
+ "c8c58bab29faa2e9": "앱",
"df02c1d51d158d32": "이 스레드의 응답 모델 전환",
"929781439a221aae": "새 사이드 채팅에서 곁가지 질문하기",
"378c3404c430fc05": "/side와 같음",
diff --git a/apps/web/src/i18n/translations/pt-BR.json b/apps/web/src/i18n/translations/pt-BR.json
index 2aee29dc38f9d..ed49f68fe4199 100644
--- a/apps/web/src/i18n/translations/pt-BR.json
+++ b/apps/web/src/i18n/translations/pt-BR.json
@@ -2128,6 +2128,7 @@
"."
],
"d71a78d5a0b8045d": "Obter ajuda",
+ "cc10829e6a2d7661": "App: {name}",
"b9e49ec03efc8383": "Nenhuma thread ativa",
"fc7b51a6f2a86236": "Escolha uma thread para continuar",
"7543807a82fa9ccd": "Selecione uma thread existente ou crie uma nova para começar.",
@@ -3179,7 +3180,7 @@
"3d3f30f1a1c4fc52": "Pergunte qualquer coisa...",
"305d6d72f1220d98": "Procurando o pull request...",
"cf855794f7b8ddb5": "Pesquisando skills do workspace...",
- "e159e35b83a86404": "Pesquisando arquivos do workspace...",
+ "5b31a853afbdd6bc": "Pesquisando arquivos, threads e apps...",
"c347652bd889e70f": "Mensagem para {title}",
"9bf7fa475fe78de5": "Mensagem",
"86ba20bb0ac0e5d0": "Envie um feedback para recusar esta solicitação",
@@ -3294,7 +3295,7 @@
"a4f8a7db663bbc79": "Linguagem do código: {language}",
"249192edde7f4932": "Pesquisar linguagens",
"71d8b593668d2106": "Nenhuma linguagem correspondente.",
- "326bc17df66a7f05": "Arquivos e pastas",
+ "1fcbf1a95bcb4002": "Arquivos, threads e apps",
"0e83c3bf28f8958e": "Comandos",
"75d18dac06f4f037": "Skills",
"05340d7dfcc34edf": "App",
@@ -3310,7 +3311,7 @@
"6640341b04ff6825": "Provedor",
"e9d791bbaaa61fac": "Skill do provedor",
"061c0c9f27104d23": "Nenhuma skill encontrada. Use / para ver os comandos do provedor.",
- "0a6fbecab03f7ef3": "Nenhum arquivo ou pasta correspondente.",
+ "4fcabb6aca677012": "Nenhum arquivo, thread ou app correspondente.",
"0e7b21ab11721ca7": "Nenhum comando correspondente.",
"2029db2cc30beb6d": "Abrir configurações do provedor",
"517cfb8f24123f5b": "Nenhum provedor disponível",
@@ -4577,6 +4578,7 @@
"49c80af1dd4826f1": "Falha ao baixá-lo da origem.",
"5ba22a660758e453": "O limite de anexos foi atingido.",
"181933312b23cb0c": "O rascunho rejeitou este anexo (duplicado ou limite de anexos atingido).",
+ "c8c58bab29faa2e9": "App",
"df02c1d51d158d32": "Trocar o modelo de resposta desta thread",
"929781439a221aae": "Fazer uma pergunta paralela em um novo chat paralelo",
"378c3404c430fc05": "Igual a /side",
diff --git a/apps/web/src/i18n/translations/ru.json b/apps/web/src/i18n/translations/ru.json
index b7bcae501d913..f37ad87b97d2f 100644
--- a/apps/web/src/i18n/translations/ru.json
+++ b/apps/web/src/i18n/translations/ru.json
@@ -2128,6 +2128,7 @@
"."
],
"d71a78d5a0b8045d": "Получить помощь",
+ "cc10829e6a2d7661": "Приложение: {name}",
"b9e49ec03efc8383": "Нет активного треда",
"fc7b51a6f2a86236": "Выберите тред, чтобы продолжить",
"7543807a82fa9ccd": "Выберите существующий тред или создайте новый, чтобы начать.",
@@ -3179,7 +3180,7 @@
"3d3f30f1a1c4fc52": "Спросите что угодно...",
"305d6d72f1220d98": "Поиск пул-реквеста...",
"cf855794f7b8ddb5": "Поиск навыков рабочей области...",
- "e159e35b83a86404": "Поиск файлов рабочей области...",
+ "5b31a853afbdd6bc": "Поиск файлов, тредов и приложений...",
"c347652bd889e70f": "Сообщение для {title}",
"9bf7fa475fe78de5": "Сообщение",
"86ba20bb0ac0e5d0": "Отправьте отзыв, чтобы отклонить этот запрос",
@@ -3294,7 +3295,7 @@
"a4f8a7db663bbc79": "Язык кода: {language}",
"249192edde7f4932": "Поиск языков",
"71d8b593668d2106": "Нет подходящих языков.",
- "326bc17df66a7f05": "Файлы и папки",
+ "1fcbf1a95bcb4002": "Файлы, треды и приложения",
"0e83c3bf28f8958e": "Команды",
"75d18dac06f4f037": "Навыки",
"05340d7dfcc34edf": "Приложение",
@@ -3310,7 +3311,7 @@
"6640341b04ff6825": "Провайдер",
"e9d791bbaaa61fac": "Навык провайдера",
"061c0c9f27104d23": "Навыки не найдены. Введите /, чтобы просмотреть команды провайдера.",
- "0a6fbecab03f7ef3": "Нет подходящих файлов или папок.",
+ "4fcabb6aca677012": "Нет подходящих файлов, тредов или приложений.",
"0e7b21ab11721ca7": "Нет подходящей команды.",
"2029db2cc30beb6d": "Открыть настройки провайдера",
"517cfb8f24123f5b": "Нет доступного провайдера",
@@ -4577,6 +4578,7 @@
"49c80af1dd4826f1": "Не удалось скачать его из источника.",
"5ba22a660758e453": "Достигнут лимит вложений.",
"181933312b23cb0c": "Черновик отклонил это вложение (дубликат или достигнут лимит вложений).",
+ "c8c58bab29faa2e9": "Приложение",
"df02c1d51d158d32": "Сменить модель ответа для этого треда",
"929781439a221aae": "Задать побочный вопрос в новом боковом чате",
"378c3404c430fc05": "То же, что /side",
diff --git a/apps/web/src/i18n/translations/sv.json b/apps/web/src/i18n/translations/sv.json
index 867c1f3d6481d..0ec033a447a03 100644
--- a/apps/web/src/i18n/translations/sv.json
+++ b/apps/web/src/i18n/translations/sv.json
@@ -2128,6 +2128,7 @@
"."
],
"d71a78d5a0b8045d": "Få hjälp",
+ "cc10829e6a2d7661": "App: {name}",
"b9e49ec03efc8383": "Ingen aktiv tråd",
"fc7b51a6f2a86236": "Välj en tråd att fortsätta i",
"7543807a82fa9ccd": "Välj en befintlig tråd eller skapa en ny för att komma igång.",
@@ -3179,7 +3180,7 @@
"3d3f30f1a1c4fc52": "Fråga vad som helst...",
"305d6d72f1220d98": "Hittar pull request...",
"cf855794f7b8ddb5": "Söker färdigheter i arbetsytan...",
- "e159e35b83a86404": "Söker filer i arbetsytan...",
+ "5b31a853afbdd6bc": "Söker filer, trådar och appar...",
"c347652bd889e70f": "Meddelande till {title}",
"9bf7fa475fe78de5": "Meddelande",
"86ba20bb0ac0e5d0": "Skicka feedback för att avböja den här begäran",
@@ -3294,7 +3295,7 @@
"a4f8a7db663bbc79": "Kodspråk: {language}",
"249192edde7f4932": "Sök språk",
"71d8b593668d2106": "Inga matchande språk.",
- "326bc17df66a7f05": "Filer och mappar",
+ "1fcbf1a95bcb4002": "Filer, trådar och appar",
"0e83c3bf28f8958e": "Kommandon",
"75d18dac06f4f037": "Färdigheter",
"05340d7dfcc34edf": "App",
@@ -3310,7 +3311,7 @@
"6640341b04ff6825": "Leverantör",
"e9d791bbaaa61fac": "Leverantörsfärdighet",
"061c0c9f27104d23": "Inga färdigheter hittades. Prova / för att bläddra bland leverantörens kommandon.",
- "0a6fbecab03f7ef3": "Inga matchande filer eller mappar.",
+ "4fcabb6aca677012": "Inga matchande filer, trådar eller appar.",
"0e7b21ab11721ca7": "Inget matchande kommando.",
"2029db2cc30beb6d": "Öppna leverantörsinställningar",
"517cfb8f24123f5b": "Ingen leverantör tillgänglig",
@@ -4578,6 +4579,7 @@
"49c80af1dd4826f1": "Det gick inte att ladda ned den från källan.",
"5ba22a660758e453": "Gränsen för bilagor har nåtts.",
"181933312b23cb0c": "Utkastet avvisade den här bilagan (dubblett eller gränsen för bilagor nådd).",
+ "c8c58bab29faa2e9": "App",
"df02c1d51d158d32": "Byt svarsmodell för den här tråden",
"929781439a221aae": "Ställ en sidofråga i en ny sidochatt",
"378c3404c430fc05": "Samma som /side",
diff --git a/apps/web/src/i18n/translations/zh.json b/apps/web/src/i18n/translations/zh.json
index 9bf50aa1f0fc8..9889d0b97720b 100644
--- a/apps/web/src/i18n/translations/zh.json
+++ b/apps/web/src/i18n/translations/zh.json
@@ -2132,6 +2132,7 @@
" 管理。"
],
"d71a78d5a0b8045d": "获取帮助",
+ "cc10829e6a2d7661": "应用:{name}",
"b9e49ec03efc8383": "没有打开的会话",
"fc7b51a6f2a86236": "选择一个会话以继续",
"7543807a82fa9ccd": "选择已有会话,或新建一个会话开始。",
@@ -3183,7 +3184,7 @@
"3d3f30f1a1c4fc52": "随便问点什么…",
"305d6d72f1220d98": "正在查找 pull request…",
"cf855794f7b8ddb5": "正在搜索工作区技能…",
- "e159e35b83a86404": "正在搜索工作区文件…",
+ "5b31a853afbdd6bc": "正在搜索文件、会话和应用…",
"c347652bd889e70f": "给 {title} 发消息",
"9bf7fa475fe78de5": "消息",
"86ba20bb0ac0e5d0": "发送反馈以拒绝这个请求",
@@ -3299,7 +3300,7 @@
"a4f8a7db663bbc79": "代码语言:{language}",
"249192edde7f4932": "搜索语言",
"71d8b593668d2106": "没有匹配的语言。",
- "326bc17df66a7f05": "文件和文件夹",
+ "1fcbf1a95bcb4002": "文件、会话和应用",
"0e83c3bf28f8958e": "命令",
"75d18dac06f4f037": "技能",
"05340d7dfcc34edf": "应用",
@@ -3315,7 +3316,7 @@
"6640341b04ff6825": "提供方",
"e9d791bbaaa61fac": "提供方技能",
"061c0c9f27104d23": "未找到技能。试试输入 / 浏览提供方命令。",
- "0a6fbecab03f7ef3": "没有匹配的文件或文件夹。",
+ "4fcabb6aca677012": "没有匹配的文件、会话或应用。",
"0e7b21ab11721ca7": "没有匹配的命令。",
"2029db2cc30beb6d": "打开提供方设置",
"517cfb8f24123f5b": "没有可用的提供方",
@@ -4584,6 +4585,7 @@
"49c80af1dd4826f1": "从来源下载失败。",
"5ba22a660758e453": "已达到附件数量上限。",
"181933312b23cb0c": "草稿拒绝了这个附件(重复或已达到附件上限)。",
+ "c8c58bab29faa2e9": "应用",
"df02c1d51d158d32": "切换这个会话使用的回复模型",
"929781439a221aae": "在新的侧聊中问一个旁支问题",
"378c3404c430fc05": "与 /side 相同",
diff --git a/apps/web/src/state/cua.ts b/apps/web/src/state/cua.ts
index 7ebe5dbd6b519..0c4f073432166 100644
--- a/apps/web/src/state/cua.ts
+++ b/apps/web/src/state/cua.ts
@@ -1,5 +1,5 @@
import { createCuaEnvironmentAtoms } from "@t3tools/client-runtime/state/cua";
-import type { CuaWindowPreviewState, ScopedThreadRef } from "@t3tools/contracts";
+import type { CuaWindowPreviewState, ScopedThreadRef, EnvironmentId } from "@t3tools/contracts";
import { connectionAtomRuntime } from "../connection/runtime";
import { useEnvironmentQuery } from "./query";
@@ -8,6 +8,12 @@ const cuaEnvironment = createCuaEnvironmentAtoms(connectionAtomRuntime);
const IDLE: CuaWindowPreviewState = { status: "idle" };
+export function useCuaApplications(environmentId: EnvironmentId | null) {
+ return useEnvironmentQuery(
+ environmentId === null ? null : cuaEnvironment.applications({ environmentId, input: {} }),
+ );
+}
+
/** Subscribes while mounted; the server captures only for mounted subscribers. */
export function useCuaWindowPreview(threadRef: ScopedThreadRef | null): CuaWindowPreviewState {
const query = useEnvironmentQuery(
diff --git a/docs/user/computer-use.md b/docs/user/computer-use.md
index 44e3cc87a67a7..6c19fd1fb0de9 100644
--- a/docs/user/computer-use.md
+++ b/docs/user/computer-use.md
@@ -28,7 +28,7 @@ What the host needs depends on its OS:
at the computer may need to approve a screen capture prompt.
The desktop app includes Cua Driver. A server started with `npx t3` downloads
-the same version into T3 Code's home the first time an agent session needs it.
+the same version into T3 Code's home the first time computer use needs it.
To use your own build instead, set `T3CODE_CUA_DRIVER_PATH` to its executable
before starting the server.
@@ -37,6 +37,11 @@ keeps yours and T3 Code does not add a second one.
## Watch the agent work
+In the web or desktop composer, type `@` followed by an app name, such as
+`@Helium`, and select it to tell the agent which app to use. With Computer use
+enabled, suggestions include apps on the selected environment's computer,
+including when you connect remotely. Selecting an app does not launch it.
+
Each action an agent takes shows in the thread with the app's icon and a plain
title, such as **Clicked in Safari**.
diff --git a/packages/client-runtime/src/state/cua.ts b/packages/client-runtime/src/state/cua.ts
index 8ac30275f63ac..52b19d78671cc 100644
--- a/packages/client-runtime/src/state/cua.ts
+++ b/packages/client-runtime/src/state/cua.ts
@@ -2,12 +2,21 @@ import { WS_METHODS } from "@t3tools/contracts";
import { Atom } from "effect/reactivity";
import type { EnvironmentRegistry } from "../connection/registry.ts";
-import { createEnvironmentRpcSubscriptionAtomFamily } from "./runtime.ts";
+import {
+ createEnvironmentRpcSubscriptionAtomFamily,
+ createEnvironmentRpcQueryAtomFamily,
+} from "./runtime.ts";
export function createCuaEnvironmentAtoms(
runtime: Atom.AtomRuntime,
) {
return {
+ applications: createEnvironmentRpcQueryAtomFamily(runtime, {
+ label: "environment-data:cua:applications",
+ tag: WS_METHODS.serverListApplications,
+ staleTimeMs: 30_000,
+ idleTtlMs: 30_000,
+ }),
/** Live frames of the window the agent is driving; the server captures only while subscribed. */
windowPreview: createEnvironmentRpcSubscriptionAtomFamily(runtime, {
label: "environment-data:cua:window-preview",
diff --git a/packages/contracts/src/cua.ts b/packages/contracts/src/cua.ts
index 3d23ae59d3dac..ee73f5e9c7c6f 100644
--- a/packages/contracts/src/cua.ts
+++ b/packages/contracts/src/cua.ts
@@ -3,6 +3,23 @@ import * as Schema from "effect/Schema";
import { ThreadId, TrimmedNonEmptyString } from "./baseSchemas.ts";
import { ToolActivityNativeAppReference } from "./providerRuntime.ts";
+export const CuaApplication = Schema.Struct({
+ name: TrimmedNonEmptyString.check(Schema.isMaxLength(160)),
+ bundleId: Schema.optionalKey(TrimmedNonEmptyString.check(Schema.isMaxLength(512))),
+ launchPath: Schema.optionalKey(TrimmedNonEmptyString.check(Schema.isMaxLength(2048))),
+ pid: Schema.optionalKey(Schema.Int.check(Schema.isGreaterThan(0))),
+});
+export type CuaApplication = typeof CuaApplication.Type;
+
+export class CuaApplicationsError extends Schema.TaggedError()(
+ "CuaApplicationsError",
+ { cause: Schema.Defect() },
+) {
+ override get message() {
+ return "Could not list applications on this computer.";
+ }
+}
+
/** Native host descriptor, confined to the desktop/server control pipe. */
export const CuaDriverMcpConfiguration = Schema.Struct({
command: TrimmedNonEmptyString,
diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts
index 61298138b5bf1..4e20cfbb520c5 100644
--- a/packages/contracts/src/rpc.ts
+++ b/packages/contracts/src/rpc.ts
@@ -342,6 +342,8 @@ import {
BrowserTabsError,
BrowserTabsStatus,
CuaHostStatus,
+ CuaApplication,
+ CuaApplicationsError,
CuaWindowPreviewState,
CuaWindowPreviewSubscribeInput,
} from "./cua.ts";
@@ -541,6 +543,7 @@ export const WS_METHODS = {
serverGetConfig: "server.getConfig",
serverGetNpmRelease: "server.getNpmRelease",
serverCheckComputerUse: "server.checkComputerUse",
+ serverListApplications: "server.listApplications",
serverCheckBrowserTabs: "server.checkBrowserTabs",
serverRunBrowserTabsAction: "server.runBrowserTabsAction",
serverRefreshProviders: "server.refreshProviders",
@@ -706,6 +709,12 @@ const WsServerCheckComputerUseRpc = Rpc.make(WS_METHODS.serverCheckComputerUse,
error: EnvironmentAuthorizationError,
});
+const WsServerListApplicationsRpc = Rpc.make(WS_METHODS.serverListApplications, {
+ payload: Schema.Struct({}),
+ success: Schema.Array(CuaApplication),
+ error: Schema.Union([CuaApplicationsError, EnvironmentAuthorizationError]),
+});
+
const WsServerCheckBrowserTabsRpc = Rpc.make(WS_METHODS.serverCheckBrowserTabs, {
payload: Schema.Struct({}),
success: BrowserTabsStatus,
@@ -2300,6 +2309,7 @@ export const WsRpcGroup = RpcGroup.make(
WsServerProbeRpc,
WsServerGetConfigRpc,
WsServerCheckComputerUseRpc,
+ WsServerListApplicationsRpc,
WsServerCheckBrowserTabsRpc,
WsServerRunBrowserTabsActionRpc,
WsServerRefreshProvidersRpc,
diff --git a/packages/shared/package.json b/packages/shared/package.json
index 7f5ad87d7b196..d6cd3c9916459 100644
--- a/packages/shared/package.json
+++ b/packages/shared/package.json
@@ -15,6 +15,10 @@
"types": "./src/changedFileKind.ts",
"import": "./src/changedFileKind.ts"
},
+ "./nativeAppMentions": {
+ "types": "./src/nativeAppMentions.ts",
+ "import": "./src/nativeAppMentions.ts"
+ },
"./authScopeOptions": {
"types": "./src/authScopeOptions.ts",
"import": "./src/authScopeOptions.ts"
diff --git a/packages/shared/src/composerContextReferences.ts b/packages/shared/src/composerContextReferences.ts
index a6c2fc914b666..fef51c8e027e8 100644
--- a/packages/shared/src/composerContextReferences.ts
+++ b/packages/shared/src/composerContextReferences.ts
@@ -9,6 +9,7 @@ import {
type ScopedThreadRef,
type ThreadContextRecord,
} from "@t3tools/contracts";
+import { projectNativeAppMentions } from "./nativeAppMentions.ts";
import { expandAssistantCitationsForProvider } from "./assistantCitations.ts";
import { replaceThreadLinks, threadLinkCandidates } from "./threadLinks.ts";
@@ -395,14 +396,15 @@ export function projectComposerContextForProvider(input: {
text: string;
records: ReadonlyArray;
}): string {
- const occurrences = collectComposerContextReferences(input.text);
- if (occurrences.length === 0) return expandAssistantCitationsForProvider(input.text);
+ const text = projectNativeAppMentions(input.text);
+ const occurrences = collectComposerContextReferences(text);
+ if (occurrences.length === 0) return expandAssistantCitationsForProvider(text);
const recordsById = new Map();
for (const record of input.records) {
recordsById.set(record.contextId, recordsById.has(record.contextId) ? undefined : record);
}
const body = expandAssistantCitationsForProvider(
- replaceComposerContextReferences(input.text, (occurrence) =>
+ replaceComposerContextReferences(text, (occurrence) =>
formatComposerContextProviderMarker(
recordsById.get(occurrence.contextId)?.kind ?? occurrence.kind,
occurrence.label,
diff --git a/packages/shared/src/nativeAppMentions.test.ts b/packages/shared/src/nativeAppMentions.test.ts
new file mode 100644
index 0000000000000..80a04eaaf0ae6
--- /dev/null
+++ b/packages/shared/src/nativeAppMentions.test.ts
@@ -0,0 +1,43 @@
+import { describe, expect, it } from "vite-plus/test";
+import {
+ formatNativeAppMention,
+ parseNativeAppMention,
+ collectNativeAppMentions,
+ projectNativeAppMentions,
+} from "./nativeAppMentions.ts";
+import { projectComposerContextForProvider } from "./composerContextReferences.ts";
+
+describe("native application mentions", () => {
+ it.each([
+ { name: "Helium", bundleId: "net.imput.helium", launchPath: "/Applications/Helium.app" },
+ {
+ name: "Calculator",
+ bundleId: "Microsoft.WindowsCalculator_8wekyb3d8bbwe!App",
+ launchPath: "shell:AppsFolder\\Microsoft.WindowsCalculator_8wekyb3d8bbwe!App",
+ },
+ {
+ name: "Helium Browser",
+ bundleId: "helium.desktop",
+ launchPath: "/usr/share/applications/helium.desktop",
+ },
+ ])("round-trips $name with its host identity", (app) => {
+ const text = formatNativeAppMention(app, "desktop");
+ const mentions = collectNativeAppMentions(`Use ${text} please`);
+ expect(mentions).toHaveLength(1);
+ expect(mentions[0]).toMatchObject({ app, environmentId: "desktop" });
+ expect(parseNativeAppMention(mentions[0]!.href)?.app).toEqual(app);
+ const projected = projectComposerContextForProvider({ text, records: [] });
+ expect(projected).toContain(`bundle_id: ${app.bundleId}`);
+ expect(projected).toContain("environment: desktop");
+ expect(projected).not.toContain("t3-app://");
+ });
+ it("keeps malformed and unrelated links unchanged", () => {
+ expect(parseNativeAppMention("https://example.com")).toBeNull();
+ expect(parseNativeAppMention("t3-app://v1?name=Helium")).toBeNull();
+ expect(projectNativeAppMentions("[bad](t3-app://v1?name=)")).toBe("[bad](t3-app://v1?name=)");
+ });
+ it("preserves paths containing spaces and Markdown punctuation", () => {
+ const app = { name: "Example", launchPath: "C:\\Apps\\Example (Dev)\\app.exe" };
+ expect(collectNativeAppMentions(formatNativeAppMention(app, "host"))[0]?.app).toEqual(app);
+ });
+});
diff --git a/packages/shared/src/nativeAppMentions.ts b/packages/shared/src/nativeAppMentions.ts
new file mode 100644
index 0000000000000..3f98920464657
--- /dev/null
+++ b/packages/shared/src/nativeAppMentions.ts
@@ -0,0 +1,63 @@
+import { CuaApplication } from "@t3tools/contracts";
+import * as Schema from "effect/Schema";
+
+const PREFIX = "t3-app://v1?";
+const isApplication = Schema.is(CuaApplication);
+const LINKS = /\[([^\]\n]{1,200})\]\((t3-app:\/\/v1\?[^\s)]{1,8192})\)/gu;
+
+export function formatNativeAppMention(app: CuaApplication, environmentId: string): string {
+ const params = new URLSearchParams({ name: app.name, environment: environmentId });
+ if (app.bundleId) params.set("bundleId", app.bundleId);
+ if (app.launchPath) params.set("launchPath", app.launchPath);
+ const label = app.name.replace(/[[\]\\\r\n]/gu, " ");
+ return `[${label}](${PREFIX}${params})`;
+}
+
+export function parseNativeAppMention(href: string) {
+ if (!href.startsWith(PREFIX) || href.length > 8192) return null;
+ const params = new URLSearchParams(href.slice(PREFIX.length));
+ const app = {
+ name: params.get("name"),
+ ...(params.has("bundleId") ? { bundleId: params.get("bundleId") } : {}),
+ ...(params.has("launchPath") ? { launchPath: params.get("launchPath") } : {}),
+ };
+ const environmentId = params.get("environment");
+ return isApplication(app) && environmentId && environmentId.length <= 200
+ ? { app, environmentId }
+ : null;
+}
+
+export function collectNativeAppMentions(text: string) {
+ return Array.from(text.matchAll(LINKS)).flatMap((match) => {
+ const parsed = parseNativeAppMention(match[2]!);
+ return parsed
+ ? [
+ {
+ ...parsed,
+ source: match[0],
+ href: match[2]!,
+ start: match.index,
+ end: match.index + match[0].length,
+ },
+ ]
+ : [];
+ });
+}
+
+export function projectNativeAppMentions(text: string): string {
+ const mentions = collectNativeAppMentions(text);
+ if (!mentions.length) return text;
+ let result = text;
+ for (let index = mentions.length - 1; index >= 0; index--) {
+ const mention = mentions[index]!;
+ const fields = [
+ `Application: ${mention.app.name}`,
+ `environment: ${mention.environmentId}`,
+ ...(mention.app.bundleId ? [`bundle_id: ${mention.app.bundleId}`] : []),
+ ...(mention.app.launchPath ? [`launch_path: ${mention.app.launchPath}`] : []),
+ "Use the computer-use tools on this environment; resolve the app's current window before interacting.",
+ ];
+ result = result.slice(0, mention.start) + fields.join("\n") + result.slice(mention.end);
+ }
+ return result;
+}