From a6274721ee705caedb324b40e2ac3aca3e932bbb Mon Sep 17 00:00:00 2001 From: Badcuban <108198679+badcuban@users.noreply.github.com> Date: Fri, 20 Nov 2026 00:32:53 -0500 Subject: [PATCH 01/49] docs(devices): design and contracts for the Device panel Device panel design (docs/design/devices.md), ported from T3 Code's Device panel without its cloud, pool and SSH hosts. Contracts: device schemas, the device channel over WS RPC, a WsDeviceRpcGroup, enableDevices and agentDeviceAccess settings, and the deviceTools session flag. --- docs/design/devices.md | 243 ++++++++++++ packages/contracts/src/device.ts | 603 +++++++++++++++++++++++++++++ packages/contracts/src/index.ts | 1 + packages/contracts/src/provider.ts | 7 + packages/contracts/src/rpc.ts | 127 ++++++ packages/contracts/src/settings.ts | 15 + 6 files changed, 996 insertions(+) create mode 100644 docs/design/devices.md create mode 100644 packages/contracts/src/device.ts diff --git a/docs/design/devices.md b/docs/design/devices.md new file mode 100644 index 000000000..fbbe9e7f8 --- /dev/null +++ b/docs/design/devices.md @@ -0,0 +1,243 @@ +# Devices + +A Device tab in the right panel shows a live iOS Simulator or Android Emulator +running on the thread's computer. The user taps, types and presses hardware +buttons on it; agents drive the same device through four `device_*` tools and +the `agent-device` command line. Ported from T3 Code's Device panel (MIT, see +NOTICE; their `docs/internals/devices.md` is the reference for the parts kept +as they are). + +## Scope + +In: + +- iOS Simulators (macOS with Xcode) and Android Emulators (Android SDK) on the + machine that runs the Threadlines server. A Mac linked in Fleet shows its + devices to whichever computer opens a thread there. +- The Device tab: device picker (start, stop), the live screen with touch, + keyboard and paste, a toolbar (Home, app switcher, Back and Recents on + Android, rotate, screenshot, power off), and a Tools drawer (appearance, text + size, accessibility toggles, location, permissions, open URL, launch and quit + app, shake, test push, network on Android) that reads every value back. +- Agent access: `device_list`, `device_open`, `device_screenshot`, + `device_close`, plus `agent-device` on the agent's PATH. An agent opening a + device brings up the Device tab for anyone watching that thread. +- Settings → Devices: "Use devices" and "Agents can use devices", tool versions + with an update button. App guide place and topic. +- Works in the desktop app, the web app, the phone layout and on saved + computers over the relay. + +Out, on purpose: + +- Expo cloud simulators (billed per minute), Simlock pools (alpha service), + SSH device hosts. Linking the Mac in Fleet covers the SSH case. +- T3's 3D device models, iPhone Duo and foldable posture views. The screen sits + in a flat rounded frame drawn in CSS, which matches the design language and + drops about 6,000 lines and a three.js dependency. +- WebRTC streaming (only Expo cloud needs it). + +## Architecture + +``` +web Device tab ──WS RPC──► DeviceService ──► DeviceHost (local) + │ device.* RPCs │ ├─ expo-device-hub (child, 127.0.0.1:) + │ deviceChannel.* stream ──┘ allowlist ─────►│ serve-sim (iOS) / serve-emu (Android) + │ └─ agent-device daemon (child, only with agent access) +agent ──MCP device_*──► DeviceService +agent ──agent-device CLI (shim on PATH) ──► agent-device daemon +``` + +### Server owns devices + +Devices belong to a machine, not to a window, so `apps/server/src/device/` +owns discovery, boot, shutdown, streaming and agent access, the way it owns +terminals. Ported from T3's `apps/server/src/device/` with the EAS, Simlock and +SSH hosts removed: + +- `DeviceToolchain.ts`: installs `expo-device-hub` and `agent-device` at the + versions in `deviceToolVersions.json` into `/tools/device//` + (staged, then published with a `.install-complete` marker), only after the + matching consent: the hub on "Use devices", agent-device on "Agents can use + devices". Reuse our pinned-npm installer (`provider/acpRegistry/AcpRegistryInstaller.ts`, + `npmPinnedFlags`) and managed Node where they fit instead of copying T3's + `npm install --prefix`. Install scripts stay on: the hub's native addons + need them. Runs with the server's Node; never `npx`. Keeps the last two + versions (`deviceToolMaintenance.ts`). +- `DeviceHost.ts`: the seam. One implementation, `LocalDeviceHost.ts`. +- `LocalDeviceHost.ts`: platform checks (macOS + `xcrun` for iOS; full Android + SDK for Android), spawns the hub as a supervised child on a reserved loopback + port with `--host 127.0.0.1`, restarts it with backoff, records + `{pid, port, entryPath}` in `/device/hub.json` so the next server + stops a hub this one left behind (checked by pid AND command line, never by + name), and the same for the agent-device daemon. +- `DeviceHubOperations.ts`: list, boot, shutdown, screenshot through the hub. +- `DeviceService.ts`: state (`DeviceServiceState`), sessions per thread, + open/close/shutdown, `revision` for subscribers, stop the hub and daemon on + server shutdown. +- Claims survive a crash. `/device/claims.json` (atomic write) + records each open `{threadId, deviceId, platform, agentSession, + bootedByThreadlines, openedAt}` before it is reported open, and drops it on + close. On startup the service reloads claims, releases the agent-device + session of every claim whose thread was archived, deleted or moved while the + server was down (sweep like `secrets/AgentSecretRequests.ts`), and keeps the + rest. A reactor releases claims on `thread.archived`, `thread.move-started` + and `thread.deleted`. Closing never shuts a simulator down unless asked + (`shutdown: true` or the power button), the same as T3: a simulator the user + booted is theirs. `bootedByThreadlines` is shown in the picker so the user + can tell which ones Threadlines started. +- agent-device's daemon is started by its CLI, so the pid we spawn is the + launcher's. It is stopped through the CLI's own stop command with our + isolated state directory (T3's `LocalDeviceHost` daemon stop), never by pid + guess or name. +- `DeviceActions.ts`: the Tools drawer. Runs `simctl`, `adb` and serve-sim's + helper binaries directly, one typed action per control. Nothing here goes + through the hub's exec route. +- `AgentDeviceShim.ts`, `AgentDeviceTarget.ts`: the agent-device launcher. + +### The hub is never reachable from a client + +serve-sim has a shell-exec route whose token is readable from its own +unauthenticated `/api`; serve-emu's action routes have no auth. So: + +1. The hub binds 127.0.0.1 only. +2. **The hub runs with `--require-token`.** Every hub route except its health + and ready checks then refuses requests without the session token (bearer, + cookie or WS subprotocol; see `gateRequest`/`gatedSocket` in the hub's + `dist/server/index.mjs`). The hub makes the token and prints it once in its + startup line (`Local: http://localhost:/?token=`); the server + reads it from the child's stdout, keeps it in memory only, and adds + `Authorization: Bearer` on every request it makes. A test pins that line's + format for the pinned version. This is what protects the hub from a second + Threadlines server on the same Mac (desktop app plus a dev server), an + older release, or anything else on loopback: a port denylist in one server + cannot. +3. **`PreviewTunnel` refuses ports held by device tools** as well as its own: + any listener whose process is the hub, a serve-sim/serve-emu helper or an + agent-device daemon, found through the listening-port scan + (`preview/parseListeningPorts.ts`, `LocalServers.ts`) by pid tree and by + command line, so it also covers another server's tools. Refusing by name is + fine here; killing by name is not. The builder checks whether serve-sim's + per-device helpers listen on ports of their own (they would carry touch + input without the hub's token) and makes sure this check covers them. +4. Clients reach the hub only through the device channel below, which allows + T3's stream and read paths and drops cookies and auth headers. **Not + `/vendor/serve-sim/api` or `/vendor/serve-emu/api`**: serve-sim's body + carries the exec token. The viewer read them only to detect WebRTC-only + hubs (Expo cloud), which we do not run, so the probe is removed. + +Agents on the same machine can reach loopback anyway; they already have a +shell there, so this adds nothing they lack. + +### Transport: the device channel over WS RPC + +T3 proxies the hub over HTTP routes with cookies and WS tickets. Ours cannot: +saved computers and the relay carry only `/ws` and `/peer`, and a browser has +no cookie for another computer's origin. So every byte between the Device tab +and the hub rides the client's existing WS RPC connection, modeled on +`PreviewTunnel` (flow-controlled, base64 chunks, closed by unsubscribing): + +- `deviceChannelOpen` (stream): `{channelId, kind: "http", method, path, query, body?}` + or `{channelId, kind: "ws", path, query}`. The server checks the path against + the allowlist, opens the request or socket to the hub, and streams + `opened {status, contentType}` / `data {data}` (http body chunks) / + `message {data, binary}` (ws messages) / `end`. +- `deviceChannelSend`: one ws message from the client, in order (`seq`). +- `deviceChannelCredit`: bytes delivered, so a stalled tab holds the hub up + instead of filling server memory. + +Flow control, so a suspended phone cannot pile up video in the server: + +- Each `data`/`message` event carries at most 256 KiB before base64; a larger + ws message is split into parts with `final: false` until the last. +- The server keeps at most 2 MiB sent but not credited per channel. For an + http body it stops reading (the hub's socket backs up). For a ws upstream it + pauses the socket; if 4 MiB more queue up anyway, the channel fails with + `slow` and the client reopens. +- The client credits a chunk once the decoder has taken it, not on arrival. +- Channels never resubscribe. On any failure or reconnect the client opens a + fresh channel and a fresh decoder that waits for a keyframe, and drops input + it had not sent. A pre-disconnect gesture never reaches the new session. + +On the client, `packages/client-runtime/src/device/hubTransport.ts` turns the +channel into a `Response` with a `ReadableStream` body (`fetch`) and a +`WebSocket`-shaped object (`socket`). T3's stream client +(`client-runtime/src/device/stream.ts`) takes that transport instead of URLs, +so its AVCC/WebCodecs decoder, MJPEG fallback (fetched, painted to the canvas) +and Android SEMU socket port with small changes. One path for every client: no +special case for the desktop app. + +Owner and paired clients may both use devices (same rule as terminals). + +### Agents + +- MCP server `threadlines_device` (like `McpCommandsServer.ts`), scope flag + `device` on the credential, tools from T3's `toolkits/device/tools.ts` with + their descriptions. `device_open` returns the quick start text and a bound + launcher path; `device_screenshot` returns an image block. +- Session flag `ProviderSessionStartInput.deviceTools`, set in + `ProviderCommandReactor` when `enableDevices && agentDeviceAccess`. Like the + other per-session flags, it applies to sessions started after the switch. +- **No PATH changes.** `device_open` writes a launcher bound to that thread's + agent-device session at `/device/bin///agent-device` + and returns its absolute path; the quick start tells the agent to use that + exact path. This works the same in every provider, including OpenCode, whose + one server per instance would never see a PATH change made for a later + session. T3's launcher logic (refused commands, connection check, naming a + sandboxed shell versus a dead daemon) ports as is. +- Registered where the commands server is registered today: Codex + (`codexAppServerArgs`), Claude (`ClaudeAdapter` mcpServers), ACP + (`AcpAdapter`), Pi, Muse, OpenCode. ACP agents that do not advertise HTTP + MCP get no device tools, the same as every other Threadlines tool today. +- Instructions: `provider/deviceInstructions.ts`, a short block like + `previewPanelInstructions.ts`, sent only to sessions with `deviceTools`: + use the device tools and `agent-device` for simulator UI work, `simctl`/`adb` + stay fine for builds, installs and logs. Codex, Claude, OpenCode, Pi, Muse. + ACP providers get it through tool descriptions only (their existing rule). +- An agent's `device_open` publishes an activity the web uses to open the + Device tab on that device for anyone viewing the thread. + +### Web + +- Right-panel tab `device` (`rightPanelTabs.ts`), available for project + threads on every client. Keybinding `device.toggle`, command palette entry + "Open device". +- First open: setup steps (start the hub, platform check, agent access + choice). Opening the tab alone downloads nothing. +- Device picker, the screen in a flat frame sized to the device's aspect, + toolbar, Tools drawer (`DeviceToolsPanel`), loading and error states that say + what is missing ("Android SDK Command-line Tools are missing from …"). +- Only the visible device streams; switching tabs or threads stops the channel. +- Settings → Devices section with the two switches and tool versions. + +### Settings + +`ServerSettings.enableDevices` (default false) and +`ServerSettings.agentDeviceAccess` (default false), plus patch fields. Turning +"Use devices" off stops the hub and daemon; simulators keep running. Turning +"Agents can use devices" off removes the tools from sessions started after. +Both get app guide places (`devices.enable`, `devices.agentAccess`); agents may +propose turning on "Use devices" through `settings_change` but agent access is +user-only (`widensAgents`). + +## Contracts + +`packages/contracts/src/device.ts`: T3's device schemas without host configs +(`hostId` dropped everywhere: one host per server), the channel schemas, and +the MCP tool schemas. RPCs in their own `WsDeviceRpcGroup`: +`deviceList`, `deviceOpen`, `deviceClose`, `deviceShutdown`, `deviceDetail`, +`deviceAction`, `deviceUpdateTool`, `subscribeDeviceState` (stream), +`deviceChannelOpen` (stream), `deviceChannelSend`, `deviceChannelCredit`. + +## Risks + +- Both tools move fast (hub 0.19 → 0.25 in a month). Pinned; updated on purpose. +- serve-sim loads private CoreSimulator frameworks through a native addon. It + runs as a child so a crash restarts the hub, not the server. +- iOS first agent tap builds an XCTest runner (minutes, once). +- Video over base64 WS RPC costs about a third more bytes than T3's raw HTTP. + iOS AVCC at simulator bitrates is well inside that. +- WebCodecs needs a secure context (the desktop app, https, the relay, + localhost). On a plain-http LAN address iOS falls back to MJPEG, and + Android falls back to still frames from serve-emu's screenshot route at + about 4 per second, with touch still live and a one-line note saying a + secure connection gives full video. diff --git a/packages/contracts/src/device.ts b/packages/contracts/src/device.ts new file mode 100644 index 000000000..2a4dcdfa4 --- /dev/null +++ b/packages/contracts/src/device.ts @@ -0,0 +1,603 @@ +/** + * Simulators and emulators on the server's machine (docs/design/devices.md). + * + * The server owns them the way it owns terminals; clients watch and touch them + * through the `device.*` RPCs and the device channel, agents through the + * `device_*` MCP tools. Ported from T3 Code's device contracts without its + * remote host kinds: a server has exactly one device host, its own machine. + */ +import * as Schema from "effect/Schema"; + +import { ThreadId, TrimmedNonEmptyString } from "./baseSchemas.ts"; + +export const DevicePlatform = Schema.Literals(["ios", "android"]); +export type DevicePlatform = typeof DevicePlatform.Type; + +export const DeviceId = TrimmedNonEmptyString.check(Schema.isMaxLength(256)); +export type DeviceId = typeof DeviceId.Type; + +export const DeviceSummary = Schema.Struct({ + id: DeviceId, + platform: DevicePlatform, + name: TrimmedNonEmptyString, + /** OS version, such as "iOS 27.2" or "Android 16". */ + version: Schema.String, + booted: Schema.Boolean, + physical: Schema.Boolean, +}); +export type DeviceSummary = typeof DeviceSummary.Type; + +export const DevicePlatformAvailability = Schema.Struct({ + platform: DevicePlatform, + available: Schema.Boolean, + /** Why not, in words the user can act on ("Xcode command line tools were not found."). */ + reason: Schema.optional(Schema.String), +}); +export type DevicePlatformAvailability = typeof DevicePlatformAvailability.Type; + +/** One pinned tool: the version this server wants, what is on disk, what is running. */ +export const DeviceToolVersion = Schema.Struct({ + requiredVersion: Schema.String, + installedVersions: Schema.Array(Schema.String), + runningVersion: Schema.NullOr(Schema.String), +}); +export type DeviceToolVersion = typeof DeviceToolVersion.Type; + +export const DeviceToolName = Schema.Literals(["hub", "agent"]); +export type DeviceToolName = typeof DeviceToolName.Type; + +export const DeviceToolVersions = Schema.Struct({ + /** expo-device-hub: streams and takes input. */ + hub: DeviceToolVersion, + /** agent-device: the command line agents drive devices with. */ + agent: DeviceToolVersion, +}); +export type DeviceToolVersions = typeof DeviceToolVersions.Type; + +/** + * - `disabled`: "Use devices" is off. + * - `idle`: on, nothing started yet (nothing starts until a client or agent asks). + * - `installing` / `starting`: the hub is being installed or started. + * - `ready`: devices can be listed and opened. + * - `failed`: see `hostStatusDetail`. + */ +export const DeviceHostStatus = Schema.Literals([ + "disabled", + "idle", + "installing", + "starting", + "ready", + "failed", +]); +export type DeviceHostStatus = typeof DeviceHostStatus.Type; + +/** A thread's hold on a device, from opening it until it is closed or the thread is put away. */ +export const DeviceSession = Schema.Struct({ + threadId: ThreadId, + deviceId: DeviceId, + platform: DevicePlatform, + openedAt: Schema.String, + /** Threadlines booted it for this session, rather than finding it running. */ + bootedByThreadlines: Schema.Boolean, + /** An agent's `device_open` brings the Device tab up for everyone watching the thread; a user's does not. */ + openedBy: Schema.Literals(["user", "agent"]), +}); +export type DeviceSession = typeof DeviceSession.Type; + +export const DeviceServiceState = Schema.Struct({ + /** Settings → Devices → Use devices. */ + enabled: Schema.Boolean, + /** Settings → Devices → Agents can use devices. */ + agentAccessEnabled: Schema.Boolean, + hostStatus: DeviceHostStatus, + hostStatusDetail: Schema.optional(Schema.String), + platforms: Schema.Array(DevicePlatformAvailability), + /** Platforms whose last listing failed; their devices may be missing from `devices`. */ + incompletePlatforms: Schema.Array(DevicePlatform), + tools: Schema.optional(DeviceToolVersions), + toolInspectionError: Schema.optional(Schema.String), + hubInstalled: Schema.Boolean, + agentDeviceInstalled: Schema.Boolean, + devices: Schema.Array(DeviceSummary), + sessions: Schema.Array(DeviceSession), + /** Devices booting right now, and the thread that asked. */ + bootingDevices: Schema.Array(Schema.Struct({ ...DeviceSummary.fields, threadId: ThreadId })), + /** Rises with every change; clients drop states older than one they have. */ + revision: Schema.Int, +}); +export type DeviceServiceState = typeof DeviceServiceState.Type; + +export const DeviceListInput = Schema.Struct({ + /** Ask the hub again instead of answering from the last listing. Starts the hub if needed. */ + refresh: Schema.optional(Schema.Boolean), +}); +export type DeviceListInput = typeof DeviceListInput.Type; + +export const DeviceUpdateToolInput = Schema.Struct({ + tool: DeviceToolName, +}); +export type DeviceUpdateToolInput = typeof DeviceUpdateToolInput.Type; + +export const DeviceOpenInput = Schema.Struct({ + threadId: ThreadId, + deviceId: DeviceId, + platform: DevicePlatform, + /** Boot the simulator or emulator when it is not running. Defaults to true. */ + boot: Schema.optional(Schema.Boolean), +}); +export type DeviceOpenInput = typeof DeviceOpenInput.Type; + +export const DeviceCloseInput = Schema.Struct({ + threadId: ThreadId, + /** Omit to close every device session for the thread. */ + deviceId: Schema.optional(DeviceId), + /** Also shut the simulator or emulator down. Defaults to false. */ + shutdown: Schema.optional(Schema.Boolean), +}); +export type DeviceCloseInput = typeof DeviceCloseInput.Type; + +export const DeviceShutdownInput = Schema.Struct({ + deviceId: DeviceId, + platform: DevicePlatform, +}); +export type DeviceShutdownInput = typeof DeviceShutdownInput.Type; + +export const DeviceAppearance = Schema.Literals(["light", "dark"]); +export type DeviceAppearance = typeof DeviceAppearance.Type; + +export const DeviceTextSize = Schema.Literals(["small", "default", "large", "extra-large"]); +export type DeviceTextSize = typeof DeviceTextSize.Type; + +export const DeviceColorFilter = Schema.Literals([ + "none", + "grayscale", + "red-green", + "green-red", + "blue-yellow", +]); +export type DeviceColorFilter = typeof DeviceColorFilter.Type; + +export const DeviceOrientation = Schema.Literals([ + "portrait", + "landscape_left", + "portrait_upside_down", + "landscape_right", +]); +export type DeviceOrientation = typeof DeviceOrientation.Type; + +/** What the Tools drawer shows, read back from the device. A missing field is one the platform lacks. */ +export const DeviceSettings = Schema.Struct({ + appearance: Schema.optional(DeviceAppearance), + textSize: Schema.optional(DeviceTextSize), + reduceMotion: Schema.optional(Schema.Boolean), + increaseContrast: Schema.optional(Schema.Boolean), + reduceTransparency: Schema.optional(Schema.Boolean), + showBorders: Schema.optional(Schema.Boolean), + voiceOver: Schema.optional(Schema.Boolean), + liquidGlass: Schema.optional(Schema.Literals(["clear", "tinted"])), + colorFilter: Schema.optional(DeviceColorFilter), + networkEnabled: Schema.optional(Schema.Boolean), + location: Schema.optional( + Schema.NullOr(Schema.Struct({ latitude: Schema.Number, longitude: Schema.Number })), + ), +}); +export type DeviceSettings = typeof DeviceSettings.Type; + +export const DeviceForegroundApp = Schema.Struct({ + id: Schema.String, + name: Schema.optional(Schema.String), + version: Schema.optional(Schema.String), +}); +export type DeviceForegroundApp = typeof DeviceForegroundApp.Type; + +export const DeviceDetail = Schema.Struct({ + deviceId: DeviceId, + settings: DeviceSettings, + foregroundApp: Schema.NullOr(DeviceForegroundApp), + readAt: Schema.String, +}); +export type DeviceDetail = typeof DeviceDetail.Type; + +export const DevicePermission = Schema.Literals([ + "camera", + "microphone", + "photos", + "contacts", + "calendar", + "reminders", + "location", + "notifications", + "motion", + "media-library", + "faceid", +]); +export type DevicePermission = typeof DevicePermission.Type; + +const DeviceTarget = { + deviceId: DeviceId, +}; + +/** + * One Tools drawer control. The server runs `simctl`, `adb` or a serve-sim + * helper binary for each itself; none of them goes through the hub. + */ +export const DeviceActionInput = Schema.Union([ + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setAppearance"), + value: DeviceAppearance, + }), + Schema.Struct({ ...DeviceTarget, type: Schema.Literal("setTextSize"), value: DeviceTextSize }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setToggle"), + setting: Schema.Literals([ + "reduceMotion", + "increaseContrast", + "reduceTransparency", + "showBorders", + "voiceOver", + "networkEnabled", + ]), + value: Schema.Boolean, + }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setLiquidGlass"), + value: Schema.Literals(["clear", "tinted"]), + }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setColorFilter"), + value: DeviceColorFilter, + }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setOrientation"), + value: DeviceOrientation, + }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setLocation"), + latitude: Schema.Number.check(Schema.isBetween({ minimum: -90, maximum: 90 })), + longitude: Schema.Number.check(Schema.isBetween({ minimum: -180, maximum: 180 })), + }), + Schema.Struct({ ...DeviceTarget, type: Schema.Literal("clearLocation") }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setPermission"), + appId: TrimmedNonEmptyString, + permission: DevicePermission, + decision: Schema.Literals(["grant", "revoke", "reset"]), + }), + Schema.Struct({ ...DeviceTarget, type: Schema.Literal("openUrl"), url: TrimmedNonEmptyString }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("launchApp"), + appId: TrimmedNonEmptyString, + }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("terminateApp"), + appId: TrimmedNonEmptyString, + }), + Schema.Struct({ ...DeviceTarget, type: Schema.Literal("shake") }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("sendPush"), + appId: TrimmedNonEmptyString, + payload: Schema.Union([Schema.String, Schema.Record(Schema.String, Schema.Unknown)]), + }), +]); +export type DeviceActionInput = typeof DeviceActionInput.Type; +export type DeviceActionType = DeviceActionInput["type"]; + +export const DeviceDetailInput = Schema.Struct(DeviceTarget); +export type DeviceDetailInput = typeof DeviceDetailInput.Type; + +// ── Errors ───────────────────────────────────────────────────────────── + +/** Devices are off, or the hub could not be installed or started. */ +export class DeviceHostUnavailableError extends Schema.TaggedError()( + "DeviceHostUnavailableError", + { reason: Schema.String }, +) { + override get message(): string { + return `Devices are unavailable: ${this.reason}`; + } +} + +export class DevicePlatformUnavailableError extends Schema.TaggedError()( + "DevicePlatformUnavailableError", + { platform: DevicePlatform, reason: Schema.String }, +) { + override get message(): string { + return `${this.platform === "ios" ? "iOS" : "Android"} devices are unavailable: ${this.reason}`; + } +} + +export class DeviceNotFoundError extends Schema.TaggedError()( + "DeviceNotFoundError", + { deviceId: DeviceId }, +) { + override get message(): string { + return `Device ${this.deviceId} was not found.`; + } +} + +export class DeviceBootError extends Schema.TaggedError()("DeviceBootError", { + deviceId: DeviceId, + reason: Schema.Literals(["disk_space", "timeout", "launch_failed"]), + diagnostics: Schema.optional(Schema.String.check(Schema.isMaxLength(2_000))), +}) { + override get message(): string { + const explanation = { + disk_space: "There is not enough free disk space.", + timeout: "The device did not become ready in time.", + launch_failed: "The simulator or emulator could not start.", + }[this.reason]; + const summary = `Device ${this.deviceId} failed to start: ${explanation}`; + return this.diagnostics ? `${summary}\n\n${this.diagnostics}` : summary; + } +} + +export class DeviceOperationError extends Schema.TaggedError()( + "DeviceOperationError", + { + operation: Schema.String, + reason: Schema.Literals([ + "command_failed", + "request_failed", + "invalid_payload", + "settings_failed", + "hub_rejected", + "package_manager_missing", + ]), + exitCode: Schema.optional(Schema.Number), + /** What the tool said, trimmed. No stack traces: this reaches the browser. */ + detail: Schema.optional(Schema.String.check(Schema.isMaxLength(2_000))), + }, +) { + override get message(): string { + const explanation = { + command_failed: `The device command failed${this.exitCode === undefined ? "" : ` (exit code ${this.exitCode})`}.`, + request_failed: "Could not reach device support. Try refreshing devices.", + invalid_payload: "The device request could not be encoded.", + settings_failed: "Could not read or save device settings.", + hub_rejected: "The device hub could not complete the request.", + package_manager_missing: + "Device support needs npm, which is not on the Threadlines server's PATH. Install Node.js from nodejs.org, restart Threadlines, then try again.", + }[this.reason]; + const base = `Device ${this.operation} failed: ${explanation}`; + return this.detail ? `${base}\n\n${this.detail}` : base; + } +} + +export class DeviceActionUnavailableError extends Schema.TaggedError()( + "DeviceActionUnavailableError", + { + operation: Schema.String, + platform: DevicePlatform, + reason: Schema.Literals(["unsupported", "helper_missing"]), + }, +) { + override get message(): string { + return this.reason === "helper_missing" + ? `Device ${this.operation} needs a helper missing from this install. Update the device hub in Settings → Devices.` + : `Device ${this.operation} is not supported on ${this.platform === "ios" ? "iOS" : "Android"}.`; + } +} + +export const DeviceError = Schema.Union([ + DeviceHostUnavailableError, + DevicePlatformUnavailableError, + DeviceNotFoundError, + DeviceBootError, + DeviceOperationError, + DeviceActionUnavailableError, +]); +export type DeviceError = typeof DeviceError.Type; + +// ── Device channel ───────────────────────────────────────────────────── +// +// Every byte between the Device tab and the hub rides the client's WS RPC +// connection, so it works the same from the desktop app, a browser, a phone +// and a saved computer over the relay. Modeled on the preview tunnel +// (previewTunnel.ts): open while subscribed, never resubscribed, flow +// controlled by credit. The server checks every path against its allowlist and +// adds the hub's token itself; clients never see it. + +/** The most one `data` or `message` event carries, before base64. */ +export const DEVICE_CHANNEL_CHUNK_BYTES = 256 * 1024; + +/** Bytes the server sends down a channel before the client credits them. */ +export const DEVICE_CHANNEL_WINDOW_BYTES = 2 * 1024 * 1024; + +const DeviceChannelId = Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(128)); + +const DeviceChannelPath = Schema.String.check( + Schema.isMinLength(1), + Schema.isMaxLength(512), + Schema.isPattern(/^\/[^?#]*$/), +); + +const DeviceChannelQuery = Schema.Record(Schema.String, Schema.String); + +export const DeviceChannelOpen = Schema.Union([ + Schema.TaggedStruct("http", { + /** Chosen by the client, so sends and credits can name the channel. */ + channelId: DeviceChannelId, + method: Schema.Literals(["GET", "POST"]), + /** A hub path such as `/vendor/serve-sim/helper//stream.avcc`. */ + path: DeviceChannelPath, + query: Schema.optional(DeviceChannelQuery), + /** Request body, base64. Only on the hub's few POST routes. */ + body: Schema.optional(Schema.String.check(Schema.isMaxLength(256 * 1024))), + contentType: Schema.optional(Schema.String.check(Schema.isMaxLength(128))), + }), + Schema.TaggedStruct("ws", { + channelId: DeviceChannelId, + path: DeviceChannelPath, + query: Schema.optional(DeviceChannelQuery), + protocols: Schema.optional(Schema.Array(Schema.String.check(Schema.isMaxLength(128)))), + }), +]); +export type DeviceChannelOpen = typeof DeviceChannelOpen.Type; + +/** + * What the server sends down an open channel. The subscription ends after + * `end`, and fails when the channel could not open or broke. + * + * - `opened`: http status and content type, or a ws that connected. + * - `data`: a slice of an http response body, base64, in order. + * - `message`: a ws message, or a part of one when `final` is false; parts of + * one message arrive together and in order. + * - `end`: the http body finished, or the ws closed (`code`, `reason`). + */ +export const DeviceChannelEvent = Schema.Union([ + Schema.TaggedStruct("opened", { + status: Schema.optional(Schema.Int), + contentType: Schema.optional(Schema.String), + protocol: Schema.optional(Schema.String), + }), + Schema.TaggedStruct("data", { data: Schema.String }), + Schema.TaggedStruct("message", { + data: Schema.String, + binary: Schema.Boolean, + final: Schema.Boolean, + }), + Schema.TaggedStruct("end", { + code: Schema.optional(Schema.Int), + reason: Schema.optional(Schema.String), + }), +]); +export type DeviceChannelEvent = typeof DeviceChannelEvent.Type; + +/** One ws message for the hub, in order. Answered once written to the hub's socket. */ +export const DeviceChannelSend = Schema.Struct({ + channelId: DeviceChannelId, + seq: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + /** base64 when `binary`, the text itself otherwise. */ + data: Schema.String.check(Schema.isMaxLength(512 * 1024)), + binary: Schema.Boolean, +}); +export type DeviceChannelSend = typeof DeviceChannelSend.Type; + +/** The client's decoder took this many more bytes of the channel's data. */ +export const DeviceChannelCredit = Schema.Struct({ + channelId: DeviceChannelId, + bytes: Schema.Int.check(Schema.isGreaterThan(0)), +}); +export type DeviceChannelCredit = typeof DeviceChannelCredit.Type; + +export const DeviceChannelFailureReason = Schema.Literals([ + /** The path is not one a client may use. */ + "forbidden", + /** Devices are off or the hub is not running. */ + "unavailable", + /** The hub could not be reached, or refused. */ + "unreachable", + /** The connection broke after it opened. */ + "broken", + /** The client stopped reading and the server stopped holding data for it. */ + "slow", + /** Too many channels open, or a send out of all proportion. */ + "limit", + /** A send or credit named a channel this connection does not have open. */ + "unknownChannel", +]); +export type DeviceChannelFailureReason = typeof DeviceChannelFailureReason.Type; + +export class DeviceChannelError extends Schema.TaggedError()( + "DeviceChannelError", + { reason: DeviceChannelFailureReason, detail: Schema.String }, +) { + override get message(): string { + return this.detail; + } +} + +// ── Agent tools ──────────────────────────────────────────────────────── + +export const DeviceToolListResult = Schema.Struct({ + hostStatus: DeviceHostStatus, + hostStatusDetail: Schema.optional(Schema.String), + platforms: Schema.Array(DevicePlatformAvailability), + devices: Schema.Array(DeviceSummary), + /** Devices this thread has open. */ + open: Schema.Array(Schema.Struct({ deviceId: DeviceId })), +}); +export type DeviceToolListResult = typeof DeviceToolListResult.Type; + +export const DeviceToolOpenInput = Schema.Struct({ + deviceId: Schema.optional( + DeviceId.annotate({ + description: "Device id from `device_list`. Default: the booted or last used device.", + }), + ), + platform: Schema.optional( + DevicePlatform.annotate({ + description: "Needed with no `deviceId` when both platforms exist.", + }), + ), +}); +export type DeviceToolOpenInput = typeof DeviceToolOpenInput.Type; + +export const DeviceToolOpenResult = Schema.Struct({ + device: DeviceSummary, + agentDevice: Schema.Struct({ + /** Absolute path of the launcher bound to this thread and device. */ + command: Schema.String, + session: Schema.String, + }), + quickStart: Schema.String, +}); +export type DeviceToolOpenResult = typeof DeviceToolOpenResult.Type; + +export const DeviceToolTargetInput = Schema.Struct({ + deviceId: Schema.optional( + DeviceId.annotate({ description: "Device. Default: the last one opened here." }), + ), +}); +export type DeviceToolTargetInput = typeof DeviceToolTargetInput.Type; + +export const DeviceToolScreenshotResult = Schema.Struct({ + device: DeviceSummary, + screenshot: Schema.Struct({ + mimeType: Schema.Literal("image/png"), + data: Schema.String, + width: Schema.Int, + height: Schema.Int, + }), +}); +export type DeviceToolScreenshotResult = typeof DeviceToolScreenshotResult.Type; + +export const DeviceToolCloseInput = Schema.Struct({ + deviceId: Schema.optional( + DeviceId.annotate({ description: "Device to close. None: close all here." }), + ), + shutdown: Schema.optional( + Schema.Boolean.annotate({ description: "Also turn it off. Default false." }), + ), +}); +export type DeviceToolCloseInput = typeof DeviceToolCloseInput.Type; + +export class DeviceToolUnavailableError extends Schema.TaggedError()( + "DeviceToolUnavailableError", + { reason: Schema.String }, +) { + override get message(): string { + return this.reason; + } +} + +export const DeviceToolError = Schema.Union([ + DeviceToolUnavailableError, + DeviceHostUnavailableError, + DevicePlatformUnavailableError, + DeviceNotFoundError, + DeviceBootError, + DeviceOperationError, + DeviceActionUnavailableError, +]); +export type DeviceToolError = typeof DeviceToolError.Type; diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 6de99680c..791371e74 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -10,6 +10,7 @@ export * from "./pushAlerts.ts"; export * from "./ipc.ts"; export * from "./previewAutomation.ts"; export * from "./previewTunnel.ts"; +export * from "./device.ts"; export * from "./terminal.ts"; export * from "./provider.ts"; export * from "./providerAuth.ts"; diff --git a/packages/contracts/src/provider.ts b/packages/contracts/src/provider.ts index 2b1f97a30..2c9ab5d5c 100644 --- a/packages/contracts/src/provider.ts +++ b/packages/contracts/src/provider.ts @@ -228,6 +228,13 @@ export const ProviderSessionStartInput = Schema.Struct({ * docs/terminal-command-runs.md). Never for a locked-down side runtime. */ commandRequests: Schema.optional(Schema.Boolean), + /** + * Attach the device tools endpoint (`threadlines_device`): `device_list`, + * `device_open`, `device_screenshot`, `device_close` (docs/design/devices.md). + * Set when "Use devices" and "Agents can use devices" are both on. Never for + * a locked-down side runtime. + */ + deviceTools: Schema.optional(Schema.Boolean), approvalPolicy: Schema.optional(ProviderApprovalPolicy), sandboxMode: Schema.optional(ProviderSandboxMode), runtimeMode: RuntimeMode, diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index 61c8d3ab2..a5c7f1421 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -195,6 +195,24 @@ import { PreviewTunnelOpenSchema, PreviewTunnelWriteSchema, } from "./previewTunnel.ts"; +import { + DeviceActionInput, + DeviceChannelCredit, + DeviceChannelError, + DeviceChannelEvent, + DeviceChannelOpen, + DeviceChannelSend, + DeviceCloseInput, + DeviceDetail, + DeviceDetailInput, + DeviceError, + DeviceListInput, + DeviceOpenInput, + DeviceServiceState, + DeviceSession, + DeviceShutdownInput, + DeviceUpdateToolInput, +} from "./device.ts"; import { FilesystemBrowseInput, FilesystemBrowseResult, @@ -738,6 +756,17 @@ export const WS_METHODS = { previewTunnelWrite: "previewTunnelWrite", previewTunnelCredit: "previewTunnelCredit", previewLocalServers: "previewLocalServers", + deviceList: "device.list", + deviceUpdateTool: "device.updateTool", + deviceOpen: "device.open", + deviceClose: "device.close", + deviceShutdown: "device.shutdown", + deviceDetail: "device.detail", + deviceAction: "device.action", + subscribeDeviceState: "subscribeDeviceState", + deviceChannelOpen: "device.channelOpen", + deviceChannelSend: "device.channelSend", + deviceChannelCredit: "device.channelCredit", subscribeVcsStatus: "subscribeVcsStatus", subscribeTerminalEvents: "subscribeTerminalEvents", subscribeServerConfig: "subscribeServerConfig", @@ -3115,10 +3144,108 @@ export const WsCoreRpcGroup = RpcGroup.make( WsOrchestrationSubscribeThreadRpc, ); +/** The devices on this server's machine, and a refreshed listing. Starts the hub when asked to refresh. */ +export const WsDeviceListRpc = Rpc.make(WS_METHODS.deviceList, { + payload: DeviceListInput, + success: DeviceServiceState, + error: DeviceError, +}); + +/** Install the pinned version of a device tool and restart it. */ +export const WsDeviceUpdateToolRpc = Rpc.make(WS_METHODS.deviceUpdateTool, { + payload: DeviceUpdateToolInput, + success: DeviceServiceState, + error: DeviceError, +}); + +/** A thread starts watching a device, booting it unless `boot` is false. */ +export const WsDeviceOpenRpc = Rpc.make(WS_METHODS.deviceOpen, { + payload: DeviceOpenInput, + success: DeviceSession, + error: DeviceError, +}); + +/** A thread stops holding a device (or all of them), optionally shutting it down. */ +export const WsDeviceCloseRpc = Rpc.make(WS_METHODS.deviceClose, { + payload: DeviceCloseInput, + error: DeviceError, +}); + +/** Power a device off, whoever holds it. */ +export const WsDeviceShutdownRpc = Rpc.make(WS_METHODS.deviceShutdown, { + payload: DeviceShutdownInput, + error: DeviceError, +}); + +/** The Tools drawer's values, read from the device. */ +export const WsDeviceDetailRpc = Rpc.make(WS_METHODS.deviceDetail, { + payload: DeviceDetailInput, + success: DeviceDetail, + error: DeviceError, +}); + +/** One Tools drawer control; answers with the values read back after it. */ +export const WsDeviceActionRpc = Rpc.make(WS_METHODS.deviceAction, { + payload: DeviceActionInput, + success: DeviceDetail, + error: DeviceError, +}); + +/** The device state now and after every change. */ +export const WsSubscribeDeviceStateRpc = Rpc.make(WS_METHODS.subscribeDeviceState, { + payload: Schema.Struct({}), + success: DeviceServiceState, + stream: true, +}); + +/** + * One request or socket to the device hub, for the Device tab's stream and + * input. Open while subscribed; never resubscribed (see device.ts). + */ +export const WsDeviceChannelOpenRpc = Rpc.make(WS_METHODS.deviceChannelOpen, { + payload: DeviceChannelOpen, + success: DeviceChannelEvent, + error: DeviceChannelError, + stream: true, +}); + +/** One ws message for an open channel, answered once written. */ +export const WsDeviceChannelSendRpc = Rpc.make(WS_METHODS.deviceChannelSend, { + payload: DeviceChannelSend, + success: Schema.Void, + error: DeviceChannelError, +}); + +/** The client took more of a channel's data. Fire and forget. */ +export const WsDeviceChannelCreditRpc = Rpc.make(WS_METHODS.deviceChannelCredit, { + payload: DeviceChannelCredit, + success: Schema.Void, +}); + +/** + * Devices (docs/design/devices.md). A group of its own, merged into + * WsRpcGroup, because the main group sits near TypeScript's cap on type work + * per statement. + */ +export const WsDeviceRpcGroup = RpcGroup.make( + WsDeviceListRpc, + WsDeviceUpdateToolRpc, + WsDeviceOpenRpc, + WsDeviceCloseRpc, + WsDeviceShutdownRpc, + WsDeviceDetailRpc, + WsDeviceActionRpc, + WsSubscribeDeviceStateRpc, + WsDeviceChannelOpenRpc, + WsDeviceChannelSendRpc, + WsDeviceChannelCreditRpc, +); + export const WsRpcGroup = WsCoreRpcGroup.merge( WsOutsideAgentsRpcGroup, WsThemesRpcGroup, WsThemeShowcaseRpcGroup, WsMovesRpcGroup, WsThemeVsCodeRpcGroup, + WsDeviceRpcGroup, ); diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index def954acf..59bf11f5a 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -1073,6 +1073,19 @@ export const ServerSettings = Schema.Struct({ * also refuses a running session's next publish. Never chosen: off. */ enableClaudeArtifacts: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), + /** + * Devices: the Device tab shows iOS Simulators and Android Emulators on + * this computer (docs/design/devices.md). Turning it on installs and starts + * the device hub when the tab or an agent first asks; off stops the hub. + * Simulators keep running either way. Never chosen: off. + */ + enableDevices: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), + /** + * Agents can use devices: sessions started while this and `enableDevices` + * are on get the `device_*` tools, and the agent-device command line is + * installed. Applies to sessions started after a change. Never chosen: off. + */ + agentDeviceAccess: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), /** * How many agents Claude aims for when it writes a workflow, for every * Claude account. Read when a Claude session starts and before each turn, @@ -1312,6 +1325,8 @@ export const ServerSettingsPatch = Schema.Struct({ enableRooms: Schema.optionalKey(Schema.Boolean), enableAgentPages: Schema.optionalKey(Schema.Boolean), enableClaudeArtifacts: Schema.optionalKey(Schema.Boolean), + enableDevices: Schema.optionalKey(Schema.Boolean), + agentDeviceAccess: Schema.optionalKey(Schema.Boolean), claudeWorkflowSize: Schema.optionalKey(ClaudeWorkflowSize), claudeSuggestionsWhenUsageRunsFast: Schema.optionalKey(Schema.Boolean), codexSuggestions: Schema.optionalKey(Schema.Boolean), From 2c483615fb1cb02d152f2e1d1292bfbcfd52cc20 Mon Sep 17 00:00:00 2001 From: Badcuban <108198679+badcuban@users.noreply.github.com> Date: Fri, 20 Nov 2026 00:48:25 -0500 Subject: [PATCH 02/49] feat(client-runtime): device hub transport and stream client over the device channel The Device tab reaches the device hub only through the app's WS RPC connection. hubTransport turns the device channel RPCs into a fetch that answers a streaming Response and a WebSocket-shaped socket, reassembles message parts, credits a chunk once its reader took it and never reopens a channel on its own. The stream client is T3 Code's, ported without 3D, Duo and WebRTC, and running on that transport: AVCC with WebCodecs, MJPEG painted to the canvas without it on iOS, and polled screenshots with live touch on Android without it. Also the screenshot capture and the pure state helpers (install line, platform status, new agent sessions). --- packages/client-runtime/package.json | 4 + packages/client-runtime/src/device/base64.ts | 21 + packages/client-runtime/src/device/frame.ts | 24 + .../src/device/hubTransport.test.ts | 234 ++++ .../client-runtime/src/device/hubTransport.ts | 486 ++++++++ packages/client-runtime/src/device/index.ts | 5 + packages/client-runtime/src/device/mjpeg.ts | 75 ++ packages/client-runtime/src/device/png.ts | 24 + .../client-runtime/src/device/screenshot.ts | 36 + .../client-runtime/src/device/stream.test.ts | 218 ++++ packages/client-runtime/src/device/stream.ts | 1019 +++++++++++++++++ .../src/state/deviceState.test.ts | 47 + .../client-runtime/src/state/deviceState.ts | 137 +++ 13 files changed, 2330 insertions(+) create mode 100644 packages/client-runtime/src/device/base64.ts create mode 100644 packages/client-runtime/src/device/frame.ts create mode 100644 packages/client-runtime/src/device/hubTransport.test.ts create mode 100644 packages/client-runtime/src/device/hubTransport.ts create mode 100644 packages/client-runtime/src/device/index.ts create mode 100644 packages/client-runtime/src/device/mjpeg.ts create mode 100644 packages/client-runtime/src/device/png.ts create mode 100644 packages/client-runtime/src/device/screenshot.ts create mode 100644 packages/client-runtime/src/device/stream.test.ts create mode 100644 packages/client-runtime/src/device/stream.ts create mode 100644 packages/client-runtime/src/state/deviceState.test.ts create mode 100644 packages/client-runtime/src/state/deviceState.ts diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index d43658a2c..22bb6214e 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -6,6 +6,10 @@ ".": { "types": "./src/index.ts", "import": "./src/index.ts" + }, + "./device": { + "types": "./src/device/index.ts", + "import": "./src/device/index.ts" } }, "scripts": { diff --git a/packages/client-runtime/src/device/base64.ts b/packages/client-runtime/src/device/base64.ts new file mode 100644 index 000000000..50a4b28f7 --- /dev/null +++ b/packages/client-runtime/src/device/base64.ts @@ -0,0 +1,21 @@ +/** Base64 for the device channel, using the native codec where the engine has one. */ + +export function bytesToBase64(bytes: Uint8Array): string { + const native = (bytes as Uint8Array & { toBase64?: () => string }).toBase64; + if (typeof native === "function") return native.call(bytes); + let binary = ""; + for (let offset = 0; offset < bytes.length; offset += 0x8000) { + binary += String.fromCharCode(...bytes.subarray(offset, offset + 0x8000)); + } + return btoa(binary); +} + +export function base64ToBytes(text: string): Uint8Array { + const native = (Uint8Array as { fromBase64?: (text: string) => Uint8Array }) + .fromBase64; + if (typeof native === "function") return native(text); + const binary = atob(text); + const bytes = new Uint8Array(binary.length); + for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index); + return bytes; +} diff --git a/packages/client-runtime/src/device/frame.ts b/packages/client-runtime/src/device/frame.ts new file mode 100644 index 000000000..428ac7ca4 --- /dev/null +++ b/packages/client-runtime/src/device/frame.ts @@ -0,0 +1,24 @@ +/** A synchronous, borrowed frame. The producer releases its source after `present` returns. */ +export interface DeviceFrameSink { + readonly present: (source: CanvasImageSource, width: number, height: number) => boolean; +} + +/** Draws each frame onto `canvas`, resizing it to the frame. */ +export function createCanvasFrameSink( + canvas: HTMLCanvasElement, + onFrame?: (width: number, height: number) => void, +): DeviceFrameSink { + return { + present(source, width, height) { + const context = canvas.getContext("2d"); + if (!context) return false; + if (canvas.width !== width || canvas.height !== height) { + canvas.width = width; + canvas.height = height; + } + context.drawImage(source, 0, 0, width, height); + onFrame?.(width, height); + return true; + }, + }; +} diff --git a/packages/client-runtime/src/device/hubTransport.test.ts b/packages/client-runtime/src/device/hubTransport.test.ts new file mode 100644 index 000000000..36c52ef54 --- /dev/null +++ b/packages/client-runtime/src/device/hubTransport.test.ts @@ -0,0 +1,234 @@ +import type { + DeviceChannelCredit, + DeviceChannelEvent, + DeviceChannelOpen, + DeviceChannelSend, +} from "@threadlines/contracts"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; + +import { bytesToBase64 } from "./base64.ts"; +import { + createDeviceHubTransport, + DEVICE_SOCKET_CLOSED, + DEVICE_SOCKET_OPEN, + type DeviceChannelApi, +} from "./hubTransport.ts"; + +interface OpenChannel { + readonly input: DeviceChannelOpen; + readonly emit: (event: DeviceChannelEvent) => void; + readonly close: (failure?: unknown) => void; + stopped: boolean; +} + +function fakeChannelApi() { + const channels: OpenChannel[] = []; + const sends: DeviceChannelSend[] = []; + const credits: DeviceChannelCredit[] = []; + const api: DeviceChannelApi = { + open: (input, listener, onClosed) => { + const channel: OpenChannel = { + input, + emit: listener, + close: onClosed, + stopped: false, + }; + channels.push(channel); + return () => { + channel.stopped = true; + }; + }, + send: async (input) => { + sends.push(input); + }, + credit: async (input) => { + credits.push(input); + }, + }; + let id = 0; + const transport = createDeviceHubTransport(api, { channelId: () => `ch-${++id}` }); + return { api, transport, channels, sends, credits }; +} + +const bytes = (...values: number[]) => new Uint8Array(values); +const data = (...values: number[]): DeviceChannelEvent => ({ + _tag: "data", + data: bytesToBase64(bytes(...values)), +}); + +beforeEach(() => vi.useFakeTimers()); +afterEach(() => vi.useRealTimers()); + +describe("fetch", () => { + it("opens an http channel with the path and query split, and answers a streaming Response", async () => { + const fake = fakeChannelApi(); + const pending = fake.transport.fetch("/vendor/serve-sim/helper/A%20B/stream.avcc?x=1", { + method: "POST", + body: "hi", + contentType: "text/plain", + }); + expect(fake.channels[0]?.input).toEqual({ + _tag: "http", + channelId: "ch-1", + method: "POST", + path: "/vendor/serve-sim/helper/A%20B/stream.avcc", + query: { x: "1" }, + body: bytesToBase64(new TextEncoder().encode("hi")), + contentType: "text/plain", + }); + fake.channels[0]!.emit({ _tag: "opened", status: 200, contentType: "video/avcc" }); + const response = await pending; + expect(response.status).toBe(200); + expect(response.headers.get("content-type")).toBe("video/avcc"); + + fake.channels[0]!.emit(data(1, 2, 3)); + fake.channels[0]!.emit(data(4)); + fake.channels[0]!.emit({ _tag: "end" }); + expect(new Uint8Array(await response.arrayBuffer())).toEqual(bytes(1, 2, 3, 4)); + expect(fake.channels[0]!.stopped).toBe(true); + }); + + it("credits a chunk only once a reader has taken it", async () => { + const fake = fakeChannelApi(); + const pending = fake.transport.fetch("/vendor/serve-sim/helper/A/stream.avcc"); + fake.channels[0]!.emit({ _tag: "opened", status: 200 }); + const reader = (await pending).body!.getReader(); + + fake.channels[0]!.emit(data(1, 2, 3)); + fake.channels[0]!.emit(data(4, 5)); + await vi.advanceTimersByTimeAsync(1_000); + expect(fake.credits).toEqual([]); + + expect((await reader.read()).value).toEqual(bytes(1, 2, 3)); + await vi.advanceTimersByTimeAsync(1_000); + expect(fake.credits).toEqual([{ channelId: "ch-1", bytes: 3 }]); + + expect((await reader.read()).value).toEqual(bytes(4, 5)); + await vi.advanceTimersByTimeAsync(1_000); + expect(fake.credits).toEqual([ + { channelId: "ch-1", bytes: 3 }, + { channelId: "ch-1", bytes: 2 }, + ]); + }); + + it("rejects when the channel fails before answering, and errors the body after", async () => { + const fake = fakeChannelApi(); + const refused = fake.transport.fetch("/vendor/serve-sim/helper/A/stream.avcc"); + fake.channels[0]!.close({ _tag: "DeviceChannelError", reason: "forbidden", detail: "No." }); + await expect(refused).rejects.toThrow("No."); + + const pending = fake.transport.fetch("/vendor/serve-sim/helper/A/stream.avcc"); + fake.channels[1]!.emit({ _tag: "opened", status: 200 }); + const reader = (await pending).body!.getReader(); + fake.channels[1]!.close(new Error("The connection dropped.")); + await expect(reader.read()).rejects.toThrow("The connection dropped."); + // A dropped channel is never reopened by the transport: the caller asks again. + expect(fake.channels).toHaveLength(2); + }); + + it("closes the channel when aborted", async () => { + const fake = fakeChannelApi(); + const controller = new AbortController(); + const pending = fake.transport.fetch("/vendor/serve-sim/helper/A/stream.mjpeg", { + signal: controller.signal, + }); + fake.channels[0]!.emit({ _tag: "opened", status: 200 }); + const reader = (await pending).body!.getReader(); + controller.abort(); + await expect(reader.read()).rejects.toThrow(); + expect(fake.channels[0]!.stopped).toBe(true); + }); +}); + +describe("socket", () => { + it("opens on `opened`, reassembles message parts, and credits after the handler ran", async () => { + const fake = fakeChannelApi(); + const socket = fake.transport.socket("/vendor/serve-emu/ws?device=emu-1&frame-meta=1"); + expect(fake.channels[0]?.input).toEqual({ + _tag: "ws", + channelId: "ch-1", + path: "/vendor/serve-emu/ws", + query: { device: "emu-1", "frame-meta": "1" }, + }); + const opened = vi.fn(); + const received: Array = []; + socket.addEventListener("open", opened); + socket.addEventListener("message", (event) => { + // The message has not been credited while its handler runs. + expect(fake.credits).toEqual([]); + received.push(event.data); + }); + + socket.send("dropped before open"); + expect(fake.sends).toEqual([]); + + fake.channels[0]!.emit({ _tag: "opened" }); + expect(opened).toHaveBeenCalledOnce(); + expect(socket.readyState).toBe(DEVICE_SOCKET_OPEN); + + const channel = fake.channels[0]!; + channel.emit({ _tag: "message", binary: true, final: false, data: bytesToBase64(bytes(1)) }); + channel.emit({ _tag: "message", binary: true, final: false, data: bytesToBase64(bytes(2)) }); + expect(received).toEqual([]); + channel.emit({ _tag: "message", binary: true, final: true, data: bytesToBase64(bytes(3)) }); + channel.emit({ _tag: "message", binary: false, final: true, data: '{"type":"video-session"}' }); + expect(new Uint8Array(received[0] as ArrayBuffer)).toEqual(bytes(1, 2, 3)); + expect(received[1]).toBe('{"type":"video-session"}'); + + await vi.advanceTimersByTimeAsync(1_000); + expect(fake.credits).toEqual([{ channelId: "ch-1", bytes: 3 + 24 }]); + }); + + it("sends text and binary in order with rising seq", () => { + const fake = fakeChannelApi(); + const socket = fake.transport.socket("/vendor/serve-sim/helper/ws?device=A"); + fake.channels[0]!.emit({ _tag: "opened" }); + socket.send(bytes(3, 1)); + socket.send('{"type":"home"}'); + expect(fake.sends).toEqual([ + { channelId: "ch-1", seq: 0, binary: true, data: bytesToBase64(bytes(3, 1)) }, + { channelId: "ch-1", seq: 1, binary: false, data: '{"type":"home"}' }, + ]); + }); + + it("closes once on failure, drops later input and never reopens", () => { + const fake = fakeChannelApi(); + const socket = fake.transport.socket("/vendor/serve-sim/helper/ws?device=A"); + const closed = vi.fn(); + const errored = vi.fn(); + socket.addEventListener("close", closed); + socket.addEventListener("error", errored); + fake.channels[0]!.emit({ _tag: "opened" }); + fake.channels[0]!.close({ _tag: "DeviceChannelError", reason: "slow", detail: "Too slow." }); + expect(errored).toHaveBeenCalledOnce(); + expect(closed).toHaveBeenCalledExactlyOnceWith({ + code: 1006, + reason: "Too slow.", + wasClean: false, + }); + expect(socket.readyState).toBe(DEVICE_SOCKET_CLOSED); + socket.send("late"); + expect(fake.sends).toEqual([]); + expect(fake.channels).toHaveLength(1); + }); + + it("reports the hub's close code, and its own close after the call", async () => { + const fake = fakeChannelApi(); + const ended = fake.transport.socket("/vendor/serve-sim/helper/ws?device=A"); + const endedClose = vi.fn(); + ended.addEventListener("close", endedClose); + fake.channels[0]!.emit({ _tag: "opened" }); + fake.channels[0]!.emit({ _tag: "end", code: 4000, reason: "bye" }); + expect(endedClose).toHaveBeenCalledWith({ code: 4000, reason: "bye", wasClean: true }); + expect(fake.channels[0]!.stopped).toBe(true); + + const own = fake.transport.socket("/vendor/serve-sim/helper/ws?device=A"); + const ownClose = vi.fn(); + own.addEventListener("close", ownClose); + own.close(); + expect(ownClose).not.toHaveBeenCalled(); + expect(fake.channels[1]!.stopped).toBe(true); + await Promise.resolve(); + expect(ownClose).toHaveBeenCalledWith({ code: 1000, reason: "", wasClean: true }); + }); +}); diff --git a/packages/client-runtime/src/device/hubTransport.ts b/packages/client-runtime/src/device/hubTransport.ts new file mode 100644 index 000000000..76140c7d6 --- /dev/null +++ b/packages/client-runtime/src/device/hubTransport.ts @@ -0,0 +1,486 @@ +// @effect-diagnostics globalTimers:off - This browser transport runs without an Effect runtime, like the stream client it serves. +/** + * The Device tab's way to the device hub (docs/design/devices.md, "Transport"). + * + * Saved computers and the relay carry only the app's WS RPC connection, so the + * viewer cannot fetch hub URLs or open sockets to them. This turns the device + * channel RPCs into the two things the stream client needs: `fetch`, answering + * a real `Response` whose body streams, and `socket`, a `WebSocket`-shaped + * object. Paths are hub paths (`/vendor/serve-sim/...`) with an optional query. + * + * Flow control: a chunk is credited once its reader has taken it (a body read, + * or its message listeners returning), never on arrival, so a stalled tab holds the hub + * up instead of filling the server. Channels never resubscribe: when one + * fails or the connection drops, the request errors or the socket closes, and + * the caller opens a fresh one. + */ +import type { + DeviceChannelCredit, + DeviceChannelEvent, + DeviceChannelOpen, + DeviceChannelSend, +} from "@threadlines/contracts"; + +import { base64ToBytes, bytesToBase64 } from "./base64.ts"; + +/** The three device channel RPCs, as the environment API exposes them. */ +export interface DeviceChannelApi { + /** Open until the returned call. `onClosed` comes once when it ends any other way. */ + readonly open: ( + input: DeviceChannelOpen, + listener: (event: DeviceChannelEvent) => void, + onClosed: (failure?: unknown) => void, + ) => () => void; + readonly send: (input: DeviceChannelSend) => Promise; + readonly credit: (input: DeviceChannelCredit) => Promise; +} + +export interface DeviceHubRequestInit { + readonly method?: "GET" | "POST"; + readonly body?: Uint8Array | string; + readonly contentType?: string; + readonly signal?: AbortSignal; +} + +export const DEVICE_SOCKET_CONNECTING = 0; +export const DEVICE_SOCKET_OPEN = 1; +export const DEVICE_SOCKET_CLOSING = 2; +export const DEVICE_SOCKET_CLOSED = 3; + +export interface DeviceHubSocketCloseEvent { + readonly code: number; + readonly reason: string; + readonly wasClean: boolean; +} + +export interface DeviceHubSocketEventMap { + readonly open: Record; + readonly message: { readonly data: ArrayBuffer | string }; + readonly close: DeviceHubSocketCloseEvent; + readonly error: { readonly error: unknown }; +} + +type DeviceHubSocketListener = ( + event: DeviceHubSocketEventMap[K], +) => void; + +/** The part of `WebSocket` the stream client uses. Messages are `ArrayBuffer` or text. */ +export interface DeviceHubSocket { + readonly binaryType: "arraybuffer"; + readonly readyState: number; + readonly protocol: string; + readonly addEventListener: ( + type: K, + listener: DeviceHubSocketListener, + ) => void; + readonly removeEventListener: ( + type: K, + listener: DeviceHubSocketListener, + ) => void; + /** Dropped unless open: input from before a reconnect never reaches the new session. */ + readonly send: (data: string | ArrayBuffer | ArrayBufferView) => void; + readonly close: (code?: number, reason?: string) => void; +} + +export interface DeviceHubTransport { + readonly fetch: (path: string, init?: DeviceHubRequestInit) => Promise; + readonly socket: (path: string, protocols?: ReadonlyArray) => DeviceHubSocket; +} + +/** Credits go out in batches of at least this, or after `CREDIT_DELAY_MS`. */ +const CREDIT_BATCH_BYTES = 64 * 1024; +const CREDIT_DELAY_MS = 40; +const NULL_BODY_STATUSES = new Set([101, 103, 204, 205, 304]); + +const textEncoder = new TextEncoder(); + +/** A random channel id; `crypto.randomUUID` is missing over plain-http LAN origins. */ +function newChannelId(): string { + const bytes = globalThis.crypto.getRandomValues(new Uint8Array(12)); + return `dc-${Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("")}`; +} + +/** Splits `/a/b?x=1` into the hub path and its query, as the channel takes them. */ +export function splitHubPath(path: string): { + readonly path: string; + readonly query?: Record; +} { + const mark = path.indexOf("?"); + if (mark < 0) return { path }; + const query: Record = {}; + for (const [key, value] of new URLSearchParams(path.slice(mark + 1))) query[key] = value; + return Object.keys(query).length === 0 + ? { path: path.slice(0, mark) } + : { path: path.slice(0, mark), query }; +} + +/** Why a channel failed, as an `Error` the caller can show. */ +function channelFailure(failure: unknown): Error { + if (failure instanceof Error) return failure; + const detail = (failure as { detail?: unknown } | null)?.detail; + return new Error(typeof detail === "string" ? detail : "The device connection closed."); +} + +function abortReason(signal: AbortSignal): unknown { + return signal.reason ?? new DOMException("The request was aborted.", "AbortError"); +} + +function createCreditor(api: DeviceChannelApi, channelId: string) { + let owed = 0; + let timer: ReturnType | null = null; + let closed = false; + const flush = () => { + if (timer !== null) clearTimeout(timer); + timer = null; + if (closed || owed === 0) return; + const bytes = owed; + owed = 0; + // A credit lost with the connection matters to nobody: the channel went too. + void api.credit({ channelId, bytes }).catch(() => undefined); + }; + return { + take(bytes: number) { + if (closed || bytes <= 0) return; + owed += bytes; + if (owed >= CREDIT_BATCH_BYTES) flush(); + else timer ??= setTimeout(flush, CREDIT_DELAY_MS); + }, + close() { + closed = true; + if (timer !== null) clearTimeout(timer); + timer = null; + }, + }; +} + +/** The channel's unsubscribe, safe to call again. */ +function once(stop: () => void): () => void { + let called = false; + return () => { + if (called) return; + called = true; + stop(); + }; +} + +function toBytes(data: ArrayBuffer | ArrayBufferView): Uint8Array { + return data instanceof ArrayBuffer + ? new Uint8Array(data) + : new Uint8Array(data.buffer, data.byteOffset, data.byteLength); +} + +function joinParts(parts: ReadonlyArray): ArrayBuffer { + const total = parts.reduce((sum, part) => sum + part.byteLength, 0); + const joined = new Uint8Array(total); + let offset = 0; + for (const part of parts) { + joined.set(part, offset); + offset += part.byteLength; + } + return joined.buffer; +} + +export function createDeviceHubTransport( + api: DeviceChannelApi, + options: { readonly channelId?: () => string } = {}, +): DeviceHubTransport { + const nextChannelId = options.channelId ?? newChannelId; + + const fetchThrough = (path: string, init: DeviceHubRequestInit = {}): Promise => + new Promise((resolve, reject) => { + const { signal } = init; + if (signal?.aborted) { + reject(abortReason(signal)); + return; + } + const channelId = nextChannelId(); + const creditor = createCreditor(api, channelId); + const queue: Uint8Array[] = []; + let waiting: ReadableStreamDefaultController | null = null; + let ended = false; + let failed: unknown = null; + let opened = false; + let done = false; + let stop: () => void = () => {}; + + // Hands one chunk to a waiting reader; that read is what earns the credit. + const deliver = () => { + const controller = waiting; + if (controller === null) return; + const chunk = queue.shift(); + if (chunk !== undefined) { + waiting = null; + controller.enqueue(chunk); + creditor.take(chunk.byteLength); + return; + } + if (failed !== null) { + waiting = null; + controller.error(failed); + } else if (ended) { + waiting = null; + controller.close(); + } + }; + + const finish = () => { + if (done) return; + done = true; + creditor.close(); + signal?.removeEventListener("abort", onAbort); + stop(); + }; + + const fail = (reason: unknown) => { + if (!opened) { + finish(); + reject(reason); + return; + } + if (failed !== null || ended) return; + failed = reason; + queue.length = 0; + finish(); + deliver(); + }; + + function onAbort() { + if (signal) fail(abortReason(signal)); + } + signal?.addEventListener("abort", onAbort, { once: true }); + + const body = new ReadableStream( + { + pull(controller) { + waiting = controller; + deliver(); + }, + cancel() { + queue.length = 0; + ended = true; + finish(); + }, + }, + // Nothing is pulled ahead of a read, so a credit always means a reader took it. + { highWaterMark: 0 }, + ); + + const { path: hubPath, query } = splitHubPath(path); + const bodyBytes = + init.body === undefined + ? undefined + : typeof init.body === "string" + ? textEncoder.encode(init.body) + : init.body; + stop = once( + api.open( + { + _tag: "http", + channelId, + method: init.method ?? "GET", + path: hubPath, + ...(query ? { query } : {}), + ...(bodyBytes ? { body: bytesToBase64(bodyBytes) } : {}), + ...(init.contentType ? { contentType: init.contentType } : {}), + }, + (event) => { + if (done) return; + switch (event._tag) { + case "opened": { + if (opened) return; + const status = event.status ?? 200; + if (status < 200 || status > 599) { + fail(new Error(`The device hub answered with status ${status}.`)); + return; + } + opened = true; + const headers = new Headers(); + if (event.contentType) headers.set("content-type", event.contentType); + if (NULL_BODY_STATUSES.has(status)) { + resolve(new Response(null, { status, headers })); + ended = true; + finish(); + return; + } + resolve(new Response(body, { status, headers })); + return; + } + case "data": { + const bytes = base64ToBytes(event.data); + if (bytes.byteLength === 0) return; + queue.push(bytes); + deliver(); + return; + } + case "message": + return; + case "end": { + if (!opened) { + fail(new Error("The device hub closed the request before answering.")); + return; + } + ended = true; + finish(); + deliver(); + return; + } + } + }, + (failure) => { + if (done) return; + fail( + failure === undefined + ? new Error("The device connection closed.") + : channelFailure(failure), + ); + }, + ), + ); + // `open` can fail synchronously into `onClosed`; drop the channel it never kept. + if (done) stop(); + }); + + const socket = (path: string, protocols?: ReadonlyArray): DeviceHubSocket => { + const channelId = nextChannelId(); + const creditor = createCreditor(api, channelId); + let readyState = DEVICE_SOCKET_CONNECTING; + let protocol = ""; + let seq = 0; + let binaryParts: Uint8Array[] = []; + let textParts: string[] = []; + let partBytes = 0; + let stop: () => void = () => {}; + const listeners: { [K in keyof DeviceHubSocketEventMap]: Set> } = { + open: new Set(), + message: new Set(), + close: new Set(), + error: new Set(), + }; + const dispatch = ( + type: K, + event: DeviceHubSocketEventMap[K], + ) => { + // A copy: a listener may remove itself while this runs. + for (const listener of Array.from(listeners[type])) listener(event); + }; + + const closeWith = (event: DeviceHubSocketCloseEvent, error?: unknown) => { + if (readyState === DEVICE_SOCKET_CLOSED) return; + readyState = DEVICE_SOCKET_CLOSED; + creditor.close(); + binaryParts = []; + textParts = []; + stop(); + if (error !== undefined) dispatch("error", { error }); + dispatch("close", event); + }; + + const handle: DeviceHubSocket = { + binaryType: "arraybuffer", + get readyState() { + return readyState; + }, + get protocol() { + return protocol; + }, + addEventListener(type, listener) { + listeners[type].add(listener); + }, + removeEventListener(type, listener) { + listeners[type].delete(listener); + }, + send(data) { + if (readyState !== DEVICE_SOCKET_OPEN) return; + const binary = typeof data !== "string"; + const message: DeviceChannelSend = { + channelId, + seq: seq++, + binary, + data: binary ? bytesToBase64(toBytes(data)) : data, + }; + api.send(message).catch((failure: unknown) => { + closeWith( + { code: 1006, reason: channelFailure(failure).message, wasClean: false }, + failure, + ); + }); + }, + close(code = 1000, reason = "") { + if (readyState === DEVICE_SOCKET_CLOSED || readyState === DEVICE_SOCKET_CLOSING) return; + readyState = DEVICE_SOCKET_CLOSING; + creditor.close(); + stop(); + // Like a WebSocket, the close event follows the call instead of firing inside it. + queueMicrotask(() => closeWith({ code, reason, wasClean: true })); + }, + }; + + const { path: hubPath, query } = splitHubPath(path); + stop = once( + api.open( + { + _tag: "ws", + channelId, + path: hubPath, + ...(query ? { query } : {}), + ...(protocols && protocols.length > 0 ? { protocols: [...protocols] } : {}), + }, + (event) => { + if (readyState === DEVICE_SOCKET_CLOSED || readyState === DEVICE_SOCKET_CLOSING) return; + switch (event._tag) { + case "opened": + if (readyState !== DEVICE_SOCKET_CONNECTING) return; + readyState = DEVICE_SOCKET_OPEN; + protocol = event.protocol ?? ""; + dispatch("open", {}); + return; + case "message": { + if (event.binary) { + const bytes = base64ToBytes(event.data); + binaryParts.push(bytes); + partBytes += bytes.byteLength; + } else { + textParts.push(event.data); + partBytes += textEncoder.encode(event.data).byteLength; + } + if (!event.final) return; + const data = event.binary ? joinParts(binaryParts) : textParts.join(""); + const bytes = partBytes; + binaryParts = []; + textParts = []; + partBytes = 0; + try { + dispatch("message", { data }); + } finally { + // The listeners have taken the message (the decoder queued it), so it is credited. + creditor.take(bytes); + } + return; + } + case "data": + return; + case "end": + closeWith({ code: event.code ?? 1005, reason: event.reason ?? "", wasClean: true }); + return; + } + }, + (failure) => { + if (readyState === DEVICE_SOCKET_CLOSED) return; + const error = failure === undefined ? undefined : channelFailure(failure); + closeWith( + { + code: 1006, + reason: error?.message ?? "The device connection closed.", + wasClean: false, + }, + error ?? new Error("The device connection closed."), + ); + }, + ), + ); + if (readyState === DEVICE_SOCKET_CLOSED) stop(); + return handle; + }; + + return { fetch: fetchThrough, socket }; +} diff --git a/packages/client-runtime/src/device/index.ts b/packages/client-runtime/src/device/index.ts new file mode 100644 index 000000000..ed57105bd --- /dev/null +++ b/packages/client-runtime/src/device/index.ts @@ -0,0 +1,5 @@ +export * from "./frame.ts"; +export * from "./hubTransport.ts"; +export * from "./screenshot.ts"; +export * from "./stream.ts"; +export * from "../state/deviceState.ts"; diff --git a/packages/client-runtime/src/device/mjpeg.ts b/packages/client-runtime/src/device/mjpeg.ts new file mode 100644 index 000000000..4cb2b0a88 --- /dev/null +++ b/packages/client-runtime/src/device/mjpeg.ts @@ -0,0 +1,75 @@ +/** + * Splits a `multipart/x-mixed-replace` MJPEG body into JPEG frames, for + * painting serve-sim's MJPEG stream onto the canvas when this browser cannot + * decode H.264. A part's `Content-Length` says where it ends; a part without + * one ends at the JPEG end-of-image marker. + */ +const HEADER_END = [13, 10, 13, 10]; +const MAX_HEADER_BYTES = 8 * 1024; +const latin1 = new TextDecoder("latin1"); + +function indexOfSequence(buffer: Uint8Array, sequence: ReadonlyArray, from: number) { + outer: for (let index = from; index + sequence.length <= buffer.length; index += 1) { + for (let offset = 0; offset < sequence.length; offset += 1) { + if (buffer[index + offset] !== sequence[offset]) continue outer; + } + return index; + } + return -1; +} + +export class MultipartJpegParser { + private buffer = new Uint8Array(256 * 1024); + private length = 0; + /** Bytes of the current part's body, once its headers are read; null for "until EOI". */ + private expected: number | null | undefined = undefined; + + push(bytes: Uint8Array): Uint8Array[] { + if (this.length + bytes.length > this.buffer.length) { + let capacity = this.buffer.length; + while (capacity < this.length + bytes.length) capacity *= 2; + const grown = new Uint8Array(capacity); + grown.set(this.buffer.subarray(0, this.length)); + this.buffer = grown; + } + this.buffer.set(bytes, this.length); + this.length += bytes.length; + + const frames: Uint8Array[] = []; + const view = this.buffer.subarray(0, this.length); + let offset = 0; + for (;;) { + if (this.expected === undefined) { + const end = indexOfSequence(view, HEADER_END, offset); + if (end < 0) { + // A runaway header block is not a stream we can read; start over. + if (this.length - offset > MAX_HEADER_BYTES) offset = this.length; + break; + } + const headers = latin1.decode(view.subarray(offset, end)); + const length = /content-length:\s*(\d+)/i.exec(headers)?.[1]; + this.expected = length === undefined ? null : Number(length); + offset = end + HEADER_END.length; + continue; + } + let frameEnd: number; + if (this.expected === null) { + const eoi = indexOfSequence(view, [0xff, 0xd9], offset + 2); + if (eoi < 0) break; + frameEnd = eoi + 2; + } else { + if (this.length - offset < this.expected) break; + frameEnd = offset + this.expected; + } + const frame = view.slice(offset, frameEnd); + if (frame[0] === 0xff && frame[1] === 0xd8) frames.push(frame); + offset = frameEnd; + this.expected = undefined; + } + if (offset > 0) { + this.buffer.copyWithin(0, offset, this.length); + this.length -= offset; + } + return frames; + } +} diff --git a/packages/client-runtime/src/device/png.ts b/packages/client-runtime/src/device/png.ts new file mode 100644 index 000000000..149afbcc7 --- /dev/null +++ b/packages/client-runtime/src/device/png.ts @@ -0,0 +1,24 @@ +const PNG_SIGNATURE = [137, 80, 78, 71, 13, 10, 26, 10]; + +/** + * The first complete PNG in `bytes`, or undefined. `adb exec-out screencap -p` + * can print warnings (a foldable's several displays) around the image. + */ +export function extractPng( + bytes: Uint8Array, +): Uint8Array | undefined { + const start = bytes.findIndex((_, index) => + PNG_SIGNATURE.every((value, offset) => bytes[index + offset] === value), + ); + if (start < 0) return undefined; + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + for (let chunk = start + PNG_SIGNATURE.length; chunk + 12 <= bytes.length;) { + const end = chunk + 12 + view.getUint32(chunk); + if (end > bytes.length) return undefined; + if (String.fromCharCode(...bytes.subarray(chunk + 4, chunk + 8)) === "IEND") { + return bytes.subarray(start, end); + } + chunk = end; + } + return undefined; +} diff --git a/packages/client-runtime/src/device/screenshot.ts b/packages/client-runtime/src/device/screenshot.ts new file mode 100644 index 000000000..2e84f29ee --- /dev/null +++ b/packages/client-runtime/src/device/screenshot.ts @@ -0,0 +1,36 @@ +// @effect-diagnostics globalTimers:off - This browser transport runs without an Effect runtime, like the live stream. +import type { DevicePlatform } from "@threadlines/contracts"; + +import type { DeviceHubTransport } from "./hubTransport.ts"; +import { extractPng } from "./png.ts"; +import { deviceHubPaths } from "./stream.ts"; + +const SCREENSHOT_TIMEOUT_MS = 20_000; + +/** A PNG of the device's screen, through the hub's screenshot route. */ +export async function captureDeviceScreenshot( + transport: DeviceHubTransport, + target: { readonly platform: DevicePlatform; readonly deviceId: string }, +): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), SCREENSHOT_TIMEOUT_MS); + try { + const response = await transport.fetch( + deviceHubPaths.screenshot(target.platform, encodeURIComponent(target.deviceId)), + { method: "POST", signal: controller.signal }, + ); + if (!response.ok) { + await response.body?.cancel().catch(() => {}); + throw new Error(`The screenshot failed (${response.status}). Try again.`); + } + const png = extractPng(new Uint8Array(await response.arrayBuffer())); + if (!png) throw new Error("The device sent an incomplete screenshot. Try again."); + return new Blob([png as BlobPart], { type: "image/png" }); + } catch (cause) { + if (controller.signal.aborted) + throw new Error("The screenshot took too long. Try again.", { cause }); + throw cause; + } finally { + clearTimeout(timeout); + } +} diff --git a/packages/client-runtime/src/device/stream.test.ts b/packages/client-runtime/src/device/stream.test.ts new file mode 100644 index 000000000..547e34810 --- /dev/null +++ b/packages/client-runtime/src/device/stream.test.ts @@ -0,0 +1,218 @@ +import { afterEach, describe, expect, it, vi } from "vite-plus/test"; + +import { + DEVICE_SOCKET_CONNECTING, + DEVICE_SOCKET_OPEN, + type DeviceHubSocket, + type DeviceHubTransport, +} from "./hubTransport.ts"; +import { MultipartJpegParser } from "./mjpeg.ts"; +import { + AvccDemuxer, + avcCodecString, + createDeviceStreamClient, + parseSemuPacket, + STILL_FRAME_INTERVAL_MS, +} from "./stream.ts"; + +afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); +}); + +/** A hub reached without any channel: sockets the test opens, requests it answers. */ +function fakeTransport(answer: (path: string, method: string) => Response | Promise) { + const sockets: Array<{ + path: string; + socket: DeviceHubSocket; + sent: unknown[]; + dispatch: (type: string, event: unknown) => void; + }> = []; + const requests: Array<{ path: string; method: string }> = []; + const transport: DeviceHubTransport = { + fetch: async (path, init) => { + const method = init?.method ?? "GET"; + requests.push({ path, method }); + return answer(path, method); + }, + socket: (path) => { + const sent: unknown[] = []; + const listeners = new Map void>>(); + const dispatch = (type: string, event: unknown) => { + for (const listener of listeners.get(type) ?? []) (listener as (e: unknown) => void)(event); + }; + let readyState = DEVICE_SOCKET_CONNECTING; + const socket: DeviceHubSocket = { + binaryType: "arraybuffer", + get readyState() { + return readyState; + }, + protocol: "", + addEventListener: (type, listener) => { + const set = listeners.get(type) ?? new Set(); + set.add(listener as (event: never) => void); + listeners.set(type, set); + }, + removeEventListener: (type, listener) => { + listeners.get(type)?.delete(listener as (event: never) => void); + }, + send: (data) => sent.push(data), + close: () => { + readyState = 3; + }, + }; + sockets.push({ path, socket, sent, dispatch }); + queueMicrotask(() => { + readyState = DEVICE_SOCKET_OPEN; + dispatch("open", {}); + }); + return socket; + }, + }; + return { transport, sockets, requests }; +} + +const events = () => ({ + onStatus: vi.fn(), + onScreen: vi.fn(), + onInputConnected: vi.fn(), + onVideoMode: vi.fn(), +}); + +const PNG = new Uint8Array([ + 137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130, +]); + +describe("AvccDemuxer", () => { + it("reassembles tagged envelopes split across reads", () => { + const demuxer = new AvccDemuxer(); + const envelope = new Uint8Array([0, 0, 0, 5, 1, 1, 0x64, 0, 0x1f, 0, 0, 0, 2, 2, 9]); + expect(demuxer.push(envelope.subarray(0, 6))).toEqual([]); + const chunks = demuxer.push(envelope.subarray(6)); + expect(chunks.map((chunk) => chunk.type)).toEqual(["description", "keyframe"]); + expect(avcCodecString(chunks[0]!.payload)).toBe("avc1.64001f"); + expect(chunks[1]!.payload).toEqual(new Uint8Array([9])); + }); +}); + +describe("parseSemuPacket", () => { + it("strips serve-emu's header and reads the keyframe flag and timestamp", () => { + const packet = new Uint8Array(20); + const view = new DataView(packet.buffer); + view.setUint32(0, 0x53454d55); + view.setUint8(4, 1); + view.setUint8(5, 1); + view.setBigUint64(8, 42n); + packet.set([7, 7, 7, 7], 16); + const parsed = parseSemuPacket(packet.buffer); + expect(parsed.isKey).toBe(true); + expect(parsed.timestamp).toBe(42); + expect(parsed.data).toEqual(new Uint8Array([7, 7, 7, 7])); + }); +}); + +describe("MultipartJpegParser", () => { + it("splits serve-sim's MJPEG parts by Content-Length across reads", () => { + const jpeg = (fill: number) => new Uint8Array([0xff, 0xd8, fill, 0xff, 0xd9]); + const part = (body: Uint8Array) => { + const head = new TextEncoder().encode( + `--frame\r\nContent-Type: image/jpeg\r\nContent-Length: ${body.length}\r\n\r\n`, + ); + const out = new Uint8Array(head.length + body.length + 2); + out.set(head); + out.set(body, head.length); + out.set([13, 10], head.length + body.length); + return out; + }; + const stream = new Uint8Array([...part(jpeg(1)), ...part(jpeg(2))]); + const parser = new MultipartJpegParser(); + const frames = [...parser.push(stream.subarray(0, 30)), ...parser.push(stream.subarray(30))]; + expect(frames).toEqual([jpeg(1), jpeg(2)]); + }); +}); + +describe("createDeviceStreamClient", () => { + it("sends iOS keys as HID usages only once the helper admitted input", async () => { + // Bodies that send one chunk and stay open, like the hub's streams. + const hub = fakeTransport( + () => + new Response( + new ReadableStream({ + start: (controller) => controller.enqueue(new Uint8Array([0])), + }), + ), + ); + const client = createDeviceStreamClient( + { platform: "ios", deviceId: "SIM 1", transport: hub.transport }, + { present: () => true }, + events(), + ); + client.start(); + await vi.waitFor(() => expect(hub.sockets).toHaveLength(1)); + const input = hub.sockets[0]!; + expect(input.path).toBe("/vendor/serve-sim/helper/ws?device=SIM%201"); + expect(hub.requests.map((request) => request.path)).toContain( + "/vendor/serve-sim/helper/SIM%201/stream.mjpeg", + ); + + const key = { code: "KeyA", key: "a" } as KeyboardEvent; + client.sendKey(key, "down"); + expect(input.sent).toEqual([]); + + input.dispatch("message", { data: new Uint8Array([0x83]).buffer }); + client.sendKey(key, "down"); + expect(input.sent).toHaveLength(1); + const packet = input.sent[0] as Uint8Array; + expect(packet[0]).toBe(0x06); + expect(JSON.parse(new TextDecoder().decode(packet.subarray(1)))).toEqual({ + type: "down", + usage: 0x04, + }); + client.stop(); + }); + + it("shows Android stills about four times a second without WebCodecs, with touch still live", async () => { + vi.useFakeTimers(); + vi.stubGlobal( + "createImageBitmap", + vi.fn(async () => ({ width: 1080, height: 2400, close: vi.fn() })), + ); + const hub = fakeTransport(() => new Response(PNG)); + const present = vi.fn(() => true); + const watch = events(); + const client = createDeviceStreamClient( + { platform: "android", deviceId: "emulator-5554", transport: hub.transport }, + { present }, + watch, + ); + client.start(); + expect(watch.onVideoMode).toHaveBeenCalledWith("stills"); + await vi.advanceTimersByTimeAsync(0); + expect(hub.sockets[0]?.path).toBe("/vendor/serve-emu/ws?device=emulator-5554&video=0"); + expect(hub.requests[0]).toEqual({ + path: "/vendor/serve-emu/api/screenshot?device=emulator-5554", + method: "POST", + }); + expect(present).toHaveBeenCalledWith(expect.anything(), 1080, 2400); + expect(watch.onStatus).toHaveBeenLastCalledWith("streaming", undefined); + expect(watch.onScreen).toHaveBeenCalledWith({ + width: 1080, + height: 2400, + orientation: "portrait", + }); + + await vi.advanceTimersByTimeAsync(STILL_FRAME_INTERVAL_MS * 4); + expect(hub.requests.length).toBeGreaterThanOrEqual(4); + expect(hub.requests.length).toBeLessThanOrEqual(6); + + client.sendTouch("begin", 0.5, 0.25); + expect(hub.sockets[0]!.sent).toEqual([ + JSON.stringify({ type: "touch", action: "down", x: 0.5, y: 0.25 }), + ]); + + client.stop(); + const after = hub.requests.length; + await vi.advanceTimersByTimeAsync(STILL_FRAME_INTERVAL_MS * 4); + expect(hub.requests.length).toBe(after); + }); +}); diff --git a/packages/client-runtime/src/device/stream.ts b/packages/client-runtime/src/device/stream.ts new file mode 100644 index 000000000..74cc31a7b --- /dev/null +++ b/packages/client-runtime/src/device/stream.ts @@ -0,0 +1,1019 @@ +// @effect-diagnostics globalTimers:off - This browser stream client runs without an Effect runtime. +/** + * One device's live screen and input (docs/design/devices.md, "Transport"). + * + * Ported from T3 Code's stream client (MIT, see NOTICE) with the 3D, Duo and + * WebRTC paths removed, and every byte carried by a `DeviceHubTransport` + * instead of hub URLs: + * + * - iOS: serve-sim's AVCC stream decoded with WebCodecs, input over its helper + * socket. Without WebCodecs (or a profile this browser cannot decode) its + * MJPEG stream is fetched and each JPEG painted to the canvas. + * - Android: serve-emu's socket carries H.264 and takes input. Without + * WebCodecs the socket carries input only, and the screen is the + * screenshot route read about four times a second. + * + * Each retry opens a fresh request or socket and a fresh decoder that waits + * for a keyframe; input that had not gone out is dropped, never replayed. + */ +import type { DevicePlatform } from "@threadlines/contracts"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; + +import { createCanvasFrameSink, type DeviceFrameSink } from "./frame.ts"; +import { + DEVICE_SOCKET_OPEN, + type DeviceHubSocket, + type DeviceHubTransport, +} from "./hubTransport.ts"; +import { MultipartJpegParser } from "./mjpeg.ts"; +import { extractPng } from "./png.ts"; + +export type DeviceStreamStatus = "connecting" | "streaming" | "error"; + +/** How the screen reaches the canvas: decoded video, MJPEG frames, or polled stills. */ +export type DeviceVideoMode = "video" | "mjpeg" | "stills"; + +export type DeviceOrientationName = + | "portrait" + | "portrait_upside_down" + | "landscape_left" + | "landscape_right"; + +export interface DeviceScreenSize { + readonly width: number; + readonly height: number; + readonly orientation: DeviceOrientationName; +} + +const screenConfigSchema = Schema.Struct({ + width: Schema.Finite.check(Schema.isGreaterThan(0)), + height: Schema.Finite.check(Schema.isGreaterThan(0)), + orientation: Schema.Literals([ + "portrait", + "portrait_upside_down", + "landscape_left", + "landscape_right", + ]), + inputUnavailable: Schema.optionalKey(Schema.Boolean), +}); +const decodeScreenConfig = Schema.decodeUnknownOption(screenConfigSchema); + +export interface DeviceStreamEvents { + readonly onStatus: (status: DeviceStreamStatus, detail?: string) => void; + readonly onScreen: (screen: DeviceScreenSize) => void; + readonly onInputConnected: (connected: boolean, detail?: string) => void; + readonly onVideoMode?: (mode: DeviceVideoMode) => void; +} + +export interface DeviceStreamTarget { + readonly platform: DevicePlatform; + readonly deviceId: string; + readonly transport: DeviceHubTransport; +} + +export type DeviceHardwareButton = "home" | "back" | "recents" | "power" | "appSwitcher"; + +const RETRY_DELAY_MS = 1_000; +const FIRST_FRAME_TIMEOUT_MS = 15_000; +const PRIME_TIMEOUT_MS = 2_000; +/** Android without WebCodecs: one screenshot at a time, at most this often. */ +export const STILL_FRAME_INTERVAL_MS = 250; +const STILL_FRAME_TIMEOUT_MS = 5_000; +const STILL_FRAME_MAX_FAILURES = 5; +const FRAME_DURATION_US = 16_667; +const SEMU_MAGIC = 0x53454d55; +const SEMU_HEADER_BYTES = 16; +const SEMU_FLAG_KEY = 1; +const SOFT_DECODE_QUEUE = 8; +// expo-device-hub serve-emu input.js MAX_TEXT_BYTES limits packets to 300 UTF-8 bytes. +const ANDROID_MAX_TEXT_BYTES = 300; +// The hub's queue depth is 128; the client does not read queue rejections. +const ANDROID_MAX_TEXT_CHUNKS = 64; + +const IOS_MSG_TOUCH = 0x03; +const IOS_MSG_BUTTON = 0x04; +const IOS_MSG_KEY = 0x06; +const IOS_MSG_ORIENTATION = 0x07; +const IOS_TAG_SCREEN_CONFIG = 0x82; +const IOS_TAG_INPUT_ADMITTED = 0x83; + +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +const isWebCodecsSupported = (): boolean => + typeof globalThis !== "undefined" && + "VideoDecoder" in globalThis && + "EncodedVideoChunk" in globalThis; + +function utf8ByteLength(codePoint: number): number { + if (codePoint <= 0x7f) return 1; + if (codePoint <= 0x7ff) return 2; + if (codePoint <= 0xffff) return 3; + return 4; +} + +function taggedJson(tag: number, payload: unknown): Uint8Array { + const json = encoder.encode(JSON.stringify(payload)); + const out = new Uint8Array(1 + json.length); + out[0] = tag; + out.set(json, 1); + return out; +} + +export function avcCodecString(bytes: Uint8Array): string { + if (bytes.length < 4) return "avc1.42E01E"; + const hex = (byte: number) => byte.toString(16).padStart(2, "0"); + return `avc1.${hex(bytes[1]!)}${hex(bytes[2]!)}${hex(bytes[3]!)}`; +} + +export function parseSemuPacket(raw: ArrayBuffer): { + readonly data: Uint8Array; + readonly isKey: boolean | null; + readonly timestamp: number | null; +} { + const bytes = new Uint8Array(raw); + if (bytes.byteLength > SEMU_HEADER_BYTES) { + const view = new DataView(raw, 0, SEMU_HEADER_BYTES); + if (view.getUint32(0, false) === SEMU_MAGIC && view.getUint8(4) === 1) { + const pts = view.getBigUint64(8, false); + return { + data: bytes.subarray(SEMU_HEADER_BYTES), + isKey: (view.getUint8(5) & SEMU_FLAG_KEY) !== 0, + timestamp: pts <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(pts) : null, + }; + } + } + return { data: bytes, isKey: null, timestamp: null }; +} + +const isVideoSessionMessage = (text: string) => { + try { + const message = JSON.parse(text) as { type?: unknown }; + return message.type === "video-session"; + } catch { + return false; + } +}; + +export function scanAccessUnit(buf: Uint8Array): { isKey: boolean; sps: Uint8Array | null } { + let isKey = false; + let sps: Uint8Array | null = null; + const len = buf.length; + let i = 0; + while (i + 2 < len) { + if (buf[i] === 0 && buf[i + 1] === 0) { + let codeLen = 0; + if (buf[i + 2] === 1) codeLen = 3; + else if (i + 3 < len && buf[i + 2] === 0 && buf[i + 3] === 1) codeLen = 4; + if (codeLen) { + const nalType = buf[i + codeLen]! & 0x1f; + if (nalType === 7 && !sps) sps = buf.subarray(i + codeLen); + if (nalType === 5) isKey = true; + i += codeLen + 1; + continue; + } + } + i++; + } + return { isKey, sps }; +} + +export type AvccChunk = { + readonly type: "description" | "keyframe" | "delta" | "seed"; + readonly payload: Uint8Array; +}; + +const AVCC_TAGS: Record = { + 1: "description", + 2: "keyframe", + 3: "delta", + 4: "seed", +}; + +/** Splits serve-sim's length-prefixed, tagged AVCC stream into chunks. */ +export class AvccDemuxer { + private buffer = new Uint8Array(64 * 1024); + private length = 0; + + push(bytes: Uint8Array): AvccChunk[] { + if (this.length + bytes.length > this.buffer.length) { + let capacity = this.buffer.length; + while (capacity < this.length + bytes.length) capacity *= 2; + const grown = new Uint8Array(capacity); + grown.set(this.buffer.subarray(0, this.length)); + this.buffer = grown; + } + this.buffer.set(bytes, this.length); + this.length += bytes.length; + + const chunks: AvccChunk[] = []; + let offset = 0; + while (this.length - offset >= 4) { + const view = new DataView(this.buffer.buffer, this.buffer.byteOffset + offset, 4); + const frameLength = view.getUint32(0, false); + if (this.length - offset - 4 < frameLength) break; + if (frameLength >= 1) { + const type = AVCC_TAGS[this.buffer[offset + 4]!]; + if (type) { + chunks.push({ type, payload: this.buffer.slice(offset + 5, offset + 4 + frameLength) }); + } + } + offset += 4 + frameLength; + } + if (offset > 0) { + this.buffer.copyWithin(0, offset, this.length); + this.length -= offset; + } + return chunks; + } +} + +export interface DeviceStreamClient { + readonly start: () => void; + readonly stop: () => void; + /** Normalized 0..1 coordinates in the displayed frame. */ + readonly sendTouch: (phase: "begin" | "move" | "end", x: number, y: number) => void; + readonly sendKey: (event: KeyboardEvent, phase: "down" | "up") => void; + /** Android only: iOS input is HID key codes, which cannot carry arbitrary text. */ + readonly sendText: (text: string) => void; + readonly pressButton: (button: DeviceHardwareButton) => void; + /** iOS only; Android rotates through the Tools drawer. */ + readonly rotate: () => void; +} + +const HID_USAGE_BY_CODE: Readonly> = { + Enter: 0x28, + Escape: 0x29, + Backspace: 0x2a, + Tab: 0x2b, + Space: 0x2c, + Minus: 0x2d, + Equal: 0x2e, + BracketLeft: 0x2f, + BracketRight: 0x30, + Backslash: 0x31, + Semicolon: 0x33, + Quote: 0x34, + Backquote: 0x35, + Comma: 0x36, + Period: 0x37, + Slash: 0x38, + Delete: 0x4c, + ArrowRight: 0x4f, + ArrowLeft: 0x50, + ArrowDown: 0x51, + ArrowUp: 0x52, + NumpadDivide: 0x54, + NumpadMultiply: 0x55, + NumpadSubtract: 0x56, + NumpadAdd: 0x57, + NumpadEnter: 0x58, + Numpad1: 0x59, + Numpad2: 0x5a, + Numpad3: 0x5b, + Numpad4: 0x5c, + Numpad5: 0x5d, + Numpad6: 0x5e, + Numpad7: 0x5f, + Numpad8: 0x60, + Numpad9: 0x61, + Numpad0: 0x62, + NumpadDecimal: 0x63, + NumpadEqual: 0x67, + ControlLeft: 0xe0, + ShiftLeft: 0xe1, + AltLeft: 0xe2, + MetaLeft: 0xe3, + ControlRight: 0xe4, + ShiftRight: 0xe5, + AltRight: 0xe6, + MetaRight: 0xe7, +}; + +function hidUsageForCode(code: string): number | null { + if (/^Key[A-Z]$/.test(code)) return 0x04 + (code.charCodeAt(3) - 65); + if (/^Digit[1-9]$/.test(code)) return 0x1e + (code.charCodeAt(5) - 49); + if (code === "Digit0") return 0x27; + return HID_USAGE_BY_CODE[code] ?? null; +} + +const ANDROID_KEYCODE_BY_KEY: Readonly> = { + ArrowUp: 19, + ArrowDown: 20, + ArrowLeft: 21, + ArrowRight: 22, + Tab: 61, + Enter: 66, + Backspace: 67, + Delete: 112, + Home: 122, + End: 123, + PageUp: 92, + PageDown: 93, +}; + +const IOS_ORIENTATIONS: ReadonlyArray = [ + "portrait", + "landscape_left", + "portrait_upside_down", + "landscape_right", +]; + +/** The hub paths the viewer uses, so the server's allowlist can be checked against one list. */ +export const deviceHubPaths = { + iosVideo: (device: string) => `/vendor/serve-sim/helper/${device}/stream.avcc`, + iosMjpeg: (device: string) => `/vendor/serve-sim/helper/${device}/stream.mjpeg`, + iosInput: (device: string) => `/vendor/serve-sim/helper/ws?device=${device}`, + androidSocket: (device: string, inputOnly: boolean) => + `/vendor/serve-emu/ws?device=${device}&${inputOnly ? "video=0" : "frame-meta=1"}`, + screenshot: (platform: DevicePlatform, device: string) => + `/vendor/${platform === "ios" ? "serve-sim" : "serve-emu"}/api/screenshot?device=${device}`, +}; + +export function createDeviceStreamClient( + target: DeviceStreamTarget, + output: HTMLCanvasElement | DeviceFrameSink, + events: DeviceStreamEvents, +): DeviceStreamClient { + const { platform, deviceId, transport } = target; + const sink = "present" in output ? output : createCanvasFrameSink(output); + const ios = platform === "ios"; + const device = encodeURIComponent(deviceId); + const useWebCodecs = isWebCodecsSupported(); + + let stopped = true; + let socket: DeviceHubSocket | null = null; + let controller: AbortController | null = null; + const retryTimers = new Map<"video" | "input", ReturnType>(); + let primeController: AbortController | null = null; + let stillsTimer: ReturnType | null = null; + let videoDecoder: VideoDecoder | null = null; + let decodedVideoGeneration: number | null = null; + let iosInputAdmitted = false; + let iosInputUnavailable = false; + let timestamp = 0; + let awaitingKeyframe = true; + let screen: DeviceScreenSize | null = null; + let firstFrame = false; + let configuring = false; + let mode: DeviceVideoMode = "video"; + let generation = 0; + let decoderEpoch = 0; + let frameTimer: ReturnType | null = null; + let videoReadTimer: ReturnType | null = null; + let videoGeneration = 0; + + const setStatus = (status: DeviceStreamStatus, detail?: string) => { + if (!stopped) events.onStatus(status, detail); + }; + + const setMode = (next: DeviceVideoMode) => { + if (mode === next) return; + mode = next; + events.onVideoMode?.(next); + }; + + const clearFrameTimer = () => { + if (frameTimer !== null) clearTimeout(frameTimer); + frameTimer = null; + }; + + const fail = (detail: string) => { + if (stopped) return; + stop(); + events.onInputConnected(false, detail); + events.onStatus("error", detail); + }; + + const connecting = (detail?: string) => { + firstFrame = false; + if (frameTimer === null) { + frameTimer = setTimeout( + () => fail("No video came from the device. Reconnect to try again."), + FIRST_FRAME_TIMEOUT_MS, + ); + } + setStatus("connecting", detail); + }; + + const frameReceived = () => { + if (firstFrame) return; + firstFrame = true; + clearFrameTimer(); + setStatus("streaming"); + }; + + const paint = (source: CanvasImageSource, width: number, height: number) => { + if (stopped) return; + if (!ios && (screen?.width !== width || screen.height !== height)) { + screen = { width, height, orientation: width > height ? "landscape_left" : "portrait" }; + events.onScreen(screen); + } + if (!sink.present(source, width, height)) { + fail("Could not show the device stream. Reconnect to try again."); + return; + } + frameReceived(); + }; + + /** Decodes an image and paints it, unless a newer frame or a stop came first. */ + const paintImage = async (blob: Blob, isCurrent: () => boolean) => { + const bitmap = await createImageBitmap(blob); + try { + if (isCurrent()) paint(bitmap, bitmap.width, bitmap.height); + } finally { + bitmap.close(); + } + }; + + const closeDecoder = () => { + decoderEpoch++; + try { + videoDecoder?.close(); + } catch {} + videoDecoder = null; + awaitingKeyframe = true; + }; + + const recoverDecoder = () => { + if (ios) fallBackToMjpeg(); + else { + closeDecoder(); + connecting("Video decoder restarted."); + requestKeyframe(); + } + }; + + const makeDecoder = () => { + const feedGeneration = videoGeneration; + const created = new VideoDecoder({ + output: (frame) => { + try { + if (videoDecoder === created && (!ios || feedGeneration === videoGeneration)) { + decodedVideoGeneration = feedGeneration; + paint(frame, frame.displayWidth, frame.displayHeight); + } + } finally { + frame.close(); + } + }, + error: () => { + if (stopped || videoDecoder !== created || (ios && feedGeneration !== videoGeneration)) + return; + recoverDecoder(); + }, + }); + return created; + }; + + const configureDecoder = async ( + config: VideoDecoderConfig, + isCurrent = () => !stopped, + ): Promise => { + const epoch = decoderEpoch; + const full: VideoDecoderConfig = { ...config, optimizeForLatency: true }; + const support = await VideoDecoder.isConfigSupported(full).catch(() => ({ supported: false })); + if (!isCurrent() || epoch !== decoderEpoch) return false; + if (!support.supported) { + if (!ios) fail(`This browser cannot decode ${config.codec}.`); + return false; + } + try { + if (!videoDecoder || videoDecoder.state === "closed") videoDecoder = makeDecoder(); + videoDecoder.configure(full); + return true; + } catch (cause) { + if (!ios) fail(`Video decoder: ${(cause as Error).message}`); + return false; + } + }; + + const decode = (isKey: boolean, data: Uint8Array, pts?: number | null) => { + if (!videoDecoder || videoDecoder.state !== "configured") return; + if (awaitingKeyframe) { + if (!isKey) return; + awaitingKeyframe = false; + } + if (videoDecoder.decodeQueueSize > SOFT_DECODE_QUEUE) { + if (!ios) { + recoverDecoder(); + return; + } + // Native IDRs are spaced by frame count, so an idle simulator may take + // minutes to produce one. A new AVCC request forces a fresh IDR. + controller?.abort(); + closeDecoder(); + return; + } + try { + videoDecoder.decode( + new EncodedVideoChunk({ + type: isKey ? "key" : "delta", + timestamp: pts ?? timestamp, + data, + }), + ); + timestamp += FRAME_DURATION_US; + } catch { + recoverDecoder(); + } + }; + + const requestKeyframe = () => { + if (!ios && socket?.readyState === DEVICE_SOCKET_OPEN) { + socket.send(JSON.stringify({ type: "reset-video", ack: false })); + } + }; + + const scheduleRetry = (channel: "video" | "input", run: () => void) => { + if (stopped || retryTimers.has(channel)) return; + retryTimers.set( + channel, + setTimeout(() => { + retryTimers.delete(channel); + run(); + }, RETRY_DELAY_MS), + ); + }; + + /** + * Reads one HTTP body to its end, with a watchdog: a stream that stops + * producing for the first-frame timeout is aborted and retried. + */ + const readBody = async ( + path: string, + videoController: AbortController, + isCurrent: () => boolean, + onChunk: (bytes: Uint8Array) => Promise | boolean | void, + ): Promise => { + let retryDetail: string | undefined; + try { + const response = await transport.fetch(path, { signal: videoController.signal }); + if (!isCurrent()) { + await response.body?.cancel().catch(() => {}); + return; + } + if (!response.ok || !response.body) throw new Error(`Device stream ${response.status}`); + const reader = response.body.getReader(); + for (;;) { + const timer = setTimeout(() => { + if (!isCurrent()) return; + retryDetail = "The device stream stopped sending video. Reconnecting…"; + videoController.abort(); + }, FIRST_FRAME_TIMEOUT_MS); + videoReadTimer = timer; + let result: ReadableStreamReadResult; + try { + result = await reader.read(); + } finally { + clearTimeout(timer); + if (videoReadTimer === timer) videoReadTimer = null; + } + if (!isCurrent()) return; + if (result.done) break; + if ((await onChunk(result.value)) === false) { + await reader.cancel().catch(() => {}); + return; + } + if (!isCurrent()) return; + } + } catch (cause) { + if (!isCurrent()) return; + retryDetail ??= videoController.signal.aborted + ? "The video decoder fell behind. Reconnecting…" + : (cause as Error).message; + } + return retryDetail ?? "The device stream ended. Reconnecting…"; + }; + + const readIosVideo = async () => { + const lifecycle = generation; + const feedGeneration = ++videoGeneration; + const demuxer = new AvccDemuxer(); + const videoController = new AbortController(); + controller = videoController; + const isCurrent = () => + !stopped && + mode === "video" && + generation === lifecycle && + videoGeneration === feedGeneration && + controller === videoController; + const retryDetail = await readBody( + deviceHubPaths.iosVideo(device), + videoController, + isCurrent, + async (bytes) => { + for (const chunk of demuxer.push(bytes)) { + if (videoController.signal.aborted) return; + switch (chunk.type) { + case "seed": + // A JPEG of the current screen, shown until the first decoded frame. + void paintImage( + new Blob([chunk.payload as BlobPart], { type: "image/jpeg" }), + () => isCurrent() && decodedVideoGeneration !== feedGeneration, + ).catch(() => {}); + break; + case "description": { + awaitingKeyframe = true; + const configured = await configureDecoder( + { codec: avcCodecString(chunk.payload), description: chunk.payload }, + isCurrent, + ); + if (!isCurrent()) return false; + if (!configured) { + fallBackToMjpeg(); + return false; + } + break; + } + case "keyframe": + case "delta": + decode(chunk.type === "keyframe", chunk.payload); + break; + } + } + }, + ); + if (retryDetail !== undefined && isCurrent()) { + controller = null; + videoController.abort(); + closeDecoder(); + connecting(retryDetail); + scheduleRetry("video", () => void readIosVideo()); + } + }; + + /** serve-sim's MJPEG stream, each JPEG painted to the canvas as it lands. */ + const readIosMjpeg = async () => { + const lifecycle = generation; + const videoController = new AbortController(); + controller = videoController; + const parser = new MultipartJpegParser(); + let painting = false; + let latest: Uint8Array | null = null; + const isCurrent = () => + !stopped && mode === "mjpeg" && generation === lifecycle && controller === videoController; + // Decoding is slower than arrival on a busy machine: keep only the newest frame waiting. + const paintLatest = () => { + if (painting || latest === null) return; + const frame = latest; + latest = null; + painting = true; + void paintImage(new Blob([frame as BlobPart], { type: "image/jpeg" }), isCurrent) + .catch(() => {}) + .finally(() => { + painting = false; + paintLatest(); + }); + }; + const retryDetail = await readBody( + deviceHubPaths.iosMjpeg(device), + videoController, + isCurrent, + (bytes) => { + const frames = parser.push(bytes); + if (frames.length > 0) { + latest = frames.at(-1)!; + paintLatest(); + } + }, + ); + if (retryDetail !== undefined && isCurrent()) { + controller = null; + videoController.abort(); + connecting(retryDetail); + scheduleRetry("video", () => void readIosMjpeg()); + } + }; + + const fallBackToMjpeg = () => { + if (stopped || mode === "mjpeg") return; + setMode("mjpeg"); + controller?.abort(); + controller = null; + closeDecoder(); + connecting(); + void readIosMjpeg(); + }; + + /** + * serve-sim's helper only accepts HID and pushes its screen config once + * screen capture is running, and the AVCC stream does not reliably start + * it. Touching the MJPEG endpoint does; one aborted request is enough. + */ + const primeIosHelper = async (session: number) => { + const prime = new AbortController(); + primeController = prime; + const timeout = setTimeout(() => prime.abort(), PRIME_TIMEOUT_MS); + try { + const response = await transport.fetch(deviceHubPaths.iosMjpeg(device), { + signal: prime.signal, + }); + if (stopped || generation !== session) return; + await response.body?.getReader().read(); + } catch { + } finally { + clearTimeout(timeout); + prime.abort(); + if (primeController === prime) primeController = null; + } + }; + + const connectIosInput = async () => { + if (stopped) return; + const session = generation; + await primeIosHelper(session); + if (stopped || generation !== session) return; + const ws = transport.socket(deviceHubPaths.iosInput(device)); + socket = ws; + iosInputAdmitted = false; + iosInputUnavailable = false; + ws.addEventListener("message", (event) => { + if (stopped || socket !== ws) return; + if (!(event.data instanceof ArrayBuffer)) return; + const bytes = new Uint8Array(event.data); + if (bytes.length < 1) return; + if (bytes[0] === IOS_TAG_INPUT_ADMITTED) { + iosInputAdmitted = true; + events.onInputConnected(!iosInputUnavailable); + return; + } + if (bytes[0] !== IOS_TAG_SCREEN_CONFIG) return; + try { + const config = decodeScreenConfig(JSON.parse(decoder.decode(bytes.subarray(1)))); + if (Option.isNone(config)) return; + iosInputUnavailable = config.value.inputUnavailable === true; + events.onInputConnected( + iosInputAdmitted && !iosInputUnavailable, + iosInputUnavailable ? "Simulator input is unavailable." : undefined, + ); + screen = { + width: config.value.width, + height: config.value.height, + orientation: config.value.orientation, + }; + events.onScreen(screen); + } catch {} + }); + ws.addEventListener("close", (event) => { + if (socket !== ws) return; + socket = null; + iosInputAdmitted = false; + if (stopped) return; + events.onInputConnected(false, event.reason || `closed ${event.code}`); + scheduleRetry("input", () => void connectIosInput()); + }); + }; + + /** Android without WebCodecs: the screenshot route, one at a time, about four a second. */ + const pollStills = async () => { + const lifecycle = generation; + const isCurrent = () => !stopped && generation === lifecycle && mode === "stills"; + let failures = 0; + while (isCurrent()) { + const started = Date.now(); + const request = new AbortController(); + controller = request; + const timeout = setTimeout(() => request.abort(), STILL_FRAME_TIMEOUT_MS); + try { + const response = await transport.fetch(deviceHubPaths.screenshot(platform, device), { + method: "POST", + signal: request.signal, + }); + if (!isCurrent()) { + await response.body?.cancel().catch(() => {}); + return; + } + if (!response.ok) throw new Error(`Screenshot ${response.status}`); + const png = extractPng(new Uint8Array(await response.arrayBuffer())); + if (!png) throw new Error("The device sent an incomplete screenshot."); + await paintImage(new Blob([png as BlobPart], { type: "image/png" }), isCurrent); + failures = 0; + } catch (cause) { + if (!isCurrent()) return; + failures += 1; + if (failures >= STILL_FRAME_MAX_FAILURES) { + fail(`Could not read the device screen: ${(cause as Error).message}`); + return; + } + } finally { + clearTimeout(timeout); + if (controller === request) controller = null; + } + const wait = Math.max(0, STILL_FRAME_INTERVAL_MS - (Date.now() - started)); + await new Promise((resolve) => { + stillsTimer = setTimeout(() => { + stillsTimer = null; + resolve(); + }, wait); + }); + } + }; + + /** `inputOnly` keeps a socket for input beside polled stills, so its reconnects leave them alone. */ + const connectAndroid = (inputOnly: boolean) => { + if (stopped) return; + const ws = transport.socket(deviceHubPaths.androidSocket(device, inputOnly)); + socket = ws; + ws.addEventListener("open", () => { + if (stopped || socket !== ws) return; + if (!inputOnly) connecting(); + events.onInputConnected(true); + }); + ws.addEventListener("message", (event) => { + if (stopped || socket !== ws || inputOnly) return; + if (typeof event.data === "string") { + // The encoder restarts at a new size when the device rotates; the + // next keyframe carries a fresh SPS, so the decoder is rebuilt from it. + if (isVideoSessionMessage(event.data)) { + closeDecoder(); + configuring = false; + connecting(); + requestKeyframe(); + } + return; + } + const packet = parseSemuPacket(event.data); + const needsScan = + packet.isKey === null || + (packet.isKey && (!videoDecoder || videoDecoder.state !== "configured")); + const scanned = needsScan ? scanAccessUnit(packet.data) : null; + const isKey = packet.isKey ?? scanned?.isKey ?? false; + if (scanned?.sps && (!videoDecoder || videoDecoder.state !== "configured")) { + if (configuring) return; + configuring = true; + const epoch = decoderEpoch; + const isCurrent = () => !stopped && socket === ws; + void configureDecoder({ codec: avcCodecString(scanned.sps) }, isCurrent).then( + (configured) => { + if (!isCurrent() || epoch !== decoderEpoch) return; + configuring = false; + awaitingKeyframe = true; + if (configured) requestKeyframe(); + }, + ); + return; + } + if (!videoDecoder || videoDecoder.state !== "configured") { + if (!isKey) requestKeyframe(); + return; + } + decode(isKey, packet.data, packet.timestamp); + }); + ws.addEventListener("close", (event) => { + if (socket !== ws) return; + socket = null; + if (!inputOnly) closeDecoder(); + if (stopped) return; + events.onInputConnected(false, event.reason || `closed ${event.code}`); + if (!inputOnly) { + configuring = false; + connecting(event.reason || undefined); + } + scheduleRetry("input", () => connectAndroid(inputOnly)); + }); + }; + + const start = () => { + if (!stopped) return; + stopped = false; + generation++; + configuring = false; + iosInputAdmitted = false; + iosInputUnavailable = false; + connecting(); + if (ios) { + void connectIosInput(); + if (useWebCodecs) { + setMode("video"); + void readIosVideo(); + } else { + mode = "video"; + fallBackToMjpeg(); + } + } else if (useWebCodecs) { + setMode("video"); + connectAndroid(false); + } else { + setMode("stills"); + connectAndroid(true); + void pollStills(); + } + }; + + const stop = () => { + if (stopped) return; + stopped = true; + generation++; + videoGeneration++; + clearFrameTimer(); + if (videoReadTimer !== null) clearTimeout(videoReadTimer); + videoReadTimer = null; + if (stillsTimer !== null) clearTimeout(stillsTimer); + stillsTimer = null; + for (const timer of retryTimers.values()) clearTimeout(timer); + retryTimers.clear(); + primeController?.abort(); + primeController = null; + controller?.abort(); + controller = null; + const discarded = socket; + socket = null; + discarded?.close(); + closeDecoder(); + }; + + const send = (payload: Uint8Array | string) => { + if ( + !stopped && + (!ios || (iosInputAdmitted && !iosInputUnavailable)) && + socket?.readyState === DEVICE_SOCKET_OPEN + ) + socket.send(payload); + }; + + const rawPoint = (x: number, y: number) => { + // serve-sim streams the raw framebuffer; rotated devices need input + // remapped into that raw space. + if (!ios || !screen || screen.width > screen.height) return { x, y }; + switch (screen.orientation) { + case "landscape_left": + return { x: y, y: 1 - x }; + case "landscape_right": + return { x: 1 - y, y: x }; + case "portrait_upside_down": + return { x: 1 - x, y: 1 - y }; + default: + return { x, y }; + } + }; + + return { + start, + stop, + sendTouch: (phase, x, y) => { + if (ios) { + send(taggedJson(IOS_MSG_TOUCH, { type: phase, ...rawPoint(x, y) })); + return; + } + const action = phase === "begin" ? "down" : phase === "move" ? "move" : "up"; + send(JSON.stringify({ type: "touch", action, x, y })); + }, + sendKey: (event, phase) => { + if (ios) { + const usage = hidUsageForCode(event.code); + if (usage !== null) send(taggedJson(IOS_MSG_KEY, { type: phase, usage })); + return; + } + if (phase !== "down") return; + if (event.key === "Escape") return send(JSON.stringify({ type: "back" })); + const keycode = ANDROID_KEYCODE_BY_KEY[event.key]; + if (keycode !== undefined) return send(JSON.stringify({ type: "key", keycode })); + if (event.key.length === 1 && !event.metaKey && !event.ctrlKey) { + send(JSON.stringify({ type: "text", text: event.key })); + } + }, + sendText: (text) => { + if (ios) return; + let chunk = ""; + let chunkBytes = 0; + let sentChunks = 0; + for (const character of text) { + const characterBytes = utf8ByteLength(character.codePointAt(0)!); + if (chunkBytes + characterBytes > ANDROID_MAX_TEXT_BYTES) { + send(JSON.stringify({ type: "text", text: chunk })); + if (++sentChunks >= ANDROID_MAX_TEXT_CHUNKS) return; + chunk = ""; + chunkBytes = 0; + } + chunk += character; + chunkBytes += characterBytes; + } + if (chunk.length > 0) send(JSON.stringify({ type: "text", text: chunk })); + }, + pressButton: (button) => { + if (ios) { + const name = + button === "home" + ? "home" + : button === "appSwitcher" || button === "recents" + ? "app_switcher" + : button === "power" + ? "lock" + : null; + if (name) send(taggedJson(IOS_MSG_BUTTON, { button: name })); + return; + } + const type = button === "appSwitcher" ? "recents" : button; + send(JSON.stringify({ type })); + }, + rotate: () => { + if (!ios) return; + const current = screen?.orientation ?? "portrait"; + const next = + IOS_ORIENTATIONS[(IOS_ORIENTATIONS.indexOf(current) + 1) % IOS_ORIENTATIONS.length]!; + send(taggedJson(IOS_MSG_ORIENTATION, { orientation: next })); + }, + }; +} diff --git a/packages/client-runtime/src/state/deviceState.test.ts b/packages/client-runtime/src/state/deviceState.test.ts new file mode 100644 index 000000000..2f41eee68 --- /dev/null +++ b/packages/client-runtime/src/state/deviceState.test.ts @@ -0,0 +1,47 @@ +import type { DeviceServiceState, DeviceSession, ThreadId } from "@threadlines/contracts"; +import { describe, expect, it } from "vite-plus/test"; + +import { deviceToolInstallMessage, newAgentDeviceSessions } from "./deviceState.ts"; + +describe("deviceToolInstallMessage", () => { + it("says installing for a fresh tool and updating from the old version otherwise", () => { + expect( + deviceToolInstallMessage("expo-device-hub", { + requiredVersion: "0.22.1", + installedVersions: [], + runningVersion: null, + }), + ).toBe("Installing expo-device-hub 0.22.1…"); + expect( + deviceToolInstallMessage("expo-device-hub", { + requiredVersion: "0.22.1", + installedVersions: ["0.9.0", "0.21.0"], + runningVersion: null, + }), + ).toBe("Updating expo-device-hub from 0.21.0 to 0.22.1…"); + }); +}); + +describe("newAgentDeviceSessions", () => { + const thread = "thread-1" as ThreadId; + const session = (deviceId: string, openedBy: "user" | "agent"): DeviceSession => ({ + threadId: thread, + deviceId, + platform: "ios", + openedAt: "2026-11-20T00:00:00.000Z", + bootedByThreadlines: false, + openedBy, + }); + const state = (sessions: DeviceSession[]) => ({ sessions }) as unknown as DeviceServiceState; + + it("finds only agent sessions this thread did not have before", () => { + const before = state([session("A", "agent")]); + const after = state([ + session("A", "agent"), + session("B", "user"), + session("C", "agent"), + { ...session("D", "agent"), threadId: "other" as ThreadId }, + ]); + expect(newAgentDeviceSessions(before, after, thread).map((s) => s.deviceId)).toEqual(["C"]); + }); +}); diff --git a/packages/client-runtime/src/state/deviceState.ts b/packages/client-runtime/src/state/deviceState.ts new file mode 100644 index 000000000..0885c504f --- /dev/null +++ b/packages/client-runtime/src/state/deviceState.ts @@ -0,0 +1,137 @@ +/** + * Reading a `DeviceServiceState` the way the Device tab and Settings → Devices + * both need it: which platforms work, what the install line says, which + * version of a tool is on disk. Pure, so every client words it the same. + */ +import type { + DevicePlatform, + DeviceServiceState, + DeviceSession, + DeviceSummary, + DeviceToolVersion, + ThreadId, +} from "@threadlines/contracts"; + +export const devicePlatformName = (platform: DevicePlatform) => + platform === "ios" ? "iOS" : "Android"; + +/** + * The line shown while a tool installs: "Installing expo-device-hub 0.22.1…", + * or "Updating expo-device-hub from 0.21.0 to 0.22.1…" when an older one is + * on disk. Ported from T3 Code's `deviceToolInstallMessage`. + */ +export function deviceToolInstallMessage(name: string, tool: DeviceToolVersion | undefined) { + if (!tool) return `Installing ${name}…`; + const previous = + tool.runningVersion ?? + [...tool.installedVersions].sort((a, b) => a.localeCompare(b, "en", { numeric: true })).at(-1); + return previous && !tool.installedVersions.includes(tool.requiredVersion) + ? `Updating ${name} from ${previous} to ${tool.requiredVersion}…` + : `Installing ${name} ${tool.requiredVersion}…`; +} + +/** The version to show for a tool: the running one, else the newest on disk. Null when none. */ +export function deviceToolShownVersion(tool: DeviceToolVersion | undefined): string | null { + if (!tool) return null; + return ( + tool.runningVersion ?? + (tool.installedVersions.includes(tool.requiredVersion) + ? tool.requiredVersion + : ([...tool.installedVersions] + .sort((a, b) => a.localeCompare(b, "en", { numeric: true })) + .at(-1) ?? null)) + ); +} + +/** A tool on disk that is not the version this server wants. */ +export const deviceToolNeedsUpdate = (tool: DeviceToolVersion | undefined) => + tool !== undefined && + tool.installedVersions.length > 0 && + !tool.installedVersions.includes(tool.requiredVersion); + +export const isDeviceHostBusy = (state: DeviceServiceState) => + state.hostStatus === "installing" || state.hostStatus === "starting"; + +/** Whether `platform` can be used, and the one line saying why not or what is there. */ +export function devicePlatformStatus( + state: DeviceServiceState, + platform: DevicePlatform, +): { readonly ready: boolean; readonly message: string } { + const name = devicePlatformName(platform); + const availability = state.platforms.find((candidate) => candidate.platform === platform); + if (!availability || !availability.available) { + return { + ready: false, + message: + availability?.reason ?? + (platform === "ios" + ? "iOS Simulators need a Mac with Xcode installed." + : "Android Emulators need the Android SDK with the emulator installed."), + }; + } + const hasDevices = state.devices.some((device) => device.platform === platform); + if (state.hostStatus === "ready" && !hasDevices) { + if (state.incompletePlatforms.includes(platform)) { + return { + ready: false, + message: `No ${name} devices were listed because the listing failed. Refresh to try again.`, + }; + } + return { + ready: false, + message: + platform === "ios" + ? "Xcode is installed, but there is no iOS Simulator. Add a runtime in Xcode → Settings → Components." + : "The Android SDK is installed, but there is no virtual device. Make one in Android Studio → Device Manager.", + }; + } + return { + ready: true, + message: + platform === "ios" + ? "Xcode and the iOS Simulator are installed." + : "The Android SDK and Emulator are installed.", + }; +} + +/** Booted devices first, then by name, grouped by platform (iOS first). */ +export function groupDevicesByPlatform(devices: ReadonlyArray) { + const groups: Array<{ platform: DevicePlatform; devices: DeviceSummary[] }> = []; + for (const platform of ["ios", "android"] as const) { + const listed = devices + .filter((device) => device.platform === platform) + .toSorted((a, b) => Number(b.booted) - Number(a.booted) || a.name.localeCompare(b.name)); + if (listed.length > 0) groups.push({ platform, devices: listed }); + } + return groups; +} + +export const threadDeviceSessions = ( + state: DeviceServiceState, + threadId: ThreadId, +): ReadonlyArray => + state.sessions.filter((session) => session.threadId === threadId); + +/** + * Sessions an agent opened for a thread that a previous state did not have: + * the ones that should bring the Device tab up for whoever is watching. The + * first state a client sees is not compared: a session already open then did + * not just appear. + */ +export function newAgentDeviceSessions( + previous: DeviceServiceState, + next: DeviceServiceState, + threadId: ThreadId, +): ReadonlyArray { + const known = new Set( + previous.sessions + .filter((session) => session.threadId === threadId) + .map((session) => `${session.deviceId}\u0000${session.openedAt}`), + ); + return next.sessions.filter( + (session) => + session.threadId === threadId && + session.openedBy === "agent" && + !known.has(`${session.deviceId}\u0000${session.openedAt}`), + ); +} From 31335c7a98b21f1ce0872ba7761dfcb6665a6b9b Mon Sep 17 00:00:00 2001 From: Badcuban <108198679+badcuban@users.noreply.github.com> Date: Fri, 20 Nov 2026 01:23:10 -0500 Subject: [PATCH 03/49] feat(server): devices: hub, service, channel and agent tools endpoint Port T3 Code's device host to one local host per server. The pinned expo-device-hub and agent-device install on Threadlines' managed Node.js with install scripts on. The hub runs on loopback with --require-token; its token is read from the startup line, kept in memory and sent as a bearer on every request. hub.json lets the next server stop a leftover hub, checked by pid and command line. DeviceService keeps sessions in claims.json before reporting them open, reloads them on start, and a reactor ends them when a thread is archived, deleted or moved. Use devices off stops the hub and daemon; agent access off stops the daemon. The device channel carries the Device tab's requests and sockets over WS RPC with an allowlist (no serve-sim or serve-emu /api), 256 KiB chunks, a 2 MiB credit window and a slow cap. The threadlines_device MCP endpoint serves device_list, device_open (with a bound agent-device launcher), device_screenshot and device_close. --- .../server/src/device/AgentDeviceShim.test.ts | 157 +++ apps/server/src/device/AgentDeviceShim.ts | 322 +++++++ apps/server/src/device/AgentDeviceTarget.ts | 95 ++ apps/server/src/device/DeviceActions.test.ts | 234 +++++ apps/server/src/device/DeviceActions.ts | 585 +++++++++++ apps/server/src/device/DeviceChannels.test.ts | 239 +++++ apps/server/src/device/DeviceChannels.ts | 467 +++++++++ apps/server/src/device/DeviceHost.ts | 151 +++ apps/server/src/device/DeviceHubOperations.ts | 407 ++++++++ apps/server/src/device/DeviceService.test.ts | 135 +++ apps/server/src/device/DeviceService.ts | 791 +++++++++++++++ apps/server/src/device/DeviceThreadReactor.ts | 62 ++ apps/server/src/device/DeviceToolchain.ts | 398 ++++++++ .../server/src/device/LocalDeviceHost.test.ts | 63 ++ apps/server/src/device/LocalDeviceHost.ts | 910 ++++++++++++++++++ .../server/src/device/deviceToolVersions.json | 4 + apps/server/src/device/mjpegStillFrame.ts | 121 +++ apps/server/src/mcp/McpDeviceServer.ts | 229 +++++ apps/server/src/mcp/McpHttpServer.ts | 9 + apps/server/src/mcp/McpSessionRegistry.ts | 5 + .../mcp/alertAssistantsToolHandlers.test.ts | 1 + apps/server/src/mcp/deviceToolHandlers.ts | 201 ++++ apps/server/src/mcp/deviceTools.ts | 93 ++ .../src/mcp/projectNoteToolHandlers.test.ts | 1 + apps/server/src/mcp/roomToolHandlers.test.ts | 1 + .../src/mcp/secretRequestToolHandlers.test.ts | 1 + .../mcp/settingsChangeToolHandlers.test.ts | 1 + .../server/src/mcp/threadToolHandlers.test.ts | 1 + apps/server/src/pages/AgentPages.test.ts | 1 + .../acpRegistry/AcpRegistryInstaller.ts | 2 +- .../src/scheduled/ScheduledTasks.test.ts | 1 + apps/server/src/server.test.ts | 5 + apps/server/src/server.ts | 13 + apps/server/src/ws.ts | 51 + 34 files changed, 5756 insertions(+), 1 deletion(-) create mode 100644 apps/server/src/device/AgentDeviceShim.test.ts create mode 100644 apps/server/src/device/AgentDeviceShim.ts create mode 100644 apps/server/src/device/AgentDeviceTarget.ts create mode 100644 apps/server/src/device/DeviceActions.test.ts create mode 100644 apps/server/src/device/DeviceActions.ts create mode 100644 apps/server/src/device/DeviceChannels.test.ts create mode 100644 apps/server/src/device/DeviceChannels.ts create mode 100644 apps/server/src/device/DeviceHost.ts create mode 100644 apps/server/src/device/DeviceHubOperations.ts create mode 100644 apps/server/src/device/DeviceService.test.ts create mode 100644 apps/server/src/device/DeviceService.ts create mode 100644 apps/server/src/device/DeviceThreadReactor.ts create mode 100644 apps/server/src/device/DeviceToolchain.ts create mode 100644 apps/server/src/device/LocalDeviceHost.test.ts create mode 100644 apps/server/src/device/LocalDeviceHost.ts create mode 100644 apps/server/src/device/deviceToolVersions.json create mode 100644 apps/server/src/device/mjpegStillFrame.ts create mode 100644 apps/server/src/mcp/McpDeviceServer.ts create mode 100644 apps/server/src/mcp/deviceToolHandlers.ts create mode 100644 apps/server/src/mcp/deviceTools.ts diff --git a/apps/server/src/device/AgentDeviceShim.test.ts b/apps/server/src/device/AgentDeviceShim.test.ts new file mode 100644 index 000000000..0e81ef677 --- /dev/null +++ b/apps/server/src/device/AgentDeviceShim.test.ts @@ -0,0 +1,157 @@ +// @effect-diagnostics nodeBuiltinImport:off - runs the generated launcher as a real subprocess. +import * as NodeChildProcess from "node:child_process"; +import * as NodeNet from "node:net"; +import * as NodeUtil from "node:util"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { describe, expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; + +import { + agentDeviceLauncherKey, + agentDeviceLaunchersDir, + bindAgentDeviceLauncher, + pruneAgentDeviceLaunchers, +} from "./AgentDeviceShim.ts"; +import { agentDeviceConfigPath, writeAgentDeviceConfig } from "./AgentDeviceTarget.ts"; + +const exec = NodeUtil.promisify(NodeChildProcess.execFile); + +/** A loopback port that accepts connections until the scope closes, like the agent-device daemon. */ +const listeningPort = Effect.acquireRelease( + Effect.promise( + () => + new Promise((resolve) => { + const server = NodeNet.createServer((socket) => socket.destroy()); + server.listen(0, "127.0.0.1", () => resolve(server)); + }), + ), + (server) => Effect.promise(() => new Promise((resolve) => server.close(() => resolve()))), +).pipe(Effect.map((server) => (server.address() as NodeNet.AddressInfo).port)); + +describe.skipIf(process.platform === "win32")("agent-device launcher", () => { + it.live("binds one device's flags, lets the agent's own flags win, and leaves help alone", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const dir = yield* fs.makeTempDirectoryScoped({ prefix: "tl-device-launcher-" }); + const port = yield* listeningPort; + const entryPath = path.join(dir, "cli.mjs"); + yield* fs.writeFileString(entryPath, "console.log(JSON.stringify(process.argv.slice(2)));"); + const config = agentDeviceConfigPath(path, dir); + yield* writeAgentDeviceConfig(config, { baseUrl: `http://127.0.0.1:${port}`, token: "t" }); + const target = [ + "--platform", + "ios", + "--udid", + "UDID-1", + "--config", + config, + "--session", + "tl-a", + ]; + const command = yield* bindAgentDeviceLauncher({ + stateDir: dir, + threadId: "thread-1", + deviceId: "UDID-1", + node: process.execPath, + entryPath, + targetArgs: target, + }); + expect(command).toBe( + path.join(agentDeviceLaunchersDir(path, dir), "thread-1", "UDID-1", "agent-device"), + ); + const launch = (args: ReadonlyArray) => + exec(command, [...args]).then((result) => JSON.parse(result.stdout)); + + expect(yield* Effect.promise(() => launch(["snapshot", "-i"]))).toEqual([ + "snapshot", + "-i", + ...target, + ]); + expect(yield* Effect.promise(() => launch(["open", "--session", "mine"]))).toEqual([ + "open", + "--session", + "mine", + ...target.slice(0, 6), + ]); + // After `--` everything is text, and bound flags go before it. + expect(yield* Effect.promise(() => launch(["fill", "@e5", "--", "--help"]))).toEqual([ + "fill", + "@e5", + ...target, + "--", + "--help", + ]); + for (const args of [["help", "gesture"], ["open", "--help"], []]) { + expect(yield* Effect.promise(() => launch(args))).toEqual(args); + } + // Commands that would replace Threadlines' connection are refused. + yield* Effect.promise(() => + expect(launch(["daemon", "stop"])).rejects.toThrow("Threadlines manages this command"), + ); + // A runtime that moved since device_open names the fix. + yield* fs.remove(entryPath); + yield* Effect.promise(() => + expect(launch(["snapshot"])).rejects.toThrow("Call device_open again"), + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.live("says the device connection is down before running agent-device", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const dir = yield* fs.makeTempDirectoryScoped({ prefix: "tl-device-launcher-" }); + const entryPath = path.join(dir, "cli.mjs"); + yield* fs.writeFileString(entryPath, "process.exit(3);"); + const port = yield* Effect.scoped(listeningPort); + const config = agentDeviceConfigPath(path, dir); + yield* writeAgentDeviceConfig(config, { baseUrl: `http://127.0.0.1:${port}`, token: "t" }); + const command = yield* bindAgentDeviceLauncher({ + stateDir: dir, + threadId: "thread-1", + deviceId: "UDID-1", + node: process.execPath, + entryPath, + targetArgs: ["--config", config, "--session", "tl-a"], + }); + yield* Effect.promise(() => + expect(exec(command, ["snapshot"])).rejects.toThrow("is not reachable from this shell"), + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("removes the launchers of sessions that ended", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const dir = yield* fs.makeTempDirectoryScoped({ prefix: "tl-device-launcher-" }); + for (const [threadId, deviceId] of [ + ["thread-1", "UDID-1"], + ["thread-1", "Pixel 9"], + ["thread-2", "UDID-1"], + ] as const) { + yield* bindAgentDeviceLauncher({ + stateDir: dir, + threadId, + deviceId, + node: process.execPath, + entryPath: "/missing.mjs", + targetArgs: [], + }); + } + yield* pruneAgentDeviceLaunchers({ + stateDir: dir, + live: new Set([agentDeviceLauncherKey("thread-1", "Pixel 9")]), + }); + const root = agentDeviceLaunchersDir(path, dir); + expect(yield* fs.readDirectory(root)).toEqual(["thread-1"]); + expect(yield* fs.readDirectory(path.join(root, "thread-1"))).toEqual([ + agentDeviceLauncherKey("thread-1", "Pixel 9").split("/")[1], + ]); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +}); diff --git a/apps/server/src/device/AgentDeviceShim.ts b/apps/server/src/device/AgentDeviceShim.ts new file mode 100644 index 000000000..3afea3d9a --- /dev/null +++ b/apps/server/src/device/AgentDeviceShim.ts @@ -0,0 +1,322 @@ +// @effect-diagnostics globalJsonInEffect:off - JSON string literals embed paths safely into generated JavaScript. +/** + * The `agent-device` launcher `device_open` hands an agent + * (docs/design/devices.md). One per thread and device, at + * `/device/bin///agent-device`, with that thread's + * agent-device session, config and target flags built in, so the agent passes + * none and no provider's PATH changes. Ported from T3 Code's AgentDeviceShim: + * the launcher leaves out flags a command refuses, refuses the commands that + * would replace Threadlines' connection, and checks the connection first so a + * sandboxed shell and a dead daemon get different messages. + */ +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; + +import { deviceStateDir } from "./DeviceToolchain.ts"; + +const STALE_RUNTIME = + "This agent-device command is out of date because Threadlines moved or updated its runtime. Call device_open again for a fresh one."; + +const quoteShell = (value: string) => "'" + value.replaceAll("'", "'\"'\"'") + "'"; + +/** agent-device 0.21.36's flags that take a value, so the launcher can find the command past them. */ +const VALUE_FLAGS: ReadonlyArray = [ + "--activity", + "--all", + "--android-device-allowlist", + "--app", + "--appium-version", + "--artifact", + "--artifacts-dir", + "--aws-app-arn", + "--aws-device-arn", + "--aws-interaction-mode", + "--aws-project-arn", + "--aws-region", + "-b", + "--baseline", + "--bearer-token", + "--bundle-url", + "--button", + "--config", + "--count", + "--crop-on", + "--custom-network", + "-d", + "--daemon-auth-token", + "--daemon-base-url", + "--daemon-server-mode", + "--daemon-transport", + "--delay-ms", + "--depth", + "--device", + "--device-orientation", + "--dsym", + "--duration-ms", + "--fps", + "--from", + "--geo-location", + "--github-actions-artifact", + "--header", + "--hold-ms", + "--host", + "--image-tag", + "--in-app", + "--include", + "--interval-ms", + "--ios-simulator-device-set", + "--ios-xctest-derived-data-path", + "--ios-xctest-env-dir", + "--ios-xctestrun-file", + "--jitter-px", + "--keyframes", + "--kind", + "--language", + "--launch-args", + "--launch-console", + "--launch-url", + "--lease-backend", + "--lease-id", + "--level", + "--listen-host", + "--locale", + "--max-passes", + "--max-steps", + "--metro-host", + "--metro-kind", + "--metro-port", + "--network-profile", + "--on-error", + "--os", + "--os-version", + "--out", + "--pattern", + "--pause-ms", + "--pixel-density", + "--pixels", + "--plan-digest", + "--platform", + "--pointer-count", + "--port", + "--probe-timeout-ms", + "--project-root", + "--provider", + "--provider-app", + "--provider-appium-version", + "--provider-build", + "--provider-custom-network", + "--provider-device-orientation", + "--provider-device-type", + "--provider-geo-location", + "--provider-language", + "--provider-locale", + "--provider-network-hosts", + "--provider-network-profile", + "--provider-os-version", + "--provider-project", + "--provider-session", + "--provider-session-name", + "--provider-timezone", + "--proxy-base-url", + "--public-base-url", + "--quality", + "--record-as", + "--remote-config", + "--reporter", + "--report-junit", + "--retention-ms", + "--retries", + "--run-id", + "--runtime-file", + "-s", + "--scale", + "--scope", + "--search-path", + "--serial", + "--session", + "--session-isolation", + "--session-lock", + "--settle-quiet", + "--shard-all", + "--shard-split", + "--startup-timeout-ms", + "--state-dir", + "--status-host", + "--steps", + "--steps-file", + "--surface", + "--system", + "--target", + "--target-app", + "--template", + "--tenant", + "--threshold", + "--timeout", + "--timezone", + "--udid", + "--until", + "--wait", +]; + +// Help gets no bound flags (agent-device reads --config even for help); only args before `--` are flags. +// agent-device 0.21.31 refuses some bound flags per command; connect/disconnect/auth/daemon would +// replace Threadlines' connection; device status reads this machine's claims. It reports a sandboxed shell and a +// dropped tunnel alike, so check first, at --daemon-base-url if given, as agent-device prefers it. +const launcherSource = (node: string, entryPath: string, boundArgs: ReadonlyArray) => + `import { spawn } from "node:child_process"; +import { existsSync, readFileSync } from "node:fs"; +import { connect } from "node:net"; +const entry = ${JSON.stringify(entryPath)}; +const bound = ${JSON.stringify(boundArgs)}; +const refused = new Map([["cdp", ["--platform", "--udid", "--config", "--session"]], ["connection", ["--platform", "--udid"]]]); +const managed = ["connect", "disconnect", "auth", "daemon"]; +const args = process.argv.slice(2); +const end = args.indexOf("--"); +const options = end < 0 ? args : args.slice(0, end); +const valued = new Set(${JSON.stringify(VALUE_FLAGS)}); +let name = ""; +for (let index = 0; index < args.length; index++) { + const arg = args[index]; + if (arg === "--") { name = args[index + 1] ?? ""; break; } + if (!arg.startsWith("-")) { name = arg; break; } + if (!arg.includes("=") && valued.has(arg)) index++; +} +const informational = args.length === 0 || ["help", "--help", "-h", "--version", "version"].includes(args[0]) || options.includes("--help") || options.includes("-h"); +if (!informational && managed.includes(name)) { + console.error("Threadlines manages this command's connection to the device, so \`" + name + "\` is not available through it."); + process.exit(1); +} +if (!informational && name === "device") { + console.error("\`device status\` reads this machine's device claims, not those on the device's host, so it is not available through this command. Quote the DEVICE_IN_USE error instead."); + process.exit(1); +} +const skip = refused.get(name) ?? []; +if (!informational) for (let index = 0; index + 1 < bound.length; index += 2) if (!skip.includes(bound[index]) && !options.some(arg => arg === bound[index] || arg.startsWith(bound[index] + "="))) options.push(bound[index], bound[index + 1]); +const command = end < 0 ? options : [...options, ...args.slice(end)]; +const hasValue = flag => { const index = options.indexOf(flag); return options.some(arg => arg.startsWith(flag + "=") && arg.length > flag.length + 1) || (index >= 0 && !!options[index + 1] && !options[index + 1].startsWith("--")); }; +if (!informational && !skip.includes("--session") && !(hasValue("--config") && hasValue("--session"))) { + console.error("Call device_open first and run the agent-device command it returns."); + process.exit(1); +} +if (!existsSync(entry)) { + console.error(${JSON.stringify(STALE_RUNTIME)}); + process.exit(1); +} +const valueOf = flag => { const inline = options.find(arg => arg.startsWith(flag + "=")); if (inline) return inline.slice(flag.length + 1); const index = options.indexOf(flag); return index >= 0 ? options[index + 1] : undefined; }; +const endpoint = () => { try { const url = new URL(valueOf("--daemon-base-url") ?? JSON.parse(readFileSync(valueOf("--config"), "utf8")).daemonBaseUrl); return ["127.0.0.1", "localhost", "[::1]"].includes(url.hostname) && url.port ? url : undefined; } catch { return undefined; } }; +const local = informational || refused.has(name) ? undefined : endpoint(); +if (local) { + const failure = await new Promise(resolve => { + const socket = connect({ host: local.hostname.replace(/^\\[|\\]$/g, ""), port: Number(local.port) }); + socket.setTimeout(3000, () => { socket.destroy(); resolve("ETIMEDOUT"); }); + socket.once("connect", () => { socket.destroy(); resolve(undefined); }); + socket.once("error", error => resolve(error.code ?? "ERROR")); + }); + if (failure === "EPERM" || failure === "EACCES") { + console.error("This shell cannot reach Threadlines' device connection at " + local.host + " (connect " + failure + "): a network sandbox is blocking localhost. Run this command again with network access, outside the sandbox."); + process.exit(1); + } + if (failure) { + console.error("Threadlines' device connection at " + local.host + " is not reachable from this shell (connect " + failure + "). Call device_open again to reconnect; if that succeeds and this repeats, this shell runs in a network sandbox, so run the command with network access, outside the sandbox."); + process.exit(1); + } +} +const env = { ...process.env, ELECTRON_RUN_AS_NODE: "1" }; +delete env.AGENT_DEVICE_DAEMON_BASE_URL; +delete env.AGENT_DEVICE_DAEMON_AUTH_TOKEN; +delete env.AGENT_DEVICE_CONFIG; +const child = spawn(${JSON.stringify(node)}, [entry, ...command], { stdio: "inherit", env }); +child.on("error", error => { console.error(error.message); process.exitCode = 1; }); +child.on("exit", code => { process.exitCode = code ?? 1; }); +`; + +/** Writes a launcher into `dir` and returns the executable agents run. */ +const writeLauncher = Effect.fn("AgentDeviceShim.writeLauncher")(function* (input: { + readonly dir: string; + readonly node: string; + readonly entryPath: string; + readonly boundArgs: ReadonlyArray; +}) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const node = input.node; + yield* fs.makeDirectory(input.dir, { recursive: true }); + const launcherPath = path.join(input.dir, "agent-device-launcher.mjs"); + yield* fs.writeFileString(launcherPath, launcherSource(node, input.entryPath, input.boundArgs)); + if (process.platform === "win32") { + const script = `@echo off\r\nsetlocal EnableExtensions DisableDelayedExpansion\r\nif not exist "${node}" (echo ${STALE_RUNTIME} 1>&2 & exit /b 1)\r\n"${node}" "${launcherPath}" %*\r\n`; + const scriptPath = path.join(input.dir, "agent-device.cmd"); + yield* fs.writeFileString(scriptPath, script); + return scriptPath; + } + const script = `#!/bin/sh\n[ -x ${quoteShell(node)} ] || { echo ${quoteShell(STALE_RUNTIME)} >&2; exit 1; }\nexec ${quoteShell(node)} ${quoteShell(launcherPath)} "$@"\n`; + const scriptPath = path.join(input.dir, "agent-device"); + yield* fs.writeFileString(scriptPath, script); + yield* fs.chmod(scriptPath, 0o755); + return scriptPath; +}); + +const SAFE_SEGMENT = /^[A-Za-z0-9._-]{1,128}$/; + +/** A path segment for an id: the id itself when it is plain, else a digest of it. */ +const segment = (id: string) => + SAFE_SEGMENT.test(id) && id !== "." && id !== ".." + ? id + : `x-${Buffer.from(id).toString("base64url").slice(0, 96)}`; + +/** `/device/bin`. */ +export const agentDeviceLaunchersDir = (path: Path.Path, stateDir: string) => + path.join(deviceStateDir(path, stateDir), "bin"); + +/** Where one thread's launcher for one device lives, relative to the launchers folder. */ +export const agentDeviceLauncherKey = (threadId: string, deviceId: string) => + `${segment(threadId)}/${segment(deviceId)}`; + +/** + * A launcher for one agent-device session with its target flags built in, so + * agents pass none, and a shell that does not word-split a variable cannot mangle them. + */ +export const bindAgentDeviceLauncher = Effect.fn("AgentDeviceShim.bind")(function* (input: { + readonly stateDir: string; + readonly threadId: string; + readonly deviceId: string; + readonly node: string; + readonly entryPath: string; + readonly targetArgs: ReadonlyArray; +}) { + const path = yield* Path.Path; + return yield* writeLauncher({ + dir: path.join( + agentDeviceLaunchersDir(path, input.stateDir), + ...agentDeviceLauncherKey(input.threadId, input.deviceId).split("/"), + ), + node: input.node, + entryPath: input.entryPath, + boundArgs: input.targetArgs, + }); +}); + +/** Removes every launcher whose `/` key is not in `live`. Best effort. */ +export const pruneAgentDeviceLaunchers = Effect.fn("AgentDeviceShim.prune")(function* (input: { + readonly stateDir: string; + readonly live: ReadonlySet; +}) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = agentDeviceLaunchersDir(path, input.stateDir); + const threads = yield* fs.readDirectory(root).pipe(Effect.orElseSucceed(() => [])); + for (const thread of threads) { + const threadDir = path.join(root, thread); + const devices = yield* fs.readDirectory(threadDir).pipe(Effect.orElseSucceed(() => [])); + const stale = devices.filter((device) => !input.live.has(`${thread}/${device}`)); + yield* Effect.forEach( + stale, + (device) => fs.remove(path.join(threadDir, device), { recursive: true, force: true }), + { discard: true }, + ).pipe(Effect.ignore); + if (stale.length === devices.length) { + yield* fs.remove(threadDir, { recursive: true, force: true }).pipe(Effect.ignore); + } + } +}); diff --git a/apps/server/src/device/AgentDeviceTarget.ts b/apps/server/src/device/AgentDeviceTarget.ts new file mode 100644 index 000000000..54cfbc099 --- /dev/null +++ b/apps/server/src/device/AgentDeviceTarget.ts @@ -0,0 +1,95 @@ +// @effect-diagnostics nodeBuiltinImport:off - a stable session name from a sha256 +/** + * Where agent-device finds this server's daemon, and the session each thread + * drives a device through (docs/design/devices.md). Ported from T3 Code's + * AgentDeviceTarget, with one host per server. + */ +import { createHash } from "node:crypto"; + +import { DeviceOperationError } from "@threadlines/contracts"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http"; + +import type { AgentDeviceEndpoint } from "./DeviceHost.ts"; +import { deviceStateDir } from "./DeviceToolchain.ts"; + +const encodeEndpoint = Schema.encodeEffect( + Schema.fromJsonString( + Schema.Struct({ + daemonBaseUrl: Schema.String, + daemonAuthToken: Schema.String, + }), + ), +); + +const key = (value: string) => createHash("sha256").update(value).digest("hex").slice(0, 24); + +/** The config file every launcher points agent-device at: the daemon's address and token. */ +export const agentDeviceConfigPath = (path: Path.Path, stateDir: string) => + path.join(deviceStateDir(path, stateDir), "agent-device.json"); + +/** One agent-device session per thread and device, named so it never collides across them. */ +export const agentDeviceSession = (threadId: string, deviceId: string): string => + `tl-${key(JSON.stringify([threadId, deviceId]))}`; + +/** Writes the daemon's address and token where launchers read it, readable by this user only. */ +export const writeAgentDeviceConfig = Effect.fn("AgentDeviceTarget.writeConfig")(function* ( + file: string, + endpoint: Pick, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* fs.makeDirectory(path.dirname(file), { recursive: true }); + const content = yield* encodeEndpoint({ + daemonBaseUrl: endpoint.baseUrl, + daemonAuthToken: endpoint.token, + }); + if ((yield* fs.readFileString(file).pipe(Effect.orElseSucceed(() => ""))) === content) return; + const temporary = yield* fs.makeTempFile({ directory: path.dirname(file), prefix: ".endpoint-" }); + yield* Effect.gen(function* () { + yield* fs.chmod(temporary, 0o600); + yield* fs.writeFileString(temporary, content); + yield* fs.rename(temporary, file); + }).pipe(Effect.ensuring(fs.remove(temporary, { force: true }).pipe(Effect.ignore))); +}); + +const AgentCloseResult = Schema.Union([ + Schema.Struct({ result: Schema.Struct({ ok: Schema.Literal(true) }) }), + Schema.Struct({ + error: Schema.Struct({ + data: Schema.Struct({ code: Schema.Literal("SESSION_NOT_FOUND") }), + }), + }), +]); + +/** + * Ends an agent-device session on a running daemon, which quits the app that + * session opened and releases its claim on the device. An already-released + * session succeeds. + */ +export const releaseAgentDeviceSession = + (http: HttpClient.HttpClient, endpoint: Pick) => + (session: string) => + HttpClientRequest.post(`${endpoint.baseUrl}/rpc`).pipe( + HttpClientRequest.bearerToken(endpoint.token), + HttpClientRequest.bodyJson({ + jsonrpc: "2.0", + id: session, + method: "agent_device.command", + params: { command: "close", session, positionals: [], flags: {} }, + }), + Effect.flatMap((request) => http.pipe(HttpClient.withScope).execute(request)), + // agent-device reports an already-released session as HTTP 404. + Effect.filterOrElse((response) => response.status === 404, HttpClientResponse.filterStatusOk), + Effect.flatMap(HttpClientResponse.schemaBodyJson(AgentCloseResult)), + Effect.scoped, + Effect.timeout(Duration.seconds(15)), + Effect.asVoid, + Effect.mapError( + () => new DeviceOperationError({ operation: "close", reason: "request_failed" }), + ), + ); diff --git a/apps/server/src/device/DeviceActions.test.ts b/apps/server/src/device/DeviceActions.test.ts new file mode 100644 index 000000000..ae7088a94 --- /dev/null +++ b/apps/server/src/device/DeviceActions.test.ts @@ -0,0 +1,234 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { describe, expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as ChildProcess from "effect/process/ChildProcess"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; + +import { readDeviceDetail, runDeviceAction } from "./DeviceActions.ts"; +import type { DeviceHostReady } from "./DeviceHost.ts"; + +type Call = { command: string; args: ReadonlyArray; stdin?: string }; + +const makeReady = ( + respond: (call: Call) => { stdout?: string; stderr?: string; code?: number } = () => ({}), + helpers: DeviceHostReady["helpers"] = { + serveSimAxSettings: "/hub/simax/serve-sim-ax-settings", + serveSimCli: "/hub/serve-sim.js", + }, +) => { + const calls: Call[] = []; + const unused = Effect.die("Device actions run host commands, never device operations"); + const ready: DeviceHostReady = { + nodePath: process.execPath, + hub: { origin: "http://127.0.0.1:1", token: "test-token" }, + devices: { + list: unused, + boot: () => unused, + attach: () => unused, + shutdown: () => unused, + screenshot: () => unused, + }, + helpers, + run: (command, args, options) => { + const call = { command, args, ...(options?.stdin ? { stdin: options.stdin } : {}) }; + calls.push(call); + const result = respond(call); + return Effect.succeed({ + stdout: result.stdout ?? "", + stderr: result.stderr ?? "", + code: result.code ?? 0, + }); + }, + }; + return { ready, calls }; +}; + +const udid = "SIM-1"; + +describe("runDeviceAction", () => { + it.live.each([ + "https://example.com/callback?code=1&state=2", + "https://example.com/it's?a=1&b=2", + "https://example.com/a b?key=value", + "https://example.com/plain", + ])("opens the complete Android URL %s", (url) => + Effect.gen(function* () { + if (process.platform === "win32") return; + const { ready, calls } = makeReady(); + yield* runDeviceAction(ready, "android", { + type: "openUrl", + deviceId: "emulator-5554", + url, + }); + const args = calls[0]!.args; + expect(args.slice(0, 3)).toEqual(["-s", "emulator-5554", "shell"]); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + // adb joins arguments verbatim before the Android shell parses them. + const stdout = yield* spawner.string( + ChildProcess.make("/bin/sh", [ + "-c", + `am() { printf '%s\\0' "$@"; }\n${args.slice(3).join(" ")}`, + ]), + ); + expect(stdout.split("\0").slice(0, -1)).toEqual([ + "start", + "-a", + "android.intent.action.VIEW", + "-d", + url, + ]); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect.each([ + ["emulator-5554", "Physical size: 2560x1600", "portrait", "-9.81:0:0"], + ["emulator-5554", "Physical size: 2560x1600", "landscape_left", "0:9.81:0"], + ["R5CT1234", "Physical size: 2560x1600", "portrait", "3"], + ["R5CT1234", "Physical size: 2560x1600", "landscape_left", "0"], + ["R5CT1234", "Physical size: 1080x1920\nOverride size: 1920x1080", "portrait", "3"], + ] as const)( + "rotates from the natural orientation reported by wm size: %s %j %s", + ([deviceId, wmSize, value, expected]) => + Effect.gen(function* () { + const { ready, calls } = makeReady((call) => + call.args.join(" ").endsWith("shell wm size") ? { stdout: `${wmSize}\n` } : {}, + ); + yield* runDeviceAction(ready, "android", { type: "setOrientation", deviceId, value }); + expect(calls.at(-1)?.args.at(-1)).toBe(expected); + }), + ); + + it.effect.each([ + ["camera", "grant", "android.permission.CAMERA"], + ["photos", "revoke", "android.permission.READ_MEDIA_IMAGES"], + ] as const)( + "reports an Android %s %s that no pm command applied", + ([permission, decision, firstPermission]) => + Effect.gen(function* () { + const { ready } = makeReady((call) => ({ + code: 1, + stderr: `Unknown package: ${call.args.at(-1)}`, + })); + const error = yield* Effect.flip( + runDeviceAction(ready, "android", { + type: "setPermission", + deviceId: "emulator-5554", + appId: "com.example.missing", + permission, + decision, + }), + ); + expect(error._tag).toBe("DeviceOperationError"); + expect(error.operation).toBe("permission"); + expect(error._tag === "DeviceOperationError" && error.detail).toBe( + `Unknown package: ${firstPermission}`, + ); + }), + ); + + it.effect.each([ + [ + "photos", + ["android.permission.READ_MEDIA_IMAGES", "android.permission.READ_EXTERNAL_STORAGE"], + 1, + ], + ["contacts", ["android.permission.READ_CONTACTS", "android.permission.WRITE_CONTACTS"], 0], + ] as const)( + "keeps a partially applied %s permission change", + ([permission, permissions, success]) => + Effect.gen(function* () { + const { ready, calls } = makeReady((call) => ({ + code: call.args.at(-1) === permissions[success] ? 0 : 1, + })); + const result = yield* runDeviceAction(ready, "android", { + type: "setPermission", + deviceId: "emulator-5554", + appId: "com.example.app", + permission, + decision: "grant", + }); + expect(result).toBeUndefined(); + expect(calls.map((call) => call.args.at(-1))).toEqual(permissions); + }), + ); +}); + +describe("readDeviceDetail", () => { + it.effect("reads the smallest iOS text size as small", () => + Effect.gen(function* () { + const { ready } = makeReady((call) => + call.args.join(" ").endsWith("ui SIM-1 content_size") + ? { stdout: "extra-small\n" } + : { code: 1 }, + ); + const detail = yield* readDeviceDetail(ready, "ios", udid); + expect(detail.settings.textSize).toBe("small"); + }), + ); + + it.effect("reads iOS settings from simctl and the accessibility helper", () => + Effect.gen(function* () { + const { ready } = makeReady((call) => { + const key = call.args.join(" "); + if (key.endsWith("ui SIM-1 appearance")) return { stdout: "dark\n" }; + if (key.endsWith("ui SIM-1 content_size")) return { stdout: "extra-extra-large\n" }; + if (key.endsWith("ui SIM-1 increase_contrast")) return { stdout: "enabled\n" }; + if (key.includes("serve-sim-ax-settings status")) { + return { + stdout: + '{"reduce-motion":"on","reduce-transparency":"off","show-borders":"off","voiceover":"off","liquid-glass":"tinted","color-filter":"grayscale"}', + }; + } + return { code: 1 }; + }); + const detail = yield* readDeviceDetail(ready, "ios", udid); + expect(detail.settings).toEqual({ + appearance: "dark", + textSize: "large", + increaseContrast: true, + reduceMotion: true, + reduceTransparency: false, + showBorders: false, + voiceOver: false, + liquidGlass: "tinted", + colorFilter: "grayscale", + }); + }), + ); + + it.effect("degrades unreadable values to unknown instead of failing", () => + Effect.gen(function* () { + const { ready } = makeReady(() => ({ code: 1, stderr: "boom" })); + const detail = yield* readDeviceDetail(ready, "ios", udid); + expect(detail.settings).toEqual({}); + expect(detail.foregroundApp).toBeNull(); + }), + ); + + it.effect("reads Android settings and the focused package", () => + Effect.gen(function* () { + const { ready } = makeReady((call) => { + const key = call.args.join(" "); + if (key.endsWith("cmd uimode night")) return { stdout: "Night mode: yes\n" }; + if (key.endsWith("font_scale")) return { stdout: "0.85\n" }; + if (key.endsWith("animator_duration_scale")) return { stdout: "0\n" }; + if (key.endsWith("wifi_on")) return { stdout: "1\n" }; + if (key.endsWith("dumpsys window")) { + return { + stdout: + " mFocusedApp=ActivityRecord{155579877 u0 com.example.app/.MainActivity t15}\n mCurrentFocus=Window{1a2b u0 com.example.app/com.example.app.MainActivity}\n", + }; + } + return { code: 1 }; + }); + const detail = yield* readDeviceDetail(ready, "android", "emulator-5554"); + expect(detail.settings).toEqual({ + appearance: "dark", + textSize: "small", + reduceMotion: true, + networkEnabled: true, + }); + expect(detail.foregroundApp).toEqual({ id: "com.example.app" }); + }), + ); +}); diff --git a/apps/server/src/device/DeviceActions.ts b/apps/server/src/device/DeviceActions.ts new file mode 100644 index 000000000..2333671cf --- /dev/null +++ b/apps/server/src/device/DeviceActions.ts @@ -0,0 +1,585 @@ +/** + * The Tools drawer (docs/design/devices.md): one typed action per control. + * serve-sim's preview drives its own Tools panel with shell commands over its + * exec channel; proxying that would hand any client command execution on this + * machine, so these run `simctl`, `adb` and serve-sim's helper binaries + * directly and never go through the hub. Ported from T3 Code's DeviceActions. + */ +import { + type DeviceActionInput, + type DeviceActionType, + type DeviceForegroundApp, + DeviceOperationError, + DeviceActionUnavailableError, + type DeviceOrientation, + type DevicePlatform, + type DeviceSettings, + type DeviceTextSize, +} from "@threadlines/contracts"; +import * as Effect from "effect/Effect"; +import * as Result from "effect/Result"; +import * as Schema from "effect/Schema"; + +import type { DeviceHostReady } from "./DeviceHost.ts"; + +type Runner = DeviceHostReady["run"]; + +const decodeAxStatus = Schema.decodeUnknownEffect( + Schema.fromJsonString(Schema.Record(Schema.String, Schema.String)), +); +const encodePushPayload = Schema.encodeUnknownEffect( + Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)), +); + +const IOS_ACTIONS: ReadonlySet = new Set([ + "setAppearance", + "setTextSize", + "setToggle", + "setLiquidGlass", + "setColorFilter", + "setLocation", + "clearLocation", + "setPermission", + "openUrl", + "launchApp", + "terminateApp", + "sendPush", +]); + +const ANDROID_ACTIONS: ReadonlySet = new Set([ + "setAppearance", + "setTextSize", + "setToggle", + "setOrientation", + "setLocation", + "clearLocation", + "setPermission", + "openUrl", + "launchApp", + "terminateApp", +]); + +const IOS_TOGGLES = new Set([ + "reduceMotion", + "increaseContrast", + "reduceTransparency", + "showBorders", + "voiceOver", +]); +const ANDROID_TOGGLES = new Set(["reduceMotion", "networkEnabled"]); + +const supportsAction = (platform: DevicePlatform, input: DeviceActionInput): boolean => { + const actions = platform === "ios" ? IOS_ACTIONS : ANDROID_ACTIONS; + if (!actions.has(input.type)) return false; + if (input.type === "setToggle") { + return (platform === "ios" ? IOS_TOGGLES : ANDROID_TOGGLES).has(input.setting); + } + return true; +}; + +/** What a failed command said, trimmed for the browser. */ +const failureDetail = (result: { stderr: string; stdout: string }) => { + const text = (result.stderr.trim() || result.stdout.trim()).slice(0, 1_999); + return text ? { detail: text } : {}; +}; + +const ok = (operation: string) => (result: { code: number; stderr: string; stdout: string }) => + result.code === 0 + ? Effect.succeed(result.stdout) + : Effect.fail( + new DeviceOperationError({ + operation, + reason: "command_failed", + exitCode: result.code, + ...failureDetail(result), + }), + ); + +const IOS_TEXT_SIZES: Record = { + small: "small", + default: "large", + large: "extra-extra-large", + "extra-large": "accessibility-large", +}; +const ANDROID_TEXT_SIZES: Record = { + small: "0.85", + default: "1.0", + large: "1.15", + "extra-large": "1.3", +}; + +const textSizeFromIos = (category: string): DeviceTextSize | undefined => { + const entry = (Object.entries(IOS_TEXT_SIZES) as Array<[DeviceTextSize, string]>).find( + ([, value]) => value === category, + ); + if (entry) return entry[0]; + if (category.startsWith("accessibility")) return "extra-large"; + if (category === "extra-small" || category === "small" || category === "medium") return "small"; + return category.includes("extra") ? "large" : "default"; +}; + +const textSizeFromAndroid = (scale: number): DeviceTextSize => { + if (scale <= 0.9) return "small"; + if (scale >= 1.25) return "extra-large"; + if (scale >= 1.1) return "large"; + return "default"; +}; + +const IOS_TOGGLE_OPTIONS: Record = { + reduceMotion: "reduce-motion", + increaseContrast: "increase-contrast", + reduceTransparency: "reduce-transparency", + showBorders: "show-borders", + voiceOver: "voiceover", +}; + +const IOS_TCC_SERVICES: Record = { + camera: "camera", + microphone: "microphone", + photos: "photos", + contacts: "contacts", + calendar: "calendar", + reminders: "reminders", + motion: "motion", + "media-library": "media-library", + faceid: "faceid", +}; + +const ANDROID_PERMISSIONS: Record> = { + camera: ["android.permission.CAMERA"], + microphone: ["android.permission.RECORD_AUDIO"], + photos: ["android.permission.READ_MEDIA_IMAGES", "android.permission.READ_EXTERNAL_STORAGE"], + contacts: ["android.permission.READ_CONTACTS", "android.permission.WRITE_CONTACTS"], + calendar: ["android.permission.READ_CALENDAR", "android.permission.WRITE_CALENDAR"], + location: [ + "android.permission.ACCESS_FINE_LOCATION", + "android.permission.ACCESS_COARSE_LOCATION", + ], + notifications: ["android.permission.POST_NOTIFICATIONS"], + motion: ["android.permission.ACTIVITY_RECOGNITION"], +}; + +// Android numbers rotations from the display's natural orientation and the emulator's accelerometer +// reports gravity in that same frame, so both tables are keyed by the orientation a portrait-native +// device shows at each rotation. +const ANDROID_GRAVITY: Record = { + portrait: "0:9.81:0", + landscape_left: "9.81:0:0", + portrait_upside_down: "0:-9.81:0", + landscape_right: "-9.81:0:0", +}; +const ANDROID_ROTATION: Record = { + portrait: "0", + landscape_left: "1", + portrait_upside_down: "2", + landscape_right: "3", +}; +// A landscape-native tablet sits a quarter turn further along that sequence: its portrait is +// rotation 3 and its landscape is rotation 0. +const ANDROID_LANDSCAPE_NATIVE_ROTATION: Record = { + portrait: "landscape_right", + landscape_left: "portrait", + portrait_upside_down: "landscape_left", + landscape_right: "portrait_upside_down", +}; + +// `wm size` reports the natural-orientation size; an `Override size:` line is what the window +// manager uses and is printed after `Physical size:`. +const isLandscapeNative = (wmSize: string | undefined) => { + const size = wmSize ? [...wmSize.matchAll(/size: (\d+)x(\d+)/g)].at(-1) : undefined; + return size !== undefined && Number(size[1]) > Number(size[2]); +}; + +export const runDeviceAction = Effect.fn("DeviceActions.run")(function* ( + ready: DeviceHostReady, + platform: DevicePlatform, + input: DeviceActionInput, +) { + if (!supportsAction(platform, input)) { + return yield* new DeviceActionUnavailableError({ + operation: input.type, + platform, + reason: "unsupported", + }); + } + if (platform === "ios") return yield* runIos(ready, input); + return yield* runAndroid(ready.run, input); +}); + +const simctl = (run: Runner, udid: string, args: ReadonlyArray, operation: string) => + run("xcrun", ["simctl", ...args.slice(0, 1), udid, ...args.slice(1)]).pipe( + Effect.flatMap(ok(operation)), + ); + +const axSettings = Effect.fn("DeviceActions.axSettings")(function* ( + ready: DeviceHostReady, + udid: string, + args: ReadonlyArray, +) { + const helper = ready.helpers.serveSimAxSettings; + if (!helper) { + return yield* new DeviceActionUnavailableError({ + operation: "accessibility", + platform: "ios", + reason: "helper_missing", + }); + } + return yield* ready + .run("xcrun", ["simctl", "spawn", udid, helper, ...args]) + .pipe(Effect.flatMap(ok("accessibility"))); +}); + +const runIos = Effect.fn("DeviceActions.runIos")(function* ( + ready: DeviceHostReady, + input: DeviceActionInput, +) { + const udid = input.deviceId; + const { run } = ready; + switch (input.type) { + case "setAppearance": + yield* simctl(run, udid, ["ui", "appearance", input.value], "appearance"); + return; + case "setTextSize": + yield* simctl(run, udid, ["ui", "content_size", IOS_TEXT_SIZES[input.value]], "text size"); + return; + case "setToggle": + if (input.setting === "increaseContrast") { + yield* simctl( + run, + udid, + ["ui", "increase_contrast", input.value ? "enabled" : "disabled"], + "increase contrast", + ); + return; + } + yield* axSettings(ready, udid, [ + "set", + IOS_TOGGLE_OPTIONS[input.setting]!, + input.value ? "on" : "off", + ]); + return; + case "setLiquidGlass": + yield* axSettings(ready, udid, ["set", "liquid-glass", input.value]); + return; + case "setColorFilter": + yield* axSettings(ready, udid, ["set", "color-filter", input.value]); + return; + case "setLocation": + yield* simctl( + run, + udid, + ["location", "set", `${input.latitude},${input.longitude}`], + "location", + ); + return; + case "clearLocation": + yield* simctl(run, udid, ["location", "clear"], "location"); + return; + case "setPermission": { + if (input.permission === "notifications") { + // simctl lacks notification permissions; serve-sim edits the BulletinBoard plist. + yield* serveSimPermissions(ready, udid, input); + return; + } + const service = + input.permission === "location" ? "location" : IOS_TCC_SERVICES[input.permission]; + if (!service) + return yield* new DeviceActionUnavailableError({ + operation: "permission", + platform: "ios", + reason: "unsupported", + }); + yield* simctl(run, udid, ["privacy", input.decision, service, input.appId], "permission"); + return; + } + case "openUrl": + yield* simctl(run, udid, ["openurl", input.url], "open url"); + return; + case "launchApp": + yield* simctl(run, udid, ["launch", input.appId], "launch"); + return; + case "terminateApp": + yield* simctl(run, udid, ["terminate", input.appId], "terminate"); + return; + case "sendPush": { + const payload = + typeof input.payload === "string" ? { aps: { alert: input.payload } } : input.payload; + const encoded = yield* encodePushPayload(payload).pipe( + Effect.mapError( + () => new DeviceOperationError({ operation: "push", reason: "invalid_payload" }), + ), + ); + yield* run("xcrun", ["simctl", "push", udid, input.appId, "-"], { stdin: encoded }).pipe( + Effect.flatMap(ok("push")), + ); + return; + } + case "shake": + case "setOrientation": + return yield* new DeviceActionUnavailableError({ + operation: input.type, + platform: "ios", + reason: "unsupported", + }); + } +}); + +const serveSimPermissions = Effect.fn("DeviceActions.serveSimPermissions")(function* ( + ready: DeviceHostReady, + udid: string, + input: Extract, +) { + const cli = ready.helpers.serveSimCli; + if (!cli) + return yield* new DeviceActionUnavailableError({ + operation: "permission", + platform: "ios", + reason: "helper_missing", + }); + yield* ready + .run(ready.nodePath, [ + cli, + "permissions", + input.decision, + input.permission, + input.appId, + "-d", + udid, + ]) + .pipe(Effect.flatMap(ok("permission"))); +}); + +const adb = (run: Runner, serial: string, args: ReadonlyArray, operation: string) => + run("adb", ["-s", serial, ...args]).pipe(Effect.flatMap(ok(operation))); + +// adb shell joins arguments without escaping before the device's shell parses them. +const androidShellWord = (value: string) => + /^[\w@%+=:,./-]+$/.test(value) ? value : `'${value.replaceAll("'", "'\\''")}'`; + +const runAndroid = Effect.fn("DeviceActions.runAndroid")(function* ( + run: Runner, + input: DeviceActionInput, +) { + const serial = input.deviceId; + const shell = (args: ReadonlyArray, operation: string) => + adb(run, serial, ["shell", ...args.map(androidShellWord)], operation); + switch (input.type) { + case "setAppearance": + yield* shell(["cmd", "uimode", "night", input.value === "dark" ? "yes" : "no"], "appearance"); + return; + case "setTextSize": + yield* shell( + ["settings", "put", "system", "font_scale", ANDROID_TEXT_SIZES[input.value]], + "text size", + ); + return; + case "setToggle": + if (input.setting === "networkEnabled") { + const state = input.value ? "enable" : "disable"; + yield* shell(["svc", "wifi", state], "network"); + yield* shell(["svc", "data", state], "network"); + return; + } + if (input.setting === "reduceMotion") { + const scale = input.value ? "0" : "1"; + for (const key of [ + "animator_duration_scale", + "transition_animation_scale", + "window_animation_scale", + ]) { + yield* shell(["settings", "put", "global", key, scale], "reduce motion"); + } + return; + } + return yield* new DeviceActionUnavailableError({ + operation: input.type, + platform: "android", + reason: "unsupported", + }); + case "setOrientation": { + const wmSize = yield* quiet(shell(["wm", "size"], "orientation")); + const rotation = isLandscapeNative(wmSize) + ? ANDROID_LANDSCAPE_NATIVE_ROTATION[input.value] + : input.value; + // `user-rotation lock` only rotates window content on recent images; the display the + // encoder captures stays put. Tilting the emulator's accelerometer rotates it for real, so + // that is used whenever the target is an emulator. + if (serial.startsWith("emulator-")) { + yield* shell(["settings", "put", "system", "accelerometer_rotation", "1"], "orientation"); + yield* shell(["cmd", "window", "user-rotation", "free"], "orientation"); + yield* adb( + run, + serial, + ["emu", "sensor", "set", "acceleration", ANDROID_GRAVITY[rotation]], + "orientation", + ); + return; + } + yield* shell( + ["cmd", "window", "user-rotation", "lock", ANDROID_ROTATION[rotation]], + "orientation", + ); + return; + } + case "setLocation": + yield* adb( + run, + serial, + ["emu", "geo", "fix", String(input.longitude), String(input.latitude)], + "location", + ); + return; + case "clearLocation": + // The emulator cannot clear location; keep the fix and refresh the reading. + return; + case "setPermission": { + const permissions = ANDROID_PERMISSIONS[input.permission]; + if (!permissions) { + return yield* new DeviceActionUnavailableError({ + operation: "permission", + platform: "android", + reason: "unsupported", + }); + } + const verb = input.decision === "grant" ? "grant" : "revoke"; + // `pm` rejects a permission the app or system image does not declare, and Photos lists both + // the Android 13 and the older storage permission, so fail only when none of them applied. + const results = yield* Effect.forEach(permissions, (permission) => + shell(["pm", verb, input.appId, permission], "permission").pipe(Effect.result), + ); + const failure = results.find(Result.isFailure); + if (failure && !results.some(Result.isSuccess)) return yield* failure.failure; + return; + } + case "openUrl": + yield* shell( + ["am", "start", "-a", "android.intent.action.VIEW", "-d", input.url], + "open url", + ); + return; + case "launchApp": + yield* shell( + ["monkey", "-p", input.appId, "-c", "android.intent.category.LAUNCHER", "1"], + "launch", + ); + return; + case "terminateApp": + yield* shell(["am", "force-stop", input.appId], "terminate"); + return; + case "setLiquidGlass": + case "setColorFilter": + case "shake": + case "sendPush": + return yield* new DeviceActionUnavailableError({ + operation: input.type, + platform: "android", + reason: "unsupported", + }); + } +}); + +export const readDeviceDetail = Effect.fn("DeviceActions.readDetail")(function* ( + ready: DeviceHostReady, + platform: DevicePlatform, + deviceId: string, +): Effect.fn.Return<{ settings: DeviceSettings; foregroundApp: DeviceForegroundApp | null }> { + return platform === "ios" + ? yield* readIos(ready, deviceId) + : yield* readAndroid(ready.run, deviceId); +}); + +const quiet = (effect: Effect.Effect) => + effect.pipe(Effect.orElseSucceed((): A | undefined => undefined)); + +const readIos = Effect.fn("DeviceActions.readIos")(function* ( + ready: DeviceHostReady, + udid: string, +) { + const { run } = ready; + const uiValue = (option: string) => + quiet( + simctl(run, udid, ["ui", option], option).pipe(Effect.map((out) => out.trim().toLowerCase())), + ); + const [appearance, contentSize, contrast, axStatus] = yield* Effect.all( + [ + uiValue("appearance"), + uiValue("content_size"), + uiValue("increase_contrast"), + quiet(axSettings(ready, udid, ["status"]).pipe(Effect.flatMap(decodeAxStatus))), + ], + { concurrency: 4 }, + ); + const onOff = (value: string | undefined) => + value === "on" ? true : value === "off" ? false : undefined; + const settings: DeviceSettings = { + ...(appearance === "light" || appearance === "dark" ? { appearance } : {}), + ...(contentSize ? { textSize: textSizeFromIos(contentSize) } : {}), + ...(contrast ? { increaseContrast: contrast === "enabled" } : {}), + ...(axStatus + ? { + ...(onOff(axStatus["reduce-motion"]) === undefined + ? {} + : { reduceMotion: onOff(axStatus["reduce-motion"]) }), + ...(onOff(axStatus["reduce-transparency"]) === undefined + ? {} + : { reduceTransparency: onOff(axStatus["reduce-transparency"]) }), + ...(onOff(axStatus["show-borders"]) === undefined + ? {} + : { showBorders: onOff(axStatus["show-borders"]) }), + ...(onOff(axStatus.voiceover) === undefined + ? {} + : { voiceOver: onOff(axStatus.voiceover) }), + ...(axStatus["liquid-glass"] === "clear" || axStatus["liquid-glass"] === "tinted" + ? { liquidGlass: axStatus["liquid-glass"] } + : {}), + ...(isColorFilter(axStatus["color-filter"]) + ? { colorFilter: axStatus["color-filter"] } + : {}), + } + : {}), + }; + return { settings, foregroundApp: null }; +}); + +const isColorFilter = (value: unknown): value is DeviceSettings["colorFilter"] & string => + value === "none" || + value === "grayscale" || + value === "red-green" || + value === "green-red" || + value === "blue-yellow"; + +const readAndroid = Effect.fn("DeviceActions.readAndroid")(function* (run: Runner, serial: string) { + const shell = (args: ReadonlyArray) => + quiet( + adb(run, serial, ["shell", ...args], args[0] ?? "shell").pipe(Effect.map((s) => s.trim())), + ); + const [night, fontScale, animator, wifi, focus] = yield* Effect.all( + [ + shell(["cmd", "uimode", "night"]), + shell(["settings", "get", "system", "font_scale"]), + shell(["settings", "get", "global", "animator_duration_scale"]), + shell(["settings", "get", "global", "wifi_on"]), + // API 36 dropped focus from `dumpsys window windows`; use the unfiltered dump. + shell(["dumpsys", "window"]), + ], + { concurrency: 5 }, + ); + const scale = fontScale && fontScale !== "null" ? Number(fontScale) : Number.NaN; + const focused = focus?.match(/m(?:CurrentFocus|FocusedApp)=\w+\{[^ ]+ u\d+ ([^/ ]+)\//); + const settings: DeviceSettings = { + ...(night?.includes("yes") + ? { appearance: "dark" } + : night?.includes("no") + ? { appearance: "light" } + : {}), + ...(Number.isFinite(scale) ? { textSize: textSizeFromAndroid(scale) } : {}), + ...(animator !== undefined && animator !== "null" + ? { reduceMotion: Number(animator) === 0 } + : {}), + ...(wifi === "1" || wifi === "0" ? { networkEnabled: wifi === "1" } : {}), + }; + return { + settings, + foregroundApp: focused ? { id: focused[1]! } : null, + }; +}); diff --git a/apps/server/src/device/DeviceChannels.test.ts b/apps/server/src/device/DeviceChannels.test.ts new file mode 100644 index 000000000..f9040ed29 --- /dev/null +++ b/apps/server/src/device/DeviceChannels.test.ts @@ -0,0 +1,239 @@ +import * as Http from "node:http"; +import type { AddressInfo } from "node:net"; + +import { WebSocketServer } from "@effect/platform-node/NodeWS"; +import { describe, expect, it } from "@effect/vitest"; +import { + DEVICE_CHANNEL_CHUNK_BYTES, + DEVICE_CHANNEL_WINDOW_BYTES, + type DeviceChannelEvent, +} from "@threadlines/contracts"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Queue from "effect/Queue"; +import * as Stream from "effect/Stream"; +import { FetchHttpClient } from "effect/http"; + +import * as DeviceChannels from "./DeviceChannels.ts"; +import { DeviceService } from "./DeviceService.ts"; + +const TOKEN = "hub-session-token-for-tests"; +const BODY_BYTES = 5 * 1024 * 1024; + +/** A stand-in hub: refuses requests without the token, streams a large body, echoes ws messages. */ +const startFakeHub = Effect.acquireRelease( + Effect.callback<{ origin: string; seen: Array; close: () => void }>((resume) => { + const seen: Array = []; + const server = Http.createServer((request, response) => { + seen.push(`${request.method} ${request.url} ${request.headers.authorization ?? "-"}`); + if (request.headers.authorization !== `Bearer ${TOKEN}`) { + response.writeHead(401).end(); + return; + } + response.writeHead(200, { "content-type": "application/octet-stream" }); + let sent = 0; + const pump = () => { + while (sent < BODY_BYTES) { + const size = Math.min(64 * 1024, BODY_BYTES - sent); + sent += size; + if (!response.write(Buffer.alloc(size, 7))) { + response.once("drain", pump); + return; + } + } + response.end(); + }; + pump(); + }); + const sockets = new WebSocketServer({ noServer: true }); + server.on("upgrade", (request, socket, head) => { + seen.push(`WS ${request.url} ${request.headers.authorization ?? "-"}`); + if (request.headers.authorization !== `Bearer ${TOKEN}`) { + socket.destroy(); + return; + } + sockets.handleUpgrade(request, socket, head, (ws) => { + ws.on("message", (data, isBinary) => ws.send(data, { binary: isBinary })); + }); + }); + server.listen(0, "127.0.0.1", () => { + const { port } = server.address() as AddressInfo; + resume( + Effect.succeed({ + origin: `http://127.0.0.1:${port}`, + seen, + close: () => { + sockets.close(); + server.closeAllConnections(); + server.close(); + }, + }), + ); + }); + }), + (hub) => Effect.sync(() => hub.close()), +); + +const channelsFor = (origin: string) => + DeviceChannels.make.pipe( + Effect.provide( + Layer.mergeAll( + Layer.mock(DeviceService)({ + currentHub: Effect.succeed({ origin, token: TOKEN }), + }), + FetchHttpClient.layer, + ), + ), + ); + +describe("isAllowedDevicePath", () => { + it("keeps serve-sim's and serve-emu's /api bodies, which carry the exec token, out of reach", () => { + expect(DeviceChannels.isAllowedDevicePath("GET", "/vendor/serve-sim/api")).toBe(false); + expect(DeviceChannels.isAllowedDevicePath("GET", "/vendor/serve-emu/api")).toBe(false); + expect(DeviceChannels.isAllowedDevicePath("ws", "/vendor/serve-sim/exec-ws")).toBe(false); + expect(DeviceChannels.isAllowedDevicePath("POST", "/api/devices/boot")).toBe(false); + }); + + it("refuses segments a URL parser would rewrite", () => { + expect( + DeviceChannels.isAllowedDevicePath("GET", "/vendor/serve-sim/helper/../stream.avcc"), + ).toBe(false); + expect( + DeviceChannels.isAllowedDevicePath("GET", "/vendor/serve-sim/helper/%2e%2e/stream.avcc"), + ).toBe(false); + }); + + it("allows the viewer's stream and input routes", () => { + expect( + DeviceChannels.isAllowedDevicePath("GET", "/vendor/serve-sim/helper/ABC-1/stream.avcc"), + ).toBe(true); + expect(DeviceChannels.isAllowedDevicePath("ws", "/vendor/serve-sim/helper/ws")).toBe(true); + expect(DeviceChannels.isAllowedDevicePath("ws", "/vendor/serve-emu/ws")).toBe(true); + expect(DeviceChannels.isAllowedDevicePath("POST", "/vendor/serve-emu/api/stream-mode")).toBe( + true, + ); + }); +}); + +describe("DeviceChannels", () => { + it.live("refuses a forbidden path without reaching the hub", () => + Effect.scoped( + Effect.gen(function* () { + const hub = yield* startFakeHub; + const channels = yield* channelsFor(hub.origin); + const error = yield* channels + .open("connection", { + _tag: "http", + channelId: "c1", + method: "GET", + path: "/vendor/serve-sim/api", + }) + .pipe(Effect.flip); + expect(error.reason).toBe("forbidden"); + expect(hub.seen).toEqual([]); + }), + ), + ); + + it.live("adds the hub token, slices the body, and holds it at the credit window", () => + Effect.scoped( + Effect.gen(function* () { + const hub = yield* startFakeHub; + const channels = yield* channelsFor(hub.origin); + const events = yield* channels.open("connection", { + _tag: "http", + channelId: "c1", + method: "GET", + path: "/vendor/serve-sim/helper/SIM-1/stream.avcc", + query: { token: "client-supplied", fps: "30" }, + }); + const received = yield* Queue.unbounded(); + const reader = yield* events.pipe( + Stream.runForEach((event) => Queue.offer(received, event)), + Effect.forkScoped, + ); + let delivered = 0; + const take = Effect.gen(function* () { + const event = yield* Queue.take(received); + if (event._tag === "data") { + const size = Buffer.from(event.data, "base64").length; + expect(size).toBeLessThanOrEqual(DEVICE_CHANNEL_CHUNK_BYTES); + delivered += size; + } + return event; + }); + expect((yield* take)._tag).toBe("opened"); + // Uncredited, the channel stops at the window instead of reading the whole body. + yield* Effect.sleep("300 millis"); + while ((yield* Queue.size(received)) > 0) yield* take; + expect(delivered).toBeGreaterThanOrEqual(DEVICE_CHANNEL_WINDOW_BYTES); + expect(delivered).toBeLessThan(DEVICE_CHANNEL_WINDOW_BYTES + DEVICE_CHANNEL_CHUNK_BYTES); + // Crediting lets the rest through. + let ended = false; + while (!ended) { + yield* channels.credit("connection", { channelId: "c1", bytes: delivered || 1 }); + const before = delivered; + const event = yield* take; + ended = event._tag === "end"; + if (delivered > before) { + yield* channels.credit("connection", { channelId: "c1", bytes: delivered - before }); + } + } + expect(delivered).toBe(BODY_BYTES); + yield* Fiber.join(reader); + // The client's token is dropped and the server's bearer added. + expect(hub.seen).toEqual([ + `GET /vendor/serve-sim/helper/SIM-1/stream.avcc?fps=30 Bearer ${TOKEN}`, + ]); + }), + ), + ); + + it.live("relays ws messages both ways with the hub token", () => + Effect.scoped( + Effect.gen(function* () { + const hub = yield* startFakeHub; + const channels = yield* channelsFor(hub.origin); + const events = yield* channels.open("connection", { + _tag: "ws", + channelId: "w1", + path: "/vendor/serve-sim/helper/ws", + }); + const received = yield* Queue.unbounded(); + yield* events.pipe( + Stream.runForEach((event) => Queue.offer(received, event)), + Effect.forkScoped, + ); + expect((yield* Queue.take(received))._tag).toBe("opened"); + // Sent out of order, written in order. + yield* Effect.all( + [ + channels.send("connection", { channelId: "w1", seq: 1, data: "second", binary: false }), + channels.send("connection", { + channelId: "w1", + seq: 0, + data: Buffer.from("first").toString("base64"), + binary: true, + }), + ], + { concurrency: "unbounded" }, + ); + const first = yield* Queue.take(received); + const second = yield* Queue.take(received); + expect(first).toEqual({ + _tag: "message", + data: Buffer.from("first").toString("base64"), + binary: true, + final: true, + }); + expect(second).toEqual({ _tag: "message", data: "second", binary: false, final: true }); + expect(hub.seen).toEqual([`WS /vendor/serve-sim/helper/ws Bearer ${TOKEN}`]); + // The connection going away closes its channels. + yield* channels.releaseConnection("connection"); + const end = yield* Queue.take(received); + expect(end._tag).toBe("end"); + }), + ), + ); +}); diff --git a/apps/server/src/device/DeviceChannels.ts b/apps/server/src/device/DeviceChannels.ts new file mode 100644 index 000000000..41851f5b6 --- /dev/null +++ b/apps/server/src/device/DeviceChannels.ts @@ -0,0 +1,467 @@ +/** + * The device channel (docs/design/devices.md, "Transport"): every byte + * between a Device tab and the hub rides the client's WS RPC connection, so + * the desktop app, a browser, a phone and a saved computer over the relay all + * reach the hub the same way. Modeled on PreviewTunnel. + * + * The hub is never reachable from a client directly. A channel opens only on + * a path the allowlist below names, with the hub's token added here (clients + * never see it) and nothing of the client's but the path, query and body. + * serve-sim's `/api` body carries the token of its shell-exec route, so + * neither `/vendor/serve-sim/api` nor `/vendor/serve-emu/api` is allowed. + * + * Flow control, so a suspended phone cannot pile video up in the server: + * events carry at most DEVICE_CHANNEL_CHUNK_BYTES; at most + * DEVICE_CHANNEL_WINDOW_BYTES may be sent and not yet credited. An http body + * then simply stops being read (the hub's socket backs up). A ws upstream is + * paused; when more than DEVICE_CHANNEL_SLOW_BYTES queue up past the window + * anyway, the channel fails with `slow` and the client opens a fresh one. + */ +import { + DEVICE_CHANNEL_CHUNK_BYTES, + DEVICE_CHANNEL_WINDOW_BYTES, + type DeviceChannelCredit, + DeviceChannelError, + type DeviceChannelEvent, + type DeviceChannelOpen, + type DeviceChannelSend, +} from "@threadlines/contracts"; +import { WebSocket } from "@effect/platform-node/NodeWS"; +import * as Cause from "effect/Cause"; +import * as Context from "effect/Context"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Latch from "effect/Latch"; +import * as Layer from "effect/Layer"; +import * as Queue from "effect/Queue"; +import type * as Scope from "effect/Scope"; +import * as Stream from "effect/Stream"; +import { HttpClient, HttpClientRequest } from "effect/http"; + +import { DeviceService } from "./DeviceService.ts"; +import { repeatStillFrame } from "./mjpegStillFrame.ts"; + +/** Channels one connection may hold open: a Device tab uses two or three. */ +export const DEVICE_CHANNEL_MAX_PER_CONNECTION = 16; +/** What a paused ws upstream may still queue past the window before the channel fails `slow`. */ +export const DEVICE_CHANNEL_SLOW_BYTES = 4 * 1024 * 1024; +const OPEN_TIMEOUT_MS = 10_000; +const MAX_SENDS_OUT_OF_ORDER = 64; + +/** Read routes and stream routes, for GET. */ +const GET_PATHS: ReadonlyArray = [ + /^\/api\/devices$/, + /^\/vendor\/serve-sim\/api\/screenshot$/, + /^\/vendor\/serve-sim\/api\/event-log(\/events)?$/, + /^\/vendor\/serve-sim\/helper\/[^/]+\/(stream\.mjpeg|stream\.avcc|config|health|ax|foreground)$/, + /^\/vendor\/serve-sim\/helper\/[^/]+\/panel\/(1|3)\/stream\.avcc$/, + /^\/vendor\/serve-sim\/appstate$/, + /^\/vendor\/serve-emu\/api\/(devices|screenshot|stream-mode|stream-settings|accessibility|fold)$/, + /^\/vendor\/serve-emu\/health$/, +]; + +/** The few routes that take a POST: screenshot capture and stream tuning. */ +const POST_PATHS: ReadonlyArray = [ + /^\/vendor\/serve-sim\/api\/screenshot$/, + /^\/vendor\/serve-emu\/api\/(screenshot|stream-mode|stream-settings|fold)$/, +]; + +/** Sockets: the device list, iOS input, and Android's video and input. */ +const WS_PATHS: ReadonlyArray = [ + /^\/api\/devices\/ws$/, + /^\/vendor\/serve-sim\/helper\/ws$/, + /^\/vendor\/serve-emu\/ws$/, +]; + +const SCREENSHOT_PATHS = new Set([ + "/vendor/serve-sim/api/screenshot", + "/vendor/serve-emu/api/screenshot", +]); + +/** + * Whether a client may open `path` this way. Segments that a URL parser + * would rewrite (`.`, `..`, escapes, backslashes, empty) are refused outright, + * so what is checked is what reaches the hub. + */ +export const isAllowedDevicePath = (kind: "GET" | "POST" | "ws", path: string): boolean => { + const segments = path.split("/").slice(1); + if ( + segments.some( + (segment) => + segment === "" || + segment === "." || + segment === ".." || + segment.includes("%") || + segment.includes("\\"), + ) + ) { + return false; + } + const patterns = kind === "ws" ? WS_PATHS : kind === "POST" ? POST_PATHS : GET_PATHS; + return patterns.some((pattern) => pattern.test(path)); +}; + +/** The query a client sent, without anything that could carry a token of its own. */ +const upstreamSearch = (query: Readonly> | undefined) => { + const params = new URLSearchParams(); + for (const [name, value] of Object.entries(query ?? {})) { + if (name.toLowerCase() === "token") continue; + params.append(name, value); + } + const text = params.toString(); + return text === "" ? "" : `?${text}`; +}; + +const oneEvent = (event: DeviceChannelEvent): Stream.Stream => + Stream.succeed(event); + +const failure = (reason: DeviceChannelError["reason"], detail: string) => + new DeviceChannelError({ reason, detail }); + +/** Slices of at most DEVICE_CHANNEL_CHUNK_BYTES, base64. */ +const toChunks = (bytes: Uint8Array): ReadonlyArray => { + const buffer = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const chunks: Array = []; + for (let offset = 0; offset < buffer.length; offset += DEVICE_CHANNEL_CHUNK_BYTES) { + chunks.push(buffer.subarray(offset, offset + DEVICE_CHANNEL_CHUNK_BYTES).toString("base64")); + } + return chunks; +}; + +interface PendingSend { + readonly data: Buffer | string; + readonly done: Deferred.Deferred; +} + +interface Channel { + /** Bytes sent down the channel that the client has not yet credited. */ + unacknowledged: number; + /** Credit arrived: let a waiting http body continue, or resume a paused socket. */ + readonly onCredit: () => void; + /** Set for a ws channel. */ + readonly socket: WebSocket | null; + nextSeq: number; + readonly waiting: Map; +} + +export interface DeviceChannelsShape { + /** Opens one channel. Scoped: the subscription ending closes it. */ + readonly open: ( + connectionId: string, + input: DeviceChannelOpen, + ) => Effect.Effect< + Stream.Stream, + DeviceChannelError, + Scope.Scope + >; + /** One ws message for the hub, in `seq` order; done once written to the hub's socket. */ + readonly send: ( + connectionId: string, + input: DeviceChannelSend, + ) => Effect.Effect; + /** The client's decoder took more of a channel's data. */ + readonly credit: (connectionId: string, input: DeviceChannelCredit) => Effect.Effect; + /** The connection went away: every channel it held closes. */ + readonly releaseConnection: (connectionId: string) => Effect.Effect; +} + +export class DeviceChannels extends Context.Service()( + "@threadlines/server/device/DeviceChannels", +) {} + +export const make = Effect.gen(function* () { + const devices = yield* DeviceService; + const httpClient = (yield* HttpClient.HttpClient).pipe(HttpClient.withScope); + const channels = new Map(); + /** Every channel's closer, by connection, for `releaseConnection`. */ + const closers = new Map void>>(); + const keyOf = (connectionId: string, channelId: string) => `${connectionId}\u0000${channelId}`; + + const settleWaiting = (channel: Channel, error: DeviceChannelError) => { + for (const pending of channel.waiting.values()) + Deferred.doneUnsafe(pending.done, Exit.fail(error)); + channel.waiting.clear(); + }; + + /** Holds the connection's slot and the channel's entry until the scope closes. */ + const register = (connectionId: string, channelId: string, channel: Channel, close: () => void) => + Effect.gen(function* () { + const key = keyOf(connectionId, channelId); + const held = closers.get(connectionId) ?? new Map void>(); + if (channels.has(key)) return yield* failure("limit", "That channel is already open."); + if (held.size >= DEVICE_CHANNEL_MAX_PER_CONNECTION) { + return yield* failure("limit", `Too many device channels are open (${held.size}).`); + } + held.set(channelId, close); + closers.set(connectionId, held); + channels.set(key, channel); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + if (channels.get(key) === channel) channels.delete(key); + held.delete(channelId); + if (held.size === 0 && closers.get(connectionId) === held) closers.delete(connectionId); + settleWaiting(channel, failure("broken", "The channel closed.")); + close(); + }), + ); + }); + + const hubOrUnavailable = devices.currentHub.pipe( + Effect.flatMap((hub) => + hub + ? Effect.succeed(hub) + : Effect.fail(failure("unavailable", "The device hub is not running.")), + ), + ); + + const openHttp = Effect.fn("DeviceChannels.openHttp")(function* ( + connectionId: string, + input: Extract, + ) { + if (!isAllowedDevicePath(input.method, input.path)) { + return yield* failure("forbidden", "That device path is not available."); + } + const credited = yield* Latch.make(true); + const channel: Channel = { + unacknowledged: 0, + onCredit: () => { + if (channel.unacknowledged < DEVICE_CHANNEL_WINDOW_BYTES) Latch.openUnsafe(credited); + }, + socket: null, + nextSeq: 0, + waiting: new Map(), + }; + yield* register(connectionId, input.channelId, channel, () => Latch.openUnsafe(credited)); + + // Waits for credit before each slice goes out: the body is pulled only as fast as the client takes it. + const paced = (slices: Stream.Stream) => + slices.pipe( + Stream.mapEffect((data) => + Effect.gen(function* () { + yield* credited.await; + channel.unacknowledged += Math.floor((data.length * 3) / 4); + if (channel.unacknowledged >= DEVICE_CHANNEL_WINDOW_BYTES) Latch.closeUnsafe(credited); + return { _tag: "data", data } as const; + }), + ), + ); + + // A screenshot asked for by device is the server's own, checked to be one PNG. + const screenshotDevice = input.query?.device; + if (SCREENSHOT_PATHS.has(input.path) && screenshotDevice) { + const shot = yield* devices + .screenshot(screenshotDevice) + .pipe(Effect.mapError((error) => failure("unreachable", error.message))); + return Stream.concat( + oneEvent({ _tag: "opened", status: 200, contentType: "image/png" }), + Stream.concat(paced(Stream.fromIterable(toChunks(shot.png))), oneEvent({ _tag: "end" })), + ); + } + + const hub = yield* hubOrUnavailable; + const url = `${hub.origin}${input.path}${upstreamSearch(input.query)}`; + const request = ( + input.method === "POST" ? HttpClientRequest.post(url) : HttpClientRequest.get(url) + ).pipe( + HttpClientRequest.bearerToken(hub.token), + HttpClientRequest.setHeader("accept-encoding", "identity"), + input.body === undefined + ? (self) => self + : HttpClientRequest.bodyUint8Array( + Buffer.from(input.body, "base64"), + input.contentType ?? "application/octet-stream", + ), + ); + const response = yield* httpClient.execute(request).pipe( + Effect.timeout(OPEN_TIMEOUT_MS), + Effect.mapError(() => failure("unreachable", "The device hub did not answer.")), + ); + const contentType = response.headers["content-type"]; + // A multipart MJPEG body re-sends its last frame once the screen goes still. + const body = contentType?.startsWith("multipart/x-mixed-replace") + ? repeatStillFrame(response.stream) + : response.stream; + const slices = body.pipe( + Stream.mapError(() => failure("broken", "The device hub stopped sending.")), + Stream.flatMap((bytes) => Stream.fromIterable(toChunks(bytes))), + ); + return Stream.concat( + oneEvent({ + _tag: "opened", + status: response.status, + ...(contentType ? { contentType } : {}), + }), + Stream.concat(paced(slices), oneEvent({ _tag: "end" })), + ); + }); + + const openWs = Effect.fn("DeviceChannels.openWs")(function* ( + connectionId: string, + input: Extract, + ) { + if (!isAllowedDevicePath("ws", input.path)) { + return yield* failure("forbidden", "That device path is not available."); + } + const hub = yield* hubOrUnavailable; + const url = `${hub.origin.replace(/^http/, "ws")}${input.path}${upstreamSearch(input.query)}`; + const queue = yield* Queue.unbounded(); + let paused = false; + // An `error` with no listener would take the server down: one is on from the start. + const socket = new WebSocket(url, input.protocols ? [...input.protocols] : [], { + headers: { authorization: `Bearer ${hub.token}` }, + handshakeTimeout: OPEN_TIMEOUT_MS, + perMessageDeflate: false, + }); + socket.binaryType = "nodebuffer"; + const channel: Channel = { + unacknowledged: 0, + onCredit: () => { + if (paused && channel.unacknowledged < DEVICE_CHANNEL_WINDOW_BYTES / 2) { + paused = false; + socket.resume(); + } + }, + socket, + nextSeq: 0, + waiting: new Map(), + }; + const fail = (error: DeviceChannelError) => { + settleWaiting(channel, error); + Queue.failCauseUnsafe(queue, Cause.fail(error)); + socket.terminate(); + }; + socket.on("error", (error) => fail(failure("broken", error.message))); + yield* register(connectionId, input.channelId, channel, () => socket.terminate()); + + socket.on("message", (data: Buffer | ArrayBuffer | Array, isBinary: boolean) => { + const bytes = Array.isArray(data) ? Buffer.concat(data) : Buffer.from(data as Buffer); + if (isBinary) { + const parts = toChunks(bytes); + parts.forEach((part, index) => + Queue.offerUnsafe(queue, { + _tag: "message", + data: part, + binary: true, + final: index === parts.length - 1, + }), + ); + if (parts.length === 0) + Queue.offerUnsafe(queue, { _tag: "message", data: "", binary: true, final: true }); + } else { + const text = bytes.toString("utf8"); + const step = DEVICE_CHANNEL_CHUNK_BYTES / 4; + for (let offset = 0; offset < Math.max(text.length, 1); offset += step) { + Queue.offerUnsafe(queue, { + _tag: "message", + data: text.slice(offset, offset + step), + binary: false, + final: offset + step >= text.length, + }); + } + } + channel.unacknowledged += bytes.length; + if (channel.unacknowledged > DEVICE_CHANNEL_WINDOW_BYTES + DEVICE_CHANNEL_SLOW_BYTES) { + fail(failure("slow", "The device stream got too far ahead of this client.")); + return; + } + if (!paused && channel.unacknowledged >= DEVICE_CHANNEL_WINDOW_BYTES) { + paused = true; + socket.pause(); + } + }); + socket.on("close", (code: number, reason: Buffer) => { + settleWaiting(channel, failure("broken", "The device socket closed.")); + const text = reason.toString("utf8"); + Queue.offerUnsafe(queue, { _tag: "end", code, ...(text ? { reason: text } : {}) }); + Queue.endUnsafe(queue); + }); + socket.once("open", () => { + Queue.offerUnsafe(queue, { + _tag: "opened", + ...(socket.protocol ? { protocol: socket.protocol } : {}), + }); + }); + yield* Effect.callback((resume) => { + if (socket.readyState === WebSocket.OPEN) return resume(Effect.void); + socket.once("open", () => resume(Effect.void)); + socket.once("unexpected-response", (_request, response) => { + resume( + Effect.fail( + failure("unreachable", `The device hub refused the socket (${response.statusCode}).`), + ), + ); + socket.terminate(); + }); + socket.once("close", () => + resume(Effect.fail(failure("unreachable", "The device hub closed the socket."))), + ); + socket.once("error", (error) => resume(Effect.fail(failure("unreachable", error.message)))); + }); + return Stream.fromQueue(queue); + }); + + const open: DeviceChannelsShape["open"] = (connectionId, input) => + input._tag === "http" ? openHttp(connectionId, input) : openWs(connectionId, input); + + const send: DeviceChannelsShape["send"] = (connectionId, input) => + Effect.gen(function* () { + const channel = channels.get(keyOf(connectionId, input.channelId)); + if (channel === undefined || channel.socket === null) { + return yield* failure("unknownChannel", "That device channel is not open."); + } + const socket = channel.socket; + if ( + input.seq < channel.nextSeq || + channel.waiting.has(input.seq) || + channel.waiting.size >= MAX_SENDS_OUT_OF_ORDER + ) { + const error = failure("limit", "The channel's messages arrived out of all order."); + settleWaiting(channel, error); + socket.terminate(); + return yield* error; + } + const done = yield* Deferred.make(); + channel.waiting.set(input.seq, { + data: input.binary ? Buffer.from(input.data, "base64") : input.data, + done, + }); + // Writes every message whose turn has come, in order. + for (;;) { + const next = channel.waiting.get(channel.nextSeq); + if (next === undefined) break; + channel.waiting.delete(channel.nextSeq); + channel.nextSeq += 1; + if (socket.readyState !== WebSocket.OPEN) { + Deferred.doneUnsafe(next.done, Exit.fail(failure("broken", "The device socket closed."))); + continue; + } + socket.send(next.data, { binary: typeof next.data !== "string" }, (error) => + Deferred.doneUnsafe( + next.done, + error ? Exit.fail(failure("broken", error.message)) : Exit.void, + ), + ); + } + yield* Deferred.await(done); + }); + + const credit: DeviceChannelsShape["credit"] = (connectionId, input) => + Effect.sync(() => { + const channel = channels.get(keyOf(connectionId, input.channelId)); + if (channel === undefined) return; + channel.unacknowledged = Math.max(0, channel.unacknowledged - input.bytes); + channel.onCredit(); + }); + + const releaseConnection: DeviceChannelsShape["releaseConnection"] = (connectionId) => + Effect.sync(() => { + const held = closers.get(connectionId); + if (!held) return; + for (const close of held.values()) close(); + }); + + return DeviceChannels.of({ open, send, credit, releaseConnection }); +}); + +export const layer = Layer.effect(DeviceChannels, make); diff --git a/apps/server/src/device/DeviceHost.ts b/apps/server/src/device/DeviceHost.ts new file mode 100644 index 000000000..a036b24e1 --- /dev/null +++ b/apps/server/src/device/DeviceHost.ts @@ -0,0 +1,151 @@ +/** + * The seam between the device service and the machine (docs/design/devices.md). + * One implementation, LocalDeviceHost: a Threadlines server has one device + * host, its own machine. A Mac linked in Fleet shows its devices through its + * own server. + */ +import type { + DeviceBootError, + DeviceHostUnavailableError, + DeviceOperationError, + DevicePlatform, + DevicePlatformAvailability, + DeviceSummary, + DeviceToolVersions, +} from "@threadlines/contracts"; +import * as Context from "effect/Context"; +import type * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +/** Something went wrong starting or running the device tools; `step` says where. */ +export class DeviceHostError extends Schema.TaggedError()("DeviceHostError", { + step: Schema.String, + /** Words the user can act on, when there are some. */ + detail: Schema.optional(Schema.String), + cause: Schema.optional(Schema.Defect()), +}) { + override get message(): string { + return this.detail ?? `Device support failed while ${this.step}.`; + } +} + +/** The running hub: where it listens and the session token it printed. Never leaves the server. */ +export interface DeviceHubEndpoint { + readonly origin: string; + readonly token: string; +} + +/** The running agent-device daemon, read from its own `daemon.json`. */ +export interface AgentDeviceEndpoint { + readonly baseUrl: string; + readonly token: string; + /** The agent-device CLI that talks to it. */ + readonly entryPath: string; +} + +/** Discovery and lifecycle for the devices a ready host can run. */ +export interface DeviceOperations { + readonly list: Effect.Effect< + { + readonly devices: ReadonlyArray; + readonly detail: string | undefined; + /** Platforms that could not be listed; their last known devices stay listed. */ + readonly incompletePlatforms: ReadonlyArray; + }, + DeviceOperationError + >; + /** + * Boots the device and prepares its stream. Android AVDs change id when they + * boot (AVD name to emulator serial), so the returned id is authoritative. + */ + readonly boot: ( + device: DeviceSummary, + ) => Effect.Effect; + /** Prepares the stream of a device that is already booted. */ + readonly attach: (device: DeviceSummary) => Effect.Effect; + readonly shutdown: (device: { + readonly id: string; + readonly platform: DevicePlatform; + }) => Effect.Effect; + readonly screenshot: (device: DeviceSummary) => Effect.Effect; +} + +export interface DeviceRunResult { + readonly stdout: string; + readonly stderr: string; + /** 124 on timeout, 127 when it could not start. */ + readonly code: number; +} + +export interface DeviceHostReady { + /** The Node.js the tools run on (Threadlines' managed one). */ + readonly nodePath: string; + readonly hub: DeviceHubEndpoint; + readonly devices: DeviceOperations; + /** Runs a command with the Android SDK on PATH. Never fails: see `code`. */ + readonly run: ( + command: string, + args: ReadonlyArray, + options?: { + readonly timeoutMs?: number; + readonly stdin?: string; + readonly maxOutputBytes?: number; + }, + ) => Effect.Effect; + /** serve-sim's helper binaries, for the Tools drawer. Null where the install lacks one. */ + readonly helpers: { + readonly serveSimAxSettings: string | null; + readonly serveSimCli: string | null; + }; + /** + * Ends an agent-device session on the running daemon. Present only while + * the daemon runs; cleanup never starts a daemon or a device. + */ + readonly releaseAgentSession?: (session: string) => Effect.Effect; +} + +export interface DeviceHostAgentReady extends DeviceHostReady { + readonly agentDevice: AgentDeviceEndpoint; +} + +export interface DeviceHostSummary { + readonly platforms: ReadonlyArray; + readonly hubInstalled: boolean; + readonly agentDeviceInstalled: boolean; + readonly tools: DeviceToolVersions | undefined; + readonly toolInspectionError: string | undefined; +} + +export type DeviceHostPhase = "installing" | "starting"; + +export class DeviceHost extends Context.Service< + DeviceHost, + { + readonly summary: Effect.Effect; + readonly platformAvailability: ( + platform: DevicePlatform, + ) => Effect.Effect; + /** Installs (when missing) and starts the hub. */ + readonly ensureReady: ( + onPhase: (phase: DeviceHostPhase, detail?: string) => Effect.Effect, + ) => Effect.Effect; + /** As `ensureReady`, and installs and starts the agent-device daemon too. */ + readonly ensureAgentReady: ( + onPhase: (phase: DeviceHostPhase, detail?: string) => Effect.Effect, + ) => Effect.Effect; + /** Installs the pinned version of a tool; restarts it if it was running. */ + readonly updateTool: (tool: "hub" | "agent") => Effect.Effect; + /** The running hub, without starting anything. */ + readonly current: Effect.Effect; + /** + * Ends an agent-device session on whatever daemon runs for this state + * folder, this server's or one an earlier run left. Never starts one. + * Best effort: a daemon that is gone took its sessions with it. + */ + readonly releaseAgentSession: (session: string) => Effect.Effect; + /** Stops the agent-device daemon through its own CLI. */ + readonly stopAgent: Effect.Effect; + /** Stops the hub and the daemon. Simulators and emulators keep running; the user owns those. */ + readonly stop: Effect.Effect; + } +>()("@threadlines/server/device/DeviceHost") {} diff --git a/apps/server/src/device/DeviceHubOperations.ts b/apps/server/src/device/DeviceHubOperations.ts new file mode 100644 index 000000000..a17cccf52 --- /dev/null +++ b/apps/server/src/device/DeviceHubOperations.ts @@ -0,0 +1,407 @@ +/** + * List, boot, shut down and screenshot through the device hub, with the hub's + * session token on every request (docs/design/devices.md). Ported from T3 + * Code's DeviceHubOperations. + */ +import { + DeviceBootError, + DeviceOperationError, + type DevicePlatform, + type DeviceSummary, +} from "@threadlines/contracts"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import { HttpClient, HttpClientError, HttpClientRequest, HttpClientResponse } from "effect/http"; + +import type { DeviceHostReady, DeviceOperations } from "./DeviceHost.ts"; + +type HostDevice = DeviceSummary; + +const BOOT_TIMEOUT = Duration.minutes(3); +const SCREENSHOT_TIMEOUT = Duration.seconds(20); +// Let the hub's 30 s simctl timeout report its cause first. +const LIST_TIMEOUT_SECONDS = 40; +// expo-device-hub 0.19.1 reports a failed simulator listing only as text; its error ids are hashes. +const SIMULATOR_LIST_FAILED = + /^\[apple-utils\] (?:Failed to (?:run `xcrun simctl list devices|parse JSON output:|list devices:)|`xcrun simctl list devices --json` did not respond within )/; +// The hub passes numeric display IDs that Xcode 26 rejects; iOS 27 needs them for multiple displays. +// expo-device-hub 0.22.1 returns no Android devices when avdmanager fails, before it runs adb. +const AVDMANAGER_LIST_FAILED = "[android-utils] Failed to run `avdmanager list avd`"; +const GETPROP_LINE = /^\[([^\]]+)\]: \[(.*)\]$/; +const runsOnXcode26Runtime = (version: string) => Number(/^iOS (\d+)/.exec(version)?.[1]) <= 26; +const DEFAULT_DISPLAY_SCREENSHOT = `f=$(mktemp -t threadlines-screenshot) || exit 1 +trap 'rm -f "$f"' 0 +xcrun simctl io "$1" screenshot --type=png "$f" >/dev/null && base64 -i "$f"`; + +const HubDevice = Schema.Struct({ + id: Schema.String, + name: Schema.String, + version: Schema.String, + platform: Schema.Literals(["ios", "android"]), + booted: Schema.Boolean, + physical: Schema.Boolean, +}); +const HubDeviceList = Schema.Struct({ + simulators: Schema.Array(HubDevice), + emulators: Schema.Array(HubDevice), + errors: Schema.optional( + Schema.Array(Schema.Struct({ message: Schema.String, error: Schema.optional(Schema.String) })), + ), +}); +const HubActionResult = Schema.Struct({ + ok: Schema.Boolean, + id: Schema.optional(Schema.String), + serial: Schema.optional(Schema.String), + error: Schema.optional(Schema.String), + errors: Schema.optional( + Schema.Array(Schema.Struct({ message: Schema.String, error: Schema.optional(Schema.String) })), + ), +}); + +/** A failure's class and first line, never a stack: this text reaches the browser. */ +const briefCause = (cause: unknown): string | undefined => { + if (typeof cause !== "object" || cause === null) return undefined; + const tag = "_tag" in cause && typeof cause._tag === "string" ? cause._tag : undefined; + const message = + "message" in cause && typeof cause.message === "string" + ? cause.message.split(/\r?\n/)[0] + : undefined; + const text = [tag, message].filter(Boolean).join(": "); + return text ? boundDiagnostics(text) : undefined; +}; + +// Hub diagnostics can carry a whole stderr dump. +const boundDiagnostics = (text: string) => + text.length > 2_000 ? `${text.slice(0, 1_999)}…` : text; + +// The hub's `error` is a util.inspect of Node's exec error; the stack and the +// inspected fields follow the first frame, so keep the lines before it. +const errorHead = (error: string) => { + const lines = error.split(/\r?\n/); + const firstFrame = lines.findIndex((line) => /^\s+at /.test(line)); + return (firstFrame === -1 ? lines : lines.slice(0, firstFrame)) + .map((line) => line.trim()) + .filter(Boolean) + .join(" "); +}; + +const vendorPrefix = (platform: DevicePlatform) => + platform === "ios" ? "/vendor/serve-sim" : "/vendor/serve-emu"; + +/** Device operations for a host that runs expo-device-hub at `origin`. */ +export const makeDeviceHubOperations = (options: { + readonly origin: string; + /** The hub's session token (`--require-token`), sent as a bearer on every request. */ + readonly token: string; + readonly http: HttpClient.HttpClient; + readonly run: DeviceHostReady["run"]; + /** Android gates listing AVDs, which the hub does not report until they boot. */ + readonly platformAvailable: (platform: DevicePlatform) => Effect.Effect; +}): DeviceOperations => { + const { origin, run } = options; + const httpClient = options.http.pipe( + HttpClient.mapRequest(HttpClientRequest.bearerToken(options.token)), + HttpClient.withScope, + ); + + const fetchJson = Effect.fn("DeviceHubOperations.fetchJson")(function* ( + request: HttpClientRequest.HttpClientRequest, + schema: Schema.Codec, + ) { + return yield* httpClient + .execute(request) + .pipe( + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap(HttpClientResponse.schemaBodyJson(schema)), + Effect.scoped, + ); + }); + + const hubJson = ( + request: HttpClientRequest.HttpClientRequest, + schema: Schema.Codec, + operation: string, + timeout: Duration.Input = Duration.seconds(15), + ) => + fetchJson(request, schema).pipe( + Effect.timeout(timeout), + Effect.mapError( + (cause) => + new DeviceOperationError({ + operation, + reason: "request_failed", + detail: briefCause(cause), + }), + ), + ); + + const postJson = (path: string, body: Record, operation: string) => + HttpClientRequest.post(`${origin}${path}`).pipe( + HttpClientRequest.bodyJson(body), + Effect.mapError(() => new DeviceOperationError({ operation, reason: "invalid_payload" })), + ); + + // Booting alone does not attach a serve-sim helper; the grid start does both + // and is idempotent for a booted simulator. + const attachIosStream = (udid: string, operation: string) => + postJson(`${vendorPrefix("ios")}/grid/api/start`, { udid }, operation).pipe( + Effect.flatMap((request) => hubJson(request, HubActionResult, "attach stream", BOOT_TIMEOUT)), + ); + + // Mirrors the hub's own adb discovery for when it skips it. + const listAdbDevices = Effect.gen(function* () { + const listed = yield* run("adb", ["devices", "-l"]); + if (listed.code !== 0) return undefined; + const serials = listed.stdout + .split(/\r?\n/) + .map((line) => line.trim().split(/\s+/)) + .flatMap(([serial, state]) => (serial && state === "device" ? [serial] : [])); + return yield* Effect.forEach(serials, (serial) => + Effect.gen(function* () { + const getprop = yield* run("adb", ["-s", serial, "shell", "getprop"]); + const props = new Map( + getprop.stdout.split(/\r?\n/).flatMap((line) => { + const match = GETPROP_LINE.exec(line.trim()); + return match ? [[match[1]!, match[2]!] as const] : []; + }), + ); + const emulator = props.get("ro.kernel.qemu") === "1" || props.get("ro.boot.qemu") === "1"; + const avdName = emulator + ? (yield* run("adb", ["-s", serial, "emu", "avd", "name"])).stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .find((line) => line && line !== "OK" && !line.startsWith("KO")) + : undefined; + const model = props.get("ro.product.model"); + const manufacturer = props.get("ro.product.manufacturer"); + const release = props.get("ro.build.version.release"); + return { + id: serial, + name: emulator + ? (avdName ?? serial) + : model + ? [manufacturer, model].filter(Boolean).join(" ") + : serial, + platform: "android", + version: release + ? `Android ${/^\d+$/.test(release) ? `${release}.0` : release}` + : "Android", + booted: true, + physical: !emulator, + } satisfies HostDevice; + }), + ); + }); + + const list: DeviceOperations["list"] = Effect.gen(function* () { + const listed = yield* fetchJson( + HttpClientRequest.get(`${origin}/api/devices`), + HubDeviceList, + ).pipe( + Effect.mapError( + (cause) => + new DeviceOperationError({ + operation: "list", + reason: "request_failed", + detail: briefCause(cause), + }), + ), + Effect.timeoutOption(Duration.seconds(LIST_TIMEOUT_SECONDS)), + ); + if (Option.isNone(listed)) { + return { + devices: [], + detail: `The device hub did not answer within ${LIST_TIMEOUT_SECONDS} seconds.`, + incompletePlatforms: yield* Effect.filter(["ios", "android"] as const, (platform) => + options.platformAvailable(platform), + ), + }; + } + const hubList = listed.value; + const devices: Array = [...hubList.simulators, ...hubList.emulators]; + const androidAvailable = yield* options.platformAvailable("android"); + const warnings = hubList.errors?.filter( + (error) => + // The host summary already explains a missing Android SDK; the hub's failed probes are noise. + !(!androidAvailable && error.message.startsWith("[android-utils] ")) && + // CoreSimulator can retain deleted devices; their optional usage timestamps are absent. + !( + error.message.startsWith("[apple-utils] Failed to read CoreSimulator plist at ") && + /\/CoreSimulator\/Devices\/[^/]+\/device\.plist:$/.test(error.message) && + /(?:^Error:\s*ENOENT:|\bcode:\s*['"]ENOENT['"])/.test(error.error ?? "") + ), + ); + const details = + warnings?.map((error) => { + const head = errorHead(error.error ?? ""); + return boundDiagnostics(head ? `${error.message} ${head}` : error.message); + }) ?? []; + if (androidAvailable) { + if (hubList.errors?.some((error) => error.message.startsWith(AVDMANAGER_LIST_FAILED))) { + const adbDevices = yield* listAdbDevices; + if (adbDevices === undefined) { + details.push("Could not list Android devices (adb devices -l)."); + } else { + devices.push(...adbDevices); + } + } + const avds = yield* run("emulator", ["-list-avds"]); + // Discovered devices stay listed; the AVDs that could not be listed are a warning. + if (avds.code !== 0) { + details.push( + `Could not list Android virtual devices (emulator -list-avds, exit code ${avds.code}). Discovered devices remain available.`, + ); + } + for (const name of (avds.code === 0 ? avds.stdout : "") + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean)) { + if (!devices.some((device) => device.platform === "android" && device.name === name)) { + devices.push({ + id: name, + name, + platform: "android", + version: "Android", + booted: false, + physical: false, + }); + } + } + } + const simulatorsUnlisted = + hubList.simulators.length === 0 && + (hubList.errors ?? []).some((error) => SIMULATOR_LIST_FAILED.test(error.message)); + return { + devices, + detail: details.join("\n") || undefined, + incompletePlatforms: simulatorsUnlisted ? (["ios"] as const) : [], + }; + }).pipe(Effect.withSpan("DeviceHubOperations.list")); + + const boot: DeviceOperations["boot"] = Effect.fn("DeviceHubOperations.boot")(function* (device) { + const result = yield* postJson( + "/api/devices/boot", + { platform: device.platform, id: device.id, name: device.name }, + "boot", + ).pipe(Effect.flatMap((request) => hubJson(request, HubActionResult, "boot", BOOT_TIMEOUT))); + if (!result.ok) { + // Keep the hub's launch command and nested failure available to agents for recovery. + const diagnostics = [ + result.error, + ...(result.errors ?? []).flatMap((error) => [error.message, error.error]), + ] + .filter((text) => text?.trim()) + .join("\n\n"); + return yield* new DeviceBootError({ + deviceId: device.id, + reason: /insufficient.*(?:disk|space)|not enough.*(?:disk|space)|no space left/i.test( + diagnostics, + ) + ? "disk_space" + : /timed? out|timeout/i.test(diagnostics) + ? "timeout" + : "launch_failed", + ...(diagnostics ? { diagnostics: boundDiagnostics(diagnostics) } : {}), + }); + } + if (device.platform === "ios") yield* attachIosStream(device.id, "boot"); + return result.serial ?? result.id ?? device.id; + }); + + const attach: DeviceOperations["attach"] = (device) => + device.platform === "ios" + ? attachIosStream(device.id, "open").pipe(Effect.asVoid) + : Effect.void; + + const shutdown: DeviceOperations["shutdown"] = Effect.fn("DeviceHubOperations.shutdown")( + function* (device) { + const postShutdown = (path: string, body: Record) => + postJson(path, body, "shutdown").pipe( + Effect.flatMap((request) => hubJson(request, HubActionResult, "shutdown")), + Effect.flatMap((result) => + result.ok + ? Effect.void + : Effect.fail( + new DeviceOperationError({ + operation: "shutdown", + reason: "hub_rejected", + ...(result.error ? { detail: boundDiagnostics(errorHead(result.error)) } : {}), + }), + ), + ), + ); + // serve-sim's shutdown closes its in-process capture session before it + // runs `simctl shutdown`; the hub's generic shutdown can leave that + // session cached across a reboot. serve-sim runs simctl bare, though, so + // a simulator that is already off fails there. + yield* device.platform === "ios" + ? postShutdown(`${vendorPrefix("ios")}/grid/api/shutdown`, { udid: device.id }).pipe( + Effect.catch((cause) => + list.pipe( + Effect.flatMap(({ devices }) => + devices.find((candidate) => candidate.id === device.id)?.booted === false + ? Effect.logInfo("iOS simulator was already shut down", { deviceId: device.id }) + : Effect.fail(cause), + ), + ), + ), + ) + : postShutdown("/api/devices/shutdown", { platform: device.platform, id: device.id }); + }, + ); + + const defaultDisplayScreenshot = Effect.fn("DeviceHubOperations.defaultDisplayScreenshot")( + function* (udid: string) { + const capture = yield* run("sh", ["-c", DEFAULT_DISPLAY_SCREENSHOT, "sh", udid], { + timeoutMs: Duration.toMillis(SCREENSHOT_TIMEOUT), + maxOutputBytes: 64 * 1024 * 1024, + }); + if (capture.code !== 0) { + return yield* new DeviceOperationError({ + operation: "screenshot", + reason: "command_failed", + exitCode: capture.code, + detail: boundDiagnostics(capture.stderr.trim()), + }); + } + return new Uint8Array(Buffer.from(capture.stdout, "base64")); + }, + ); + + const screenshot: DeviceOperations["screenshot"] = (device) => + httpClient + .execute( + HttpClientRequest.post( + `${origin}${vendorPrefix(device.platform)}/api/screenshot?device=${encodeURIComponent(device.id)}`, + ), + ) + .pipe( + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap((response) => response.arrayBuffer), + Effect.map((buffer) => new Uint8Array(buffer)), + Effect.scoped, + Effect.timeout(SCREENSHOT_TIMEOUT), + Effect.mapError( + (cause) => + new DeviceOperationError({ + operation: "screenshot", + reason: + HttpClientError.isHttpClientError(cause) && cause.reason._tag === "StatusCodeError" + ? "hub_rejected" + : "request_failed", + }), + ), + Effect.catchIf( + (error) => + device.platform === "ios" && + runsOnXcode26Runtime(device.version) && + error.reason === "hub_rejected", + () => defaultDisplayScreenshot(device.id), + ), + Effect.withSpan("DeviceHubOperations.screenshot"), + ); + + return { list, boot, attach, shutdown, screenshot }; +}; diff --git a/apps/server/src/device/DeviceService.test.ts b/apps/server/src/device/DeviceService.test.ts new file mode 100644 index 000000000..3335004a6 --- /dev/null +++ b/apps/server/src/device/DeviceService.test.ts @@ -0,0 +1,135 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { describe, expect, it } from "@effect/vitest"; +import { type DeviceSummary, ThreadId } from "@threadlines/contracts"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as Stream from "effect/Stream"; + +import { ServerConfig } from "../config.ts"; +import { ServerSettingsService } from "../serverSettings.ts"; +import { agentDeviceSession } from "./AgentDeviceTarget.ts"; +import * as DeviceHost from "./DeviceHost.ts"; +import * as DeviceService from "./DeviceService.ts"; + +const threadId = ThreadId.make("thread-devices"); +const unused = Effect.die("not used in this test"); + +/** A host whose one simulator boots when asked, recording the agent sessions it ends. */ +const fakeHost = (released: Array) => { + const device: { current: DeviceSummary } = { + current: { + id: "SIM-1", + platform: "ios", + name: "iPhone", + version: "iOS 27.2", + booted: false, + physical: false, + }, + }; + const ready: DeviceHost.DeviceHostReady = { + nodePath: process.execPath, + hub: { origin: "http://127.0.0.1:1", token: "token" }, + devices: { + list: Effect.sync(() => ({ + devices: [device.current], + detail: undefined, + incompletePlatforms: [], + })), + boot: (target) => + Effect.sync(() => { + device.current = { ...device.current, booted: true }; + return target.id; + }), + attach: () => Effect.void, + shutdown: () => unused, + screenshot: () => unused, + }, + run: () => Effect.succeed({ stdout: "", stderr: "", code: 0 }), + helpers: { serveSimAxSettings: null, serveSimCli: null }, + }; + return Layer.mock(DeviceHost.DeviceHost)({ + summary: Effect.succeed({ + platforms: [{ platform: "ios", available: true }], + hubInstalled: true, + agentDeviceInstalled: false, + tools: undefined, + toolInspectionError: undefined, + }), + platformAvailability: (platform) => Effect.succeed({ platform, available: true }), + ensureReady: () => Effect.succeed(ready), + current: Effect.succeed(ready), + releaseAgentSession: (session) => Effect.sync(() => void released.push(session)), + stop: Effect.void, + stopAgent: Effect.void, + }); +}; + +const settingsLayer = Layer.mock(ServerSettingsService)({ + getSettings: Effect.succeed({ enableDevices: true, agentDeviceAccess: false } as never), + streamChanges: Stream.never, +}); + +describe("DeviceService claims", () => { + it.live("records a session before it is open, reloads it after a restart, and ends it", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const base = yield* fs.makeTempDirectoryScoped({ prefix: "tl-device-claims-" }); + const released: Array = []; + const layer = Layer.mergeAll( + fakeHost(released), + settingsLayer, + ServerConfig.layerTest(process.cwd(), base), + ); + const claimsPath = path.join(base, "userdata", "device", "claims.json"); + + // First server: the user opens a simulator that was off. + const session = yield* Effect.scoped( + Effect.gen(function* () { + const service = yield* DeviceService.make; + return yield* service.open({ threadId, deviceId: "SIM-1", platform: "ios" }, "user"); + }), + ).pipe(Effect.provide(layer)); + expect(session).toMatchObject({ + deviceId: "SIM-1", + bootedByThreadlines: true, + openedBy: "user", + }); + expect(yield* fs.readFileString(claimsPath)).toContain("SIM-1"); + + // Next server: the claim is a session again, and releasing the thread ends it. + yield* Effect.scoped( + Effect.gen(function* () { + const service = yield* DeviceService.make; + expect((yield* service.state).sessions).toEqual([session]); + yield* service.releaseThread(threadId); + expect((yield* service.state).sessions).toEqual([]); + }), + ).pipe(Effect.provide(layer)); + expect(released).toEqual([agentDeviceSession(threadId, "SIM-1")]); + expect(yield* fs.readFileString(claimsPath)).not.toContain("SIM-1"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +}); + +describe("extractPng", () => { + it("finds one complete PNG and reads its size", () => { + const ihdr = Buffer.alloc(25); + ihdr.writeUInt32BE(13, 0); + ihdr.write("IHDR", 4, "latin1"); + ihdr.writeUInt32BE(390, 8); + ihdr.writeUInt32BE(844, 12); + const png = Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + ihdr, + Buffer.from([0, 0, 0, 0, 0x49, 0x45, 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82]), + ]); + const extracted = DeviceService.extractPng(Buffer.concat([Buffer.from("noise"), png])); + expect(extracted?.width).toBe(390); + expect(extracted?.height).toBe(844); + expect(extracted?.png.length).toBe(png.length); + expect(DeviceService.extractPng(png.subarray(0, 30))).toBeUndefined(); + }); +}); diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts new file mode 100644 index 000000000..8f4597335 --- /dev/null +++ b/apps/server/src/device/DeviceService.ts @@ -0,0 +1,791 @@ +/** + * Simulators and emulators on this server's machine (docs/design/devices.md). + * Ported from T3 Code's DeviceService with one host per server. + * + * Owns the state clients subscribe to (`DeviceServiceState`), the sessions + * threads hold on devices, and the agent's way in (`agentOpen`). Nothing + * installs or starts until a client or an agent asks, and only after the + * matching setting: the hub after "Use devices", agent-device after "Agents + * can use devices". + * + * Sessions survive a crash. `/device/claims.json` records each one + * before it is reported open and drops it on close; the next server loads + * them back, and DeviceThreadReactor releases those whose thread was put away + * while it was down. Closing never shuts a device down unless asked: a + * simulator the user booted is theirs. + */ +import { + type DeviceActionInput, + type DeviceCloseInput, + type DeviceDetail, + type DeviceDetailInput, + type DeviceError, + DeviceHostUnavailableError, + type DeviceId, + DeviceNotFoundError, + DeviceOperationError, + type DeviceOpenInput, + type DevicePlatform, + DevicePlatformUnavailableError, + type DeviceServiceState, + DeviceSession, + type DeviceShutdownInput, + type DeviceSummary, + type ThreadId, +} from "@threadlines/contracts"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as PubSub from "effect/PubSub"; +import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; +import * as SynchronizedRef from "effect/SynchronizedRef"; + +import { writeFileStringAtomically } from "../atomicWrite.ts"; +import { ServerConfig } from "../config.ts"; +import { ServerSettingsService } from "../serverSettings.ts"; +import { + agentDeviceLauncherKey, + bindAgentDeviceLauncher, + pruneAgentDeviceLaunchers, +} from "./AgentDeviceShim.ts"; +import { + agentDeviceConfigPath, + agentDeviceSession, + writeAgentDeviceConfig, +} from "./AgentDeviceTarget.ts"; +import { readDeviceDetail, runDeviceAction } from "./DeviceActions.ts"; +import * as DeviceHost from "./DeviceHost.ts"; +import { deviceStateDir } from "./DeviceToolchain.ts"; +import * as LocalDeviceHost from "./LocalDeviceHost.ts"; + +export interface DeviceScreenshot { + readonly device: DeviceSummary; + readonly png: Uint8Array; + readonly width: number; + readonly height: number; +} + +/** What `device_open` hands an agent: the device and the launcher bound to it. */ +export interface DeviceAgentOpened { + readonly device: DeviceSummary; + readonly session: DeviceSession; + /** Absolute path of the `agent-device` launcher for this thread and device. */ + readonly command: string; + readonly agentSession: string; +} + +/** A session as claims.json keeps it: with the agent-device session it may hold. */ +const DeviceClaim = Schema.Struct({ + ...DeviceSession.fields, + agentSession: Schema.String, +}); +type DeviceClaim = typeof DeviceClaim.Type; +const ClaimsFile = Schema.Struct({ version: Schema.Literal(1), claims: Schema.Array(DeviceClaim) }); +const decodeClaimsFile = Schema.decodeUnknownEffect(Schema.fromJsonString(ClaimsFile)); +const encodeClaimsFile = Schema.encodeUnknownEffect(Schema.fromJsonString(ClaimsFile)); + +export interface DeviceServiceShape { + readonly state: Effect.Effect; + /** The state now and after every change. */ + readonly stream: Stream.Stream; + /** The devices; with `refresh`, asks the hub again (starting it). */ + readonly list: (input: { + readonly refresh?: boolean | undefined; + }) => Effect.Effect; + readonly updateTool: (tool: "hub" | "agent") => Effect.Effect; + readonly open: ( + input: DeviceOpenInput, + openedBy?: "user" | "agent", + ) => Effect.Effect; + readonly close: (input: DeviceCloseInput) => Effect.Effect; + readonly shutdown: (input: DeviceShutdownInput) => Effect.Effect; + readonly detail: (input: DeviceDetailInput) => Effect.Effect; + readonly action: (input: DeviceActionInput) => Effect.Effect; + readonly screenshot: (deviceId: DeviceId) => Effect.Effect; + /** + * An agent opens a device: agent tools readied, the device opened for the + * thread (`openedBy: "agent"`), and a launcher bound to them. + */ + readonly agentOpen: (input: { + readonly threadId: ThreadId; + readonly deviceId: DeviceId; + readonly platform: DevicePlatform; + }) => Effect.Effect; + /** Every session a thread holds, oldest first. */ + readonly sessionsForThread: (threadId: ThreadId) => Effect.Effect>; + /** The running hub, for the device channel. Never starts it. */ + readonly currentHub: Effect.Effect; + /** A thread was archived, deleted or moved: its sessions end, devices keep running. */ + readonly releaseThread: (threadId: ThreadId) => Effect.Effect; +} + +export class DeviceService extends Context.Service()( + "@threadlines/server/device/DeviceService", +) {} + +const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]; +const PNG_END = Buffer.from([0x49, 0x45, 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82]); + +/** The one complete PNG in `bytes` (from its signature to its IEND chunk), with its size. */ +export const extractPng = ( + bytes: Uint8Array, +): { readonly png: Uint8Array; readonly width: number; readonly height: number } | undefined => { + const buffer = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const start = buffer.indexOf(Buffer.from(PNG_SIGNATURE)); + if (start < 0) return undefined; + const end = buffer.indexOf(PNG_END, start); + if (end < 0 || end - start < 24) return undefined; + const png = new Uint8Array(buffer.subarray(start, end + PNG_END.length)); + const view = new DataView(png.buffer, png.byteOffset, png.byteLength); + if (view.getUint32(12) !== 0x49484452) return undefined; + return { png, width: view.getUint32(16), height: view.getUint32(20) }; +}; + +const sameSession = ( + a: { readonly threadId: ThreadId; readonly deviceId: DeviceId }, + b: { readonly threadId: ThreadId; readonly deviceId: DeviceId }, +) => a.threadId === b.threadId && a.deviceId === b.deviceId; + +const agentTargetArgs = (device: DeviceSummary): ReadonlyArray => + device.platform === "ios" + ? ["--platform", "ios", "--udid", device.id] + : ["--platform", "android", "--serial", device.id]; + +const hostUnavailable = (error: DeviceHost.DeviceHostError) => + new DeviceHostUnavailableError({ reason: error.message }); + +export const make = Effect.gen(function* () { + const host = yield* DeviceHost.DeviceHost; + const settings = yield* ServerSettingsService; + const config = yield* ServerConfig; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const scope = yield* Scope.Scope; + const stateDir = config.stateDir; + const claimsPath = path.join(deviceStateDir(path, stateDir), "claims.json"); + const services = (effect: Effect.Effect) => + effect.pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + ); + + const readDeviceSettings = settings.getSettings.pipe( + Effect.map((value) => ({ + enabled: value.enableDevices, + agentAccessEnabled: value.agentDeviceAccess, + })), + Effect.mapError( + () => new DeviceOperationError({ operation: "settings", reason: "settings_failed" }), + ), + ); + const initialSettings = yield* readDeviceSettings.pipe( + Effect.orElseSucceed(() => ({ enabled: false, agentAccessEnabled: false })), + ); + + // Claims a previous run left. A file that can't be read is kept as it is and treated as none. + const loadedClaims = yield* fs.readFileString(claimsPath).pipe( + Effect.flatMap(decodeClaimsFile), + Effect.map((file) => file.claims), + Effect.catch(() => Effect.succeed([] as ReadonlyArray)), + ); + const claims = new Map( + loadedClaims.map((claim) => [claim.agentSession, claim]), + ); + const claimsLock = yield* Semaphore.make(1); + const persistClaims = claimsLock.withPermit( + Effect.suspend(() => encodeClaimsFile({ version: 1, claims: [...claims.values()] })).pipe( + Effect.flatMap((contents) => writeFileStringAtomically({ filePath: claimsPath, contents })), + services, + Effect.mapError( + () => new DeviceOperationError({ operation: "save claim", reason: "settings_failed" }), + ), + ), + ); + + const toSession = ({ agentSession: _agentSession, ...session }: DeviceClaim): DeviceSession => + session; + + const initialSummary = yield* host.summary; + const statePubSub = yield* PubSub.unbounded(); + const stateRef = yield* SynchronizedRef.make({ + enabled: initialSettings.enabled, + agentAccessEnabled: initialSettings.agentAccessEnabled, + hostStatus: initialSettings.enabled ? "idle" : "disabled", + platforms: initialSummary.platforms, + incompletePlatforms: [], + ...(initialSummary.tools ? { tools: initialSummary.tools } : {}), + ...(initialSummary.toolInspectionError + ? { toolInspectionError: initialSummary.toolInspectionError } + : {}), + hubInstalled: initialSummary.hubInstalled, + agentDeviceInstalled: initialSummary.agentDeviceInstalled, + devices: [], + sessions: [...claims.values()].map(toSession), + bootingDevices: [], + revision: 0, + }); + const lifecycleLock = yield* Semaphore.make(1); + const launcherLock = yield* Semaphore.make(1); + + const publish = (update: (state: DeviceServiceState) => DeviceServiceState) => + SynchronizedRef.modifyEffect(stateRef, (state) => { + const next = { ...update(state), revision: state.revision + 1 }; + return PubSub.publish(statePubSub, next).pipe(Effect.as([next, next] as const)); + }).pipe(Effect.uninterruptible); + + const currentState = SynchronizedRef.get(stateRef); + + const publishSummary = host.summary.pipe( + Effect.flatMap((summary) => + publish((state) => { + const { tools: _tools, toolInspectionError: _error, ...rest } = state; + return { + ...rest, + platforms: summary.platforms, + hubInstalled: summary.hubInstalled, + agentDeviceInstalled: summary.agentDeviceInstalled, + ...(summary.tools ? { tools: summary.tools } : {}), + ...(summary.toolInspectionError + ? { toolInspectionError: summary.toolInspectionError } + : {}), + }; + }), + ), + ); + + const setHostStatus = (status: DeviceServiceState["hostStatus"], detail?: string) => + publish((state) => { + const { hostStatusDetail: _detail, ...rest } = state; + return { ...rest, hostStatus: status, ...(detail ? { hostStatusDetail: detail } : {}) }; + }); + + const requireEnabled = Effect.gen(function* () { + if (!(yield* readDeviceSettings).enabled) { + return yield* new DeviceHostUnavailableError({ + reason: "Devices are off. Turn on Use devices in Settings → Devices first.", + }); + } + }); + + const readiness = Effect.gen(function* () { + yield* requireEnabled; + const ready = yield* host + .ensureReady((phase, detail) => setHostStatus(phase, detail).pipe(Effect.asVoid)) + .pipe( + Effect.tapError((error) => setHostStatus("failed", error.message)), + Effect.mapError(hostUnavailable), + ); + const { hostStatus } = yield* currentState; + if (hostStatus !== "ready") { + yield* setHostStatus("ready"); + yield* publishSummary; + } + return ready; + }).pipe(lifecycleLock.withPermit, Effect.withSpan("DeviceService.readiness")); + + const agentReadiness = Effect.gen(function* () { + const deviceSettings = yield* readDeviceSettings; + if (!deviceSettings.enabled || !deviceSettings.agentAccessEnabled) { + return yield* new DeviceHostUnavailableError({ + reason: + "Agents can't use devices on this computer. Ask the user to turn on Use devices and Agents can use devices in Settings → Devices.", + }); + } + const ready = yield* host + .ensureAgentReady((phase, detail) => setHostStatus(phase, detail).pipe(Effect.asVoid)) + .pipe( + Effect.tapError((error) => setHostStatus("failed", error.message)), + Effect.mapError(hostUnavailable), + ); + yield* setHostStatus("ready"); + yield* publishSummary; + return ready; + }).pipe(lifecycleLock.withPermit, Effect.withSpan("DeviceService.agentReadiness")); + + const refresh = Effect.fn("DeviceService.refresh")(function* (ready: DeviceHost.DeviceHostReady) { + const { devices, detail, incompletePlatforms } = yield* ready.devices.list; + if (!(yield* readDeviceSettings).enabled) return yield* currentState; + return yield* publish((state) => { + const { hostStatusDetail: _detail, ...rest } = state; + return { + ...rest, + ...(detail ? { hostStatusDetail: detail } : {}), + incompletePlatforms: [...incompletePlatforms], + // A platform that could not be listed keeps its last known devices. + devices: [ + ...state.devices.filter((device) => incompletePlatforms.includes(device.platform)), + ...devices.filter((device) => !incompletePlatforms.includes(device.platform)), + ], + }; + }); + }); + + const list: DeviceServiceShape["list"] = (input) => + Effect.gen(function* () { + if (!(yield* readDeviceSettings).enabled) return yield* currentState; + if (input.refresh !== true) return yield* currentState; + const platforms = (yield* host.summary).platforms; + if (!platforms.some((platform) => platform.available)) { + yield* publishSummary; + return yield* currentState; + } + const ready = yield* readiness; + return yield* refresh(ready); + }).pipe(Effect.withSpan("DeviceService.list")); + + const ensurePlatform = Effect.fn("DeviceService.ensurePlatform")(function* ( + platform: DevicePlatform, + ) { + const availability = yield* host.platformAvailability(platform); + if (!availability.available) { + return yield* new DevicePlatformUnavailableError({ + platform, + reason: availability.reason ?? "This platform's tools are missing.", + }); + } + }); + + const findDevice = (state: DeviceServiceState, deviceId: string) => + state.devices.find((device) => device.id === deviceId); + + /** serve-emu encodes emulator video with ffmpeg's libx264; without it there is nothing to show. */ + const requireAndroidVideo = Effect.fn("DeviceService.requireAndroidVideo")(function* ( + ready: DeviceHost.DeviceHostReady, + ) { + const encoders = yield* ready.run(ready.nodePath, [ + "-e", + `const { spawnSync } = require("node:child_process"); +const result = spawnSync(process.env.SERVE_EMU_FFMPEG?.trim() || "ffmpeg", ["-hide_banner", "-encoders"], { + stdio: "inherit", timeout: 10000, killSignal: "SIGKILL", +}); +process.exitCode = result.error ? 1 : (result.status ?? 1);`, + ]); + if (encoders.code !== 0) { + return yield* new DevicePlatformUnavailableError({ + platform: "android", + reason: + "Android emulator video needs ffmpeg with the libx264 encoder. Install it on this computer's PATH (or set SERVE_EMU_FFMPEG), restart Threadlines, then open the emulator again.", + }); + } + if (!/^\s*V\S*[ \t]+libx264(?:[ \t]|$)/m.test(`${encoders.stdout}\n${encoders.stderr}`)) { + return yield* new DevicePlatformUnavailableError({ + platform: "android", + reason: + "Android emulator video needs the libx264 encoder. Install an ffmpeg build that includes libx264, restart Threadlines, then open the emulator again.", + }); + } + }); + + /** Records the claim, then publishes the session: a claim is on disk before anyone sees it open. */ + const recordSession = (session: DeviceSession) => + Effect.gen(function* () { + if (!(yield* readDeviceSettings).enabled) { + return yield* new DeviceHostUnavailableError({ + reason: "Devices were turned off while the device was opening.", + }); + } + const agentSession = agentDeviceSession(session.threadId, session.deviceId); + claims.set(agentSession, { ...session, agentSession }); + yield* persistClaims; + yield* publish((state) => ({ + ...state, + sessions: [ + ...state.sessions.filter((existing) => !sameSession(existing, session)), + session, + ], + })); + }).pipe(lifecycleLock.withPermit); + + const open: DeviceServiceShape["open"] = (input, openedBy = "user") => + Effect.gen(function* () { + yield* ensurePlatform(input.platform); + const ready = yield* readiness; + let state = yield* refresh(ready); + let device = findDevice(state, input.deviceId); + if (!device) return yield* new DeviceNotFoundError({ deviceId: input.deviceId }); + if (device.platform === "android" && !device.physical) yield* requireAndroidVideo(ready); + let bootedNow = false; + if (!device.booted && input.boot !== false) { + const booting = { ...device, threadId: input.threadId }; + yield* publish((current) => ({ + ...current, + bootingDevices: [ + ...current.bootingDevices.filter((entry) => entry.id !== booting.id), + booting, + ], + })); + const bootedId = yield* ready.devices.boot(device).pipe( + Effect.ensuring( + publish((current) => ({ + ...current, + bootingDevices: current.bootingDevices.filter((entry) => entry.id !== booting.id), + })), + ), + ); + bootedNow = true; + state = yield* refresh(ready); + // Retained entries predate the boot, so only a complete listing confirms the booted device. + device = state.incompletePlatforms.includes(device.platform) + ? undefined + : (findDevice(state, bootedId) ?? findDevice(state, device.id)); + if (!device) return yield* new DeviceNotFoundError({ deviceId: input.deviceId }); + } else if (device.booted) { + yield* ready.devices.attach(device); + } + const opened = device; + const earlier = (yield* currentState).sessions.find( + (session) => session.deviceId === opened.id && session.bootedByThreadlines, + ); + const session: DeviceSession = { + threadId: input.threadId, + deviceId: opened.id, + platform: opened.platform, + openedAt: DateTime.formatIso(yield* DateTime.now), + bootedByThreadlines: bootedNow || earlier !== undefined, + openedBy, + }; + yield* recordSession(session); + return session; + }).pipe(Effect.withSpan("DeviceService.open")); + + /** Removes launchers whose session is gone. Under the launcher lock, so binding never races it. */ + const pruneLaunchers = launcherLock.withPermit( + Effect.gen(function* () { + const { sessions } = yield* currentState; + const live = new Set( + sessions.map((session) => agentDeviceLauncherKey(session.threadId, session.deviceId)), + ); + yield* pruneAgentDeviceLaunchers({ stateDir, live }).pipe(services); + }), + ); + + /** + * Ends sessions: their agent-device sessions first (which quits the app the + * agent opened and frees the device for others), then the claims, then the + * state and launchers. + */ + const endSessions = (ending: ReadonlyArray) => + Effect.gen(function* () { + if (ending.length === 0) return; + yield* Effect.forEach( + ending, + (session) => + host.releaseAgentSession(agentDeviceSession(session.threadId, session.deviceId)), + { concurrency: 4, discard: true }, + ); + for (const session of ending) { + claims.delete(agentDeviceSession(session.threadId, session.deviceId)); + } + yield* persistClaims.pipe( + Effect.catch((error) => Effect.logWarning("Could not save device claims", { error })), + ); + yield* publish((state) => ({ + ...state, + sessions: state.sessions.filter( + (session) => !ending.some((ended) => sameSession(ended, session)), + ), + })); + yield* pruneLaunchers; + }); + + const shutdownDevice = Effect.fn("DeviceService.shutdownDevice")(function* ( + ready: DeviceHost.DeviceHostReady, + deviceId: DeviceId, + platform: DevicePlatform, + ) { + yield* ready.devices.shutdown({ id: deviceId, platform }); + yield* publish((state) => ({ + ...state, + devices: state.devices.map((device) => + device.id === deviceId ? { ...device, booted: false } : device, + ), + })); + }); + + const refreshQuietly = (ready: DeviceHost.DeviceHostReady) => + refresh(ready).pipe( + Effect.catch((cause) => + Effect.logWarning("Device listing unavailable after shutdown", { cause }), + ), + ); + + const close: DeviceServiceShape["close"] = (input) => + Effect.gen(function* () { + const { sessions } = yield* currentState; + const closing = sessions.filter( + (session) => + session.threadId === input.threadId && + (input.deviceId === undefined || session.deviceId === input.deviceId), + ); + if (!input.shutdown) { + yield* lifecycleLock.withPermit(endSessions(closing)); + return; + } + // Powering off ends every thread's session on those devices, before the power-off, + // while their agent-device sessions can still quit the app. + const ending = sessions.filter((session) => + closing.some((closed) => closed.deviceId === session.deviceId), + ); + yield* lifecycleLock.withPermit(endSessions(ending)); + const ready = yield* readiness; + for (const session of closing) { + yield* shutdownDevice(ready, session.deviceId, session.platform); + } + if (closing.length > 0) yield* refreshQuietly(ready); + }).pipe(Effect.withSpan("DeviceService.close")); + + const shutdown: DeviceServiceShape["shutdown"] = (input) => + Effect.gen(function* () { + const { sessions } = yield* currentState; + yield* lifecycleLock.withPermit( + endSessions(sessions.filter((session) => session.deviceId === input.deviceId)), + ); + const ready = yield* readiness; + yield* shutdownDevice(ready, input.deviceId, input.platform); + yield* refreshQuietly(ready); + }).pipe(Effect.withSpan("DeviceService.shutdown")); + + const resolveDevice = Effect.fn("DeviceService.resolveDevice")(function* (deviceId: DeviceId) { + const ready = yield* readiness; + const device = + findDevice(yield* currentState, deviceId) ?? findDevice(yield* refresh(ready), deviceId); + if (!device) return yield* new DeviceNotFoundError({ deviceId }); + return { ready, device }; + }); + + const detail: DeviceServiceShape["detail"] = (input) => + Effect.gen(function* () { + const { ready, device } = yield* resolveDevice(input.deviceId); + const read = yield* readDeviceDetail(ready, device.platform, device.id); + return { + deviceId: device.id, + settings: read.settings, + foregroundApp: read.foregroundApp, + readAt: DateTime.formatIso(yield* DateTime.now), + }; + }).pipe(Effect.withSpan("DeviceService.detail")); + + const action: DeviceServiceShape["action"] = (input) => + Effect.gen(function* () { + const { ready, device } = yield* resolveDevice(input.deviceId); + yield* runDeviceAction(ready, device.platform, input); + return yield* detail({ deviceId: device.id }); + }).pipe(Effect.withSpan("DeviceService.action")); + + const screenshot: DeviceServiceShape["screenshot"] = (deviceId) => + Effect.gen(function* () { + const { ready, device } = yield* resolveDevice(deviceId); + const bytes = yield* ready.devices.screenshot(device); + const extracted = extractPng(bytes); + if (extracted === undefined) { + return yield* new DeviceOperationError({ + operation: "screenshot", + reason: "hub_rejected", + detail: `The device hub returned ${bytes.length} bytes without a complete PNG.`, + }); + } + return { device, ...extracted }; + }).pipe(Effect.withSpan("DeviceService.screenshot")); + + const agentOpen: DeviceServiceShape["agentOpen"] = (input) => + Effect.gen(function* () { + // Consent and agent tools first, before anything boots or a session is recorded. + const agentReady = yield* agentReadiness; + const session = yield* open(input, "agent"); + const device = findDevice(yield* currentState, session.deviceId); + if (!device) return yield* new DeviceNotFoundError({ deviceId: session.deviceId }); + const configPath = agentDeviceConfigPath(path, stateDir); + yield* writeAgentDeviceConfig(configPath, agentReady.agentDevice).pipe( + services, + Effect.mapError( + () => + new DeviceOperationError({ operation: "configure agent", reason: "settings_failed" }), + ), + ); + const agentSession = agentDeviceSession(session.threadId, session.deviceId); + const targetArgs = [ + ...agentTargetArgs(device), + "--config", + configPath, + "--session", + agentSession, + ]; + const command = yield* launcherLock.withPermit( + Effect.gen(function* () { + const { sessions } = yield* currentState; + // Closed while it opened: no launcher for a session that is gone. + if (!sessions.some((live) => sameSession(live, session))) return Option.none(); + return Option.some( + yield* bindAgentDeviceLauncher({ + stateDir, + threadId: session.threadId, + deviceId: session.deviceId, + node: agentReady.nodePath, + entryPath: agentReady.agentDevice.entryPath, + targetArgs, + }).pipe( + services, + Effect.mapError( + () => + new DeviceOperationError({ + operation: "prepare agent-device", + reason: "settings_failed", + }), + ), + ), + ); + }), + ); + if (Option.isNone(command)) { + return yield* new DeviceHostUnavailableError({ + reason: + "The device was closed while it was opening. Call device_open again only if you still need it.", + }); + } + return { device, session, command: command.value, agentSession }; + }).pipe(Effect.withSpan("DeviceService.agentOpen")); + + const sessionsForThread: DeviceServiceShape["sessionsForThread"] = (threadId) => + currentState.pipe( + Effect.map((state) => state.sessions.filter((session) => session.threadId === threadId)), + ); + + const releaseThread: DeviceServiceShape["releaseThread"] = (threadId) => + sessionsForThread(threadId).pipe( + Effect.flatMap((sessions) => lifecycleLock.withPermit(endSessions(sessions))), + Effect.catchCause((cause) => + Effect.logWarning("Could not release a thread's devices", { threadId, cause }), + ), + ); + + const updateTool: DeviceServiceShape["updateTool"] = (tool) => + Effect.gen(function* () { + yield* requireEnabled; + const deviceSettings = yield* readDeviceSettings; + if (tool === "agent" && !deviceSettings.agentAccessEnabled) { + return yield* new DeviceHostUnavailableError({ + reason: "Agent tools install only once Agents can use devices is on.", + }); + } + yield* host.updateTool(tool).pipe( + Effect.mapError( + (error) => + new DeviceOperationError({ + operation: "update device tool", + reason: "command_failed", + detail: error.message, + }), + ), + ); + return yield* publishSummary; + }).pipe(lifecycleLock.withPermit, Effect.withSpan("DeviceService.updateTool")); + + /** "Use devices" off stops the hub and daemon (devices keep running); agent access off stops the daemon. */ + const applySettings = (next: { + readonly enabled: boolean; + readonly agentAccessEnabled: boolean; + }) => + Effect.gen(function* () { + const previous = yield* currentState; + if ( + previous.enabled === next.enabled && + previous.agentAccessEnabled === next.agentAccessEnabled + ) { + return; + } + if (!next.enabled) { + yield* lifecycleLock.withPermit( + Effect.gen(function* () { + yield* endSessions(previous.sessions); + yield* host.stop; + }), + ); + } else if (!next.agentAccessEnabled && previous.agentAccessEnabled) { + yield* host.stopAgent; + } + yield* publish((state) => { + const { hostStatusDetail: _detail, ...rest } = state; + return { + ...rest, + enabled: next.enabled, + agentAccessEnabled: next.agentAccessEnabled, + hostStatus: next.enabled + ? state.hostStatus === "disabled" + ? "idle" + : state.hostStatus + : "disabled", + ...(next.enabled && state.hostStatusDetail + ? { hostStatusDetail: state.hostStatusDetail } + : {}), + devices: next.enabled ? state.devices : [], + sessions: next.enabled ? state.sessions : [], + bootingDevices: next.enabled ? state.bootingDevices : [], + }; + }); + // Agent access turned on: fetch agent-device now, in the background, so the + // first device_open is quick. Nothing starts until an agent asks. + if (next.enabled && next.agentAccessEnabled && !previous.agentAccessEnabled) { + yield* host.updateTool("agent").pipe( + Effect.andThen(publishSummary), + Effect.catch((error) => + Effect.logWarning("Could not install agent tools", { detail: error.message }), + ), + Effect.forkIn(scope), + ); + } + }); + + yield* settings.streamChanges.pipe( + Stream.map((value) => ({ + enabled: value.enableDevices, + agentAccessEnabled: value.agentDeviceAccess, + })), + Stream.runForEach(applySettings), + Effect.catchCause((cause) => Effect.logWarning("Device settings watcher stopped", { cause })), + Effect.forkIn(scope), + ); + + // Launchers for sessions that no longer exist (the claims file is the record). + yield* pruneLaunchers.pipe(Effect.ignore); + + // Stop the hub and daemon with the server. Devices keep running. + yield* Effect.addFinalizer(() => host.stop); + + return DeviceService.of({ + state: currentState, + stream: Stream.unwrap( + Effect.gen(function* () { + // Subscribe before reading so no change is lost. + const subscription = yield* PubSub.subscribe(statePubSub); + const initial = yield* currentState; + return Stream.concat( + Stream.make(initial), + Stream.fromSubscription(subscription).pipe( + Stream.filter((state) => state.revision > initial.revision), + ), + ); + }), + ), + list, + updateTool, + open, + close, + shutdown, + detail, + action, + screenshot, + agentOpen, + sessionsForThread, + currentHub: host.current.pipe(Effect.map((ready) => ready?.hub ?? null)), + releaseThread, + }); +}); + +export const layer = Layer.effect(DeviceService, make).pipe(Layer.provide(LocalDeviceHost.layer)); diff --git a/apps/server/src/device/DeviceThreadReactor.ts b/apps/server/src/device/DeviceThreadReactor.ts new file mode 100644 index 000000000..38accba03 --- /dev/null +++ b/apps/server/src/device/DeviceThreadReactor.ts @@ -0,0 +1,62 @@ +/** + * A thread put away lets go of its devices (docs/design/devices.md): archived, + * deleted, or moved to another computer, its device sessions end (and with + * them their agent-device sessions and launchers). The devices keep running. + * + * At startup, sessions reloaded from claims.json whose thread was put away + * while the server was down end the same way, like the secret requests' + * startup sweep (secrets/AgentSecretRequests.ts). + */ +import type { OrchestrationThread, ThreadId } from "@threadlines/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Stream from "effect/Stream"; + +import { OrchestrationEngineService } from "../orchestration/Services/OrchestrationEngine.ts"; +import { ProjectionSnapshotQuery } from "../orchestration/Services/ProjectionSnapshotQuery.ts"; +import { DeviceService } from "./DeviceService.ts"; + +const isPutAway = (thread: OrchestrationThread | undefined) => + thread === undefined || + thread.deletedAt !== null || + thread.archivedAt !== null || + (thread.move ?? null) !== null; + +export const layer = Layer.effectDiscard( + Effect.gen(function* () { + const devices = yield* DeviceService; + const engine = yield* OrchestrationEngineService; + const snapshots = yield* ProjectionSnapshotQuery; + + yield* Stream.runForEach(engine.streamDomainEvents, (event) => { + switch (event.type) { + case "thread.archived": + case "thread.move-started": + case "thread.deleted": + return devices.releaseThread(event.payload.threadId); + default: + return Effect.void; + } + }).pipe( + Effect.catchCause((cause) => Effect.logWarning("device thread cleanup stopped", { cause })), + Effect.forkScoped, + ); + + yield* Effect.gen(function* () { + const { sessions } = yield* devices.state; + const threads = new Set(sessions.map((session) => session.threadId)); + for (const threadId of threads) { + const read = yield* snapshots + .getThreadDetailById(threadId, { includeArchived: true }) + .pipe(Effect.map(Option.getOrUndefined), Effect.result); + // A thread whose read failed keeps its devices: never released blind. + if (read._tag === "Failure") continue; + if (isPutAway(read.success)) yield* devices.releaseThread(threadId); + } + }).pipe( + Effect.catchCause((cause) => Effect.logWarning("device claim recovery failed", { cause })), + Effect.forkScoped, + ); + }), +); diff --git a/apps/server/src/device/DeviceToolchain.ts b/apps/server/src/device/DeviceToolchain.ts new file mode 100644 index 000000000..02275af34 --- /dev/null +++ b/apps/server/src/device/DeviceToolchain.ts @@ -0,0 +1,398 @@ +// @effect-diagnostics nodeBuiltinImport:off - a synchronous check for community agents' Node.js cleanup +/** + * The two device tools (docs/design/devices.md): expo-device-hub streams and + * takes input, agent-device is the command line agents drive devices with. + * + * Each is installed from npm at the version pinned in deviceToolVersions.json + * into `/tools/device//`: npm installs into a staging + * folder beside it, the entry point is checked, a `.install-complete` marker + * is written, and only then is the folder moved into place. Install scripts + * run: the hub's native addons need them. npm and the tools run on + * Threadlines' managed Node.js, never `npx` and never the server's own + * runtime (which is Electron in the desktop app). Both tools load only N-API + * addons, so a later Node.js release runs an earlier install. + * + * Nothing here installs on its own: the hub only after "Use devices", the + * agent tool only after "Agents can use devices" (DeviceService decides). + */ +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; + +import type { DeviceToolVersions } from "@threadlines/contracts"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; + +import { npmPinnedFlags } from "../provider/acpRegistry/AcpRegistryInstaller.ts"; +import { acpRegistryNodes } from "../provider/acpRegistry/AcpRegistryNodes.ts"; +import { managedNodeEnvironment, runNode } from "../provider/acpRegistry/NpmPackage.ts"; +import { + type ManagedNodePaths, + managedNodeReleaseFor, +} from "../provider/managedRuntime/ManagedNode.ts"; +import DEVICE_TOOL_VERSIONS from "./deviceToolVersions.json" with { type: "json" }; + +const DEVICE_HUB_PACKAGE = "expo-device-hub"; +const AGENT_DEVICE_PACKAGE = "agent-device"; +export const DEVICE_HUB_VERSION = DEVICE_TOOL_VERSIONS[DEVICE_HUB_PACKAGE]; +export const AGENT_DEVICE_VERSION = DEVICE_TOOL_VERSIONS[AGENT_DEVICE_PACKAGE]; + +const INSTALL_TIMEOUT_MS = 10 * 60 * 1000; +const VERSION_PATTERN = /^[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.-]+)?$/; +/** One install at a time in this process: two would race for the same staging parent. */ +const installLock = Semaphore.makeUnsafe(1); + +export type DeviceToolKey = "hub" | "agent"; + +interface ToolSpec { + readonly key: DeviceToolKey; + readonly name: string; + readonly version: string; + /** The entry point inside the package, run as `node `. */ + readonly entry: ReadonlyArray; +} + +const SPECS: Record = { + hub: { + key: "hub", + name: DEVICE_HUB_PACKAGE, + version: DEVICE_HUB_VERSION, + entry: ["dist", "server", "cli.mjs"], + }, + agent: { + key: "agent", + name: AGENT_DEVICE_PACKAGE, + version: AGENT_DEVICE_VERSION, + entry: ["bin", "agent-device.mjs"], + }, +}; + +export interface DeviceToolPaths { + readonly installDir: string; + readonly entryPath: string; + readonly sentinelPath: string; +} + +export class DeviceToolchainInstallError extends Schema.TaggedError()( + "DeviceToolchainInstallError", + { + tool: Schema.String, + step: Schema.String, + exitCode: Schema.optional(Schema.Number), + reason: Schema.optional(Schema.Literals(["no_node", "timed_out"])), + /** npm's last lines. Kept in logs only: they can carry registry credentials. */ + tail: Schema.optional(Schema.String), + cause: Schema.optional(Schema.Defect()), + }, +) { + override get message(): string { + if (this.reason === "no_node") { + return "Device support needs Node.js, and Threadlines has no Node.js it can install on this computer."; + } + if (this.reason === "timed_out") + return `Installing ${this.tool} took too long and was stopped.`; + const suffix = this.exitCode === undefined ? "" : ` (exit code ${this.exitCode})`; + return `Installing ${this.tool} failed while ${this.step}${suffix}.`; + } +} + +/** `/tools/device`. */ +export const deviceToolsRoot = (path: Path.Path, stateDir: string) => + path.join(stateDir, "tools", "device"); + +/** `/device`: hub.json, claims.json, agent-device's state, launchers. */ +export const deviceStateDir = (path: Path.Path, stateDir: string) => path.join(stateDir, "device"); + +/** agent-device's own state folder, isolated from the user's `~/.agent-device`. */ +export const agentDeviceStateDir = (path: Path.Path, stateDir: string): string => + path.join(deviceStateDir(path, stateDir), "agent-device"); + +const toolPaths = (path: Path.Path, stateDir: string, spec: ToolSpec): DeviceToolPaths => { + const installDir = path.join(deviceToolsRoot(path, stateDir), spec.name, spec.version); + return { + installDir, + entryPath: path.join(installDir, "node_modules", spec.name, ...spec.entry), + sentinelPath: path.join(installDir, ".install-complete"), + }; +}; + +export const deviceToolPaths = (path: Path.Path, stateDir: string, tool: DeviceToolKey) => + toolPaths(path, stateDir, SPECS[tool]); + +const isInstalledAt = Effect.fn("DeviceToolchain.isInstalledAt")(function* ( + paths: DeviceToolPaths, + version: string, +) { + const fs = yield* FileSystem.FileSystem; + const [entryExists, sentinel] = yield* Effect.all([ + fs.exists(paths.entryPath).pipe(Effect.orElseSucceed(() => false)), + fs.readFileString(paths.sentinelPath).pipe(Effect.orElseSucceed(() => "")), + ]); + return entryExists && sentinel.trim() === version; +}); + +export const isDeviceToolInstalled = Effect.fn("DeviceToolchain.isInstalled")(function* ( + stateDir: string, + tool: DeviceToolKey, +) { + const path = yield* Path.Path; + return yield* isInstalledAt(deviceToolPaths(path, stateDir, tool), SPECS[tool].version); +}); + +/** Is `tool` installed under `stateDir`, without Effect: for community agents' Node.js cleanup. */ +const installedSync = (stateDir: string, tool: DeviceToolKey) => { + const spec = SPECS[tool]; + const installDir = NodePath.join(stateDir, "tools", "device", spec.name, spec.version); + try { + return ( + NodeFS.readFileSync(NodePath.join(installDir, ".install-complete"), "utf8").trim() === + spec.version + ); + } catch { + return false; + } +}; + +/** Threadlines' managed Node.js under `/tools/node`, shared with community agents. */ +const nodesFor = (path: Path.Path, stateDir: string) => { + const nodes = acpRegistryNodes(path.join(stateDir, "tools", "node")); + if (!registeredKeepers.has(stateDir)) { + registeredKeepers.add(stateDir); + // Community agents' cleanup keeps the release device tools run on while one is installed. + nodes.keepAlso( + Effect.sync(() => { + const release = managedNodeReleaseFor(process.platform, process.arch); + return release && (installedSync(stateDir, "hub") || installedSync(stateDir, "agent")) + ? [release] + : []; + }), + ); + } + return nodes; +}; +const registeredKeepers = new Set(); + +/** + * The managed Node.js, installed if missing and leased until `scope` closes, + * so community agents' cleanup leaves it while a device tool runs on it. + */ +export const leaseDeviceNode = Effect.fn("DeviceToolchain.leaseDeviceNode")(function* ( + stateDir: string, +) { + const path = yield* Path.Path; + const release = managedNodeReleaseFor(process.platform, process.arch); + if (!release) { + return yield* new DeviceToolchainInstallError({ + tool: "Node.js", + step: "finding Node.js", + reason: "no_node", + }); + } + const nodes = nodesFor(path, stateDir); + const node = nodes.nodeFor(release); + const installed = yield* node.installed; + if (!installed) { + yield* nodes + .whileInstalling(node.install()) + .pipe( + Effect.mapError( + (cause) => + new DeviceToolchainInstallError({ tool: "Node.js", step: "installing Node.js", cause }), + ), + ); + } + return yield* node.acquire.pipe( + Effect.mapError( + (cause) => + new DeviceToolchainInstallError({ tool: "Node.js", step: "starting Node.js", cause }), + ), + ); +}); + +const installTool = Effect.fn("DeviceToolchain.installTool")(function* ( + spec: ToolSpec, + paths: DeviceToolPaths, + node: ManagedNodePaths, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const fail = (step: string) => (cause: unknown) => + new DeviceToolchainInstallError({ tool: spec.name, step, cause }); + + if (yield* isInstalledAt(paths, spec.version)) return paths; + + const parentDir = path.dirname(paths.installDir); + yield* fs + .remove(paths.installDir, { recursive: true, force: true }) + .pipe(Effect.mapError(fail("removing an incomplete install"))); + yield* fs + .makeDirectory(parentDir, { recursive: true }) + .pipe(Effect.mapError(fail("preparing the install folder"))); + const stagingDir = yield* fs + .makeTempDirectory({ directory: parentDir, prefix: ".staging-" }) + .pipe(Effect.mapError(fail("preparing the install folder"))); + + return yield* Effect.gen(function* () { + const result = yield* Effect.tryPromise({ + try: (signal) => + runNode({ + node: node.node, + args: [ + node.npmCli, + "install", + "--prefix", + stagingDir, + ...npmPinnedFlags(false), + `${spec.name}@${spec.version}`, + ], + cwd: stagingDir, + env: managedNodeEnvironment(process.env, node.binDir, process.platform), + signal, + timeoutMs: INSTALL_TIMEOUT_MS, + }), + catch: fail("running npm install"), + }); + if (result.timedOut) { + return yield* new DeviceToolchainInstallError({ + tool: spec.name, + step: "running npm install", + reason: "timed_out", + }); + } + if (result.exitCode !== 0) { + return yield* new DeviceToolchainInstallError({ + tool: spec.name, + step: "running npm install", + ...(result.exitCode === null ? {} : { exitCode: result.exitCode }), + tail: result.tail, + }); + } + const stagedEntry = path.join(stagingDir, "node_modules", spec.name, ...spec.entry); + if (!(yield* fs.exists(stagedEntry).pipe(Effect.orElseSucceed(() => false)))) { + return yield* new DeviceToolchainInstallError({ + tool: spec.name, + step: "checking the installed entry point", + }); + } + yield* fs + .writeFileString(path.join(stagingDir, ".install-complete"), `${spec.version}\n`) + .pipe(Effect.mapError(fail("recording the finished install"))); + yield* fs.rename(stagingDir, paths.installDir).pipe( + // Another server on this state folder may have published the same version first. + Effect.catch((cause) => + isInstalledAt(paths, spec.version).pipe( + Effect.flatMap((published) => + published ? Effect.void : Effect.fail(fail("publishing the install")(cause)), + ), + ), + ), + ); + return paths; + }).pipe( + Effect.ensuring(fs.remove(stagingDir, { recursive: true, force: true }).pipe(Effect.ignore)), + ); +}); + +/** + * Installs the pinned version of `tool` if it is missing and returns where it + * is. Holds a lease on the managed Node.js until `scope` closes. + */ +export const ensureDeviceTool = Effect.fn("DeviceToolchain.ensure")(function* ( + stateDir: string, + tool: DeviceToolKey, +) { + const path = yield* Path.Path; + const node = yield* leaseDeviceNode(stateDir); + const spec = SPECS[tool]; + const paths = yield* installLock.withPermit( + installTool(spec, toolPaths(path, stateDir, spec), node), + ); + return { paths, node }; +}); + +/** The entry of the first installed agent-device among `versions`; never installs. */ +export const installedAgentDeviceEntry = Effect.fn("DeviceToolchain.installedAgentDeviceEntry")( + function* (stateDir: string, versions: ReadonlyArray) { + const path = yield* Path.Path; + for (const version of versions) { + if (!VERSION_PATTERN.test(version)) continue; + const paths = toolPaths(path, stateDir, { ...SPECS.agent, version }); + if (yield* isInstalledAt(paths, version)) return paths.entryPath; + } + return null; + }, +); + +/** What is on disk for each tool and what runs, for Settings → Devices. */ +export const deviceToolVersions = Effect.fn("DeviceToolchain.versions")(function* ( + stateDir: string, + running: { readonly hub?: string | undefined; readonly agent?: string | undefined } = {}, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const inspect = Effect.fn("DeviceToolchain.inspect")(function* (spec: ToolSpec) { + const directory = path.join(deviceToolsRoot(path, stateDir), spec.name); + const names = yield* fs.readDirectory(directory).pipe(Effect.orElseSucceed(() => [])); + const versions = yield* Effect.filter(names, (version) => + VERSION_PATTERN.test(version) + ? isInstalledAt(toolPaths(path, stateDir, { ...spec, version }), version) + : Effect.succeed(false), + ); + return { + requiredVersion: spec.version, + installedVersions: versions.sort(), + runningVersion: running[spec.key] ?? null, + }; + }); + return { + hub: yield* inspect(SPECS.hub), + agent: yield* inspect(SPECS.agent), + } satisfies DeviceToolVersions; +}); + +/** + * Deletes installed versions of `tool` other than the pinned one and the most + * recent other one, never one a running process was started from (another + * Threadlines server may still run an older pin). Only once the pinned + * version is installed. Best effort. + */ +export const pruneDeviceTool = Effect.fn("DeviceToolchain.prune")(function* ( + stateDir: string, + tool: DeviceToolKey, + runningCommandLines: Effect.Effect, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const spec = SPECS[tool]; + const parent = path.join(deviceToolsRoot(path, stateDir), spec.name); + const names = yield* fs.readDirectory(parent).pipe(Effect.orElseSucceed(() => [])); + const completed: Array<{ version: string; dir: string; modified: number }> = []; + for (const version of names) { + if (!VERSION_PATTERN.test(version)) continue; + const paths = toolPaths(path, stateDir, { ...spec, version }); + if (!(yield* isInstalledAt(paths, version))) continue; + const info = yield* fs.stat(paths.sentinelPath).pipe(Effect.option); + const modified = + info._tag === "Some" && info.value.mtime._tag === "Some" + ? info.value.mtime.value.getTime() + : 0; + completed.push({ version, dir: paths.installDir, modified }); + } + if (!completed.some((entry) => entry.version === spec.version)) return; + const others = completed.filter((entry) => entry.version !== spec.version); + if (others.length <= 1) return; + // A process scan that fails keeps everything: never delete what may be running. + const commandLines = yield* runningCommandLines; + if (commandLines === null) return; + const previous = others.toSorted( + (a, b) => + b.modified - a.modified || b.version.localeCompare(a.version, "en", { numeric: true }), + )[0]?.version; + for (const entry of others) { + if (entry.version === previous) continue; + if (commandLines.includes(entry.dir + path.sep)) continue; + yield* fs.remove(entry.dir, { recursive: true, force: true }).pipe(Effect.ignore); + } +}); diff --git a/apps/server/src/device/LocalDeviceHost.test.ts b/apps/server/src/device/LocalDeviceHost.test.ts new file mode 100644 index 000000000..45adfafb4 --- /dev/null +++ b/apps/server/src/device/LocalDeviceHost.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from "@effect/vitest"; + +import { hubArguments, parseHubStartupLine, platformReason } from "./LocalDeviceHost.ts"; + +describe("parseHubStartupLine", () => { + // expo-device-hub 0.22.1 with --require-token prints exactly this (cli.mjs `startupMessage`). + it("reads the port and token from the pinned hub's Local line", () => { + expect( + parseHubStartupLine( + " Local: http://localhost:47811/?token=e0t5_0X1M1GVlfqrVNtbToKqFRKH6Rr0heuytDHo_HY", + ), + ).toEqual({ port: 47811, token: "e0t5_0X1M1GVlfqrVNtbToKqFRKH6Rr0heuytDHo_HY" }); + }); + + it("ignores every other line, including a Local line without a token", () => { + expect(parseHubStartupLine("Expo Device Hub ready")).toBeNull(); + expect(parseHubStartupLine(" Local: http://localhost:47811")).toBeNull(); + expect( + parseHubStartupLine(" Network: pass --host 0.0.0.0 to expose on your local network"), + ).toBeNull(); + }); +}); + +describe("hubArguments", () => { + it("binds loopback and requires the token", () => { + const args = hubArguments("/tools/cli.mjs", 4000, "darwin"); + expect(args.slice(0, 5)).toEqual(["/tools/cli.mjs", "--port", "4000", "--host", "127.0.0.1"]); + expect(args).toContain("--require-token"); + expect(args).toContain("--hide-sidebar"); + expect(args).toContain("--hide-boot-device"); + expect(hubArguments("/tools/cli.mjs", 4000, "linux")).toContain("android"); + }); +}); + +describe("platformReason", () => { + const sdk = { + root: "/sdk", + adb: true, + emulator: true, + avdmanager: true, + legacyAvdmanager: false, + }; + + it("names what is missing in words the user can act on", () => { + expect(platformReason("ios", "linux", sdk, false)).toBe( + "iOS Simulators need macOS with Xcode.", + ); + expect(platformReason("ios", "darwin", sdk, false)).toBe( + "Xcode command line tools were not found.", + ); + expect(platformReason("android", "darwin", { ...sdk, avdmanager: false }, true)).toContain( + "Command-line Tools (latest) are missing from /sdk", + ); + expect(platformReason("android", "darwin", { ...sdk, root: null }, true)).toContain( + "Android SDK was not found", + ); + }); + + it("is null when the platform can run", () => { + expect(platformReason("ios", "darwin", sdk, true)).toBeNull(); + expect(platformReason("android", "darwin", sdk, true)).toBeNull(); + }); +}); diff --git a/apps/server/src/device/LocalDeviceHost.ts b/apps/server/src/device/LocalDeviceHost.ts new file mode 100644 index 000000000..416646e91 --- /dev/null +++ b/apps/server/src/device/LocalDeviceHost.ts @@ -0,0 +1,910 @@ +/** + * This machine as a device host (docs/design/devices.md). Ported from T3 + * Code's LocalDeviceHost. + * + * The hub runs as a supervised child on a loopback port, with + * `--require-token`: it prints a session token once in its startup line, the + * server keeps it in memory and sends it as a bearer on every request, and + * every hub route but its health checks refuses requests without it. A second + * Threadlines server, an older release, or anything else on loopback cannot + * use this hub. + * + * `/device/hub.json` records `{pid, port, entryPath}` so the next + * server stops a hub this one left behind, checked by pid and command line, + * never by name. agent-device's daemon is started by its own CLI (the pid we + * spawn is the CLI's, not the daemon's) and stopped through the CLI's own stop + * command with our isolated state folder. + */ +import type { DevicePlatform, DevicePlatformAvailability } from "@threadlines/contracts"; +import { isCommandAvailable } from "@threadlines/shared/shell"; +import * as NetService from "@threadlines/shared/Net"; +import * as Clock from "effect/Clock"; +import * as Deferred from "effect/Deferred"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; +import { HttpClient } from "effect/http"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; + +import { writeFileStringAtomically } from "../atomicWrite.ts"; +import { ServerConfig } from "../config.ts"; +import { ProcessRunner } from "../processRunner.ts"; +import { releaseAgentDeviceSession } from "./AgentDeviceTarget.ts"; +import * as DeviceHost from "./DeviceHost.ts"; +import { makeDeviceHubOperations } from "./DeviceHubOperations.ts"; +import { + AGENT_DEVICE_VERSION, + agentDeviceStateDir, + DEVICE_HUB_VERSION, + type DeviceToolchainInstallError, + type DeviceToolPaths, + deviceStateDir, + deviceToolVersions, + ensureDeviceTool, + installedAgentDeviceEntry, + isDeviceToolInstalled, + leaseDeviceNode, + pruneDeviceTool, +} from "./DeviceToolchain.ts"; + +const HUB_READY_TIMEOUT_MS = 30_000; +const DAEMON_READY_TIMEOUT_MS = 30_000; +const DAEMON_POLL_MS = 100; +const HUB_RESTART_STABLE_UPTIME_MS = 60_000; +const HUB_RESTART_MAX_DELAY_MS = 30_000; + +const HubStateFile = Schema.Struct({ + pid: Schema.Int, + port: Schema.Int, + entryPath: Schema.String, +}); +const decodeHubStateFile = Schema.decodeUnknownEffect(Schema.fromJsonString(HubStateFile)); +const encodeHubStateFile = Schema.encodeUnknownEffect(Schema.fromJsonString(HubStateFile)); + +const AgentDeviceDaemonFile = Schema.Struct({ + httpPort: Schema.Int, + token: Schema.String, + pid: Schema.optional(Schema.Int), + version: Schema.optional(Schema.String), +}); +const decodeDaemonFile = Schema.decodeUnknownEffect(Schema.fromJsonString(AgentDeviceDaemonFile)); + +/** + * The hub's startup line with `--require-token` (expo-device-hub 0.22.1, + * `startupMessage` in its cli.mjs): ` Local: http://localhost:/?token=`. + * The token is 32 random bytes in base64url. + */ +export const parseHubStartupLine = ( + line: string, +): { readonly port: number; readonly token: string } | null => { + const match = /^\s*Local:\s+http:\/\/localhost:(\d+)\/\?token=([A-Za-z0-9_-]{16,})\s*$/.exec( + line, + ); + if (!match) return null; + const port = Number(match[1]); + return Number.isInteger(port) && port > 0 && port < 65536 ? { port, token: match[2]! } : null; +}; + +/** A hub output line with any token in it hidden, for the log. */ +const redactHubLine = (line: string) => line.replace(/token=[A-Za-z0-9_-]+/g, "token="); + +interface HubProcess { + readonly nodePath: string; + readonly child: ChildProcessSpawner.ChildProcessHandle; + /** Holds the child and the lease on the managed Node.js it runs on. */ + readonly scope: Scope.Closeable; + readonly origin: string; + readonly token: string; + readonly startedAtMillis: number; +} + +interface RunningHost { + readonly hub: HubProcess; + readonly agentDevice: DeviceHost.AgentDeviceEndpoint | null; + readonly helpers: DeviceHost.DeviceHostReady["helpers"]; +} + +interface AndroidSdk { + readonly root: string | null; + readonly adb: boolean; + readonly emulator: boolean; + readonly avdmanager: boolean; + readonly legacyAvdmanager: boolean; +} + +/** Where the Android SDK is, the way Android Studio and the hub look for it. */ +export const findAndroidSdk = Effect.fn("LocalDeviceHost.findAndroidSdk")(function* ( + environment: NodeJS.ProcessEnv, + platform: NodeJS.Platform, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const windows = platform === "win32"; + const home = environment.HOME ?? environment.USERPROFILE ?? ""; + const explicit = [environment.ANDROID_HOME?.trim(), environment.ANDROID_SDK_ROOT?.trim()].filter( + (root): root is string => !!root, + ); + const exists = (file: string) => fs.exists(file).pipe(Effect.orElseSucceed(() => false)); + const adbBinary = windows ? "adb.exe" : "adb"; + const avdmanagerBinary = windows ? "avdmanager.bat" : "avdmanager"; + const inspect = Effect.fnUntraced(function* (root: string) { + const adb = yield* exists(path.join(root, "platform-tools", adbBinary)); + const emulator = yield* exists( + path.join(root, "emulator", windows ? "emulator.exe" : "emulator"), + ); + const avdmanager = yield* exists( + path.join(root, "cmdline-tools", "latest", "bin", avdmanagerBinary), + ); + const legacyAvdmanager = + !avdmanager && (yield* exists(path.join(root, "tools", "bin", avdmanagerBinary))); + return { root, adb, emulator, avdmanager, legacyAvdmanager } satisfies AndroidSdk; + }); + const candidates = [ + ...explicit, + path.join(home, "Library", "Android", "sdk"), + path.join(home, "Android", "Sdk"), + path.join(environment.LOCALAPPDATA ?? path.join(home, "AppData", "Local"), "Android", "Sdk"), + ]; + for (const root of candidates) { + const sdk = yield* inspect(root); + if (sdk.adb || sdk.emulator) return sdk; + } + // PATH entries can be slow to stat (network shares), so only after the standard places miss. + const searchPath = environment.PATH ?? environment.Path ?? ""; + for (const directory of searchPath.split(windows ? ";" : ":")) { + if (!directory) continue; + const adb = path.join(directory, adbBinary); + if (!(yield* exists(adb))) continue; + const resolved = yield* fs.realPath(adb).pipe(Effect.orElseSucceed(() => adb)); + const sdk = yield* inspect(path.dirname(path.dirname(resolved))); + if (sdk.adb || sdk.emulator) return sdk; + } + // No real SDK anywhere: report what the configured root is missing. + if (explicit[0]) return yield* inspect(explicit[0]); + return { + root: null, + adb: false, + emulator: false, + avdmanager: false, + legacyAvdmanager: false, + } satisfies AndroidSdk; +}); + +/** Why a platform can't run here, in words the user can act on, or null when it can. */ +export const platformReason = ( + platform: DevicePlatform, + hostPlatform: NodeJS.Platform, + sdk: AndroidSdk, + hasXcrun: boolean, +): string | null => { + if (platform === "ios") { + if (hostPlatform !== "darwin") return "iOS Simulators need macOS with Xcode."; + if (!hasXcrun) return "Xcode command line tools were not found."; + return null; + } + if (!sdk.root) + return "Android SDK was not found. Install it with Android Studio or set ANDROID_HOME to your SDK folder."; + if (!sdk.adb) + return `Android SDK Platform-Tools are missing from ${sdk.root}. Install them in Android Studio's SDK Manager.`; + if (!sdk.emulator) + return `Android Emulator is missing from ${sdk.root}. Install it in Android Studio's SDK Manager.`; + if (!sdk.avdmanager) { + if (sdk.legacyAvdmanager) + return `The Android SDK command-line tools in ${sdk.root} are an older, unsupported version. Install Android SDK Command-line Tools (latest) in Android Studio's SDK Manager under SDK Tools.`; + return `Android SDK Command-line Tools (latest) are missing from ${sdk.root}. Install them in Android Studio's SDK Manager.`; + } + return null; +}; + +/** The environment device tools run in: the SDK's tools first on PATH. */ +const deviceHostEnvironment = ( + environment: NodeJS.ProcessEnv, + sdkRoot: string | null, + hostPlatform: NodeJS.Platform, + path: Path.Path, +): NodeJS.ProcessEnv => { + const { NODE_OPTIONS: _nodeOptions, ...rest } = environment; + return sdkRoot + ? { + ...rest, + ANDROID_HOME: sdkRoot, + PATH: [ + path.join(sdkRoot, "platform-tools"), + path.join(sdkRoot, "emulator"), + environment.PATH ?? environment.Path ?? "", + ].join(hostPlatform === "win32" ? ";" : ":"), + } + : rest; +}; + +/** Only a Mac can list Apple simulators; elsewhere the hub's attempt surfaces as an error. */ +const hubPlatformArguments = (hostPlatform: NodeJS.Platform) => + hostPlatform === "darwin" ? [] : ["--platform", "android"]; + +/** The hub's arguments: loopback only, token required, no device list or boot controls of its own. */ +export const hubArguments = (entryPath: string, port: number, hostPlatform: NodeJS.Platform) => [ + entryPath, + "--port", + String(port), + "--host", + "127.0.0.1", + "--require-token", + "--hide-sidebar", + "--hide-boot-device", + ...hubPlatformArguments(hostPlatform), +]; + +const isProcessAlive = (pid: number) => + Effect.sync(() => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } + }); + +const hostError = + (step: string, detail?: string) => + (cause: unknown): DeviceHost.DeviceHostError => + new DeviceHost.DeviceHostError({ step, ...(detail ? { detail } : {}), cause }); + +const installFailure = (step: string) => (cause: DeviceToolchainInstallError) => + new DeviceHost.DeviceHostError({ step, detail: cause.message, cause }); + +export const make = Effect.fn("LocalDeviceHost.make")(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const config = yield* ServerConfig; + const path = yield* Path.Path; + const fs = yield* FileSystem.FileSystem; + const net = yield* NetService.NetService; + const runner = yield* ProcessRunner; + const httpClient = yield* HttpClient.HttpClient; + const environment = process.env; + const hostPlatform = process.platform; + const stateDir = config.stateDir; + const services = Effect.provideService(FileSystem.FileSystem, fs); + const withPath = Effect.provideService(Path.Path, path); + + // Resolved per use: an SDK installed after the server started is found without a restart. + const currentSdk = findAndroidSdk(environment, hostPlatform).pipe(services, withPath); + const currentHostEnvironment = currentSdk.pipe( + Effect.map((sdk) => deviceHostEnvironment(environment, sdk.root, hostPlatform, path)), + ); + const startLock = yield* Semaphore.make(1); + const runningRef = yield* Ref.make(null); + const restartDelayRef = yield* Ref.make(0); + const hubStatePath = path.join(deviceStateDir(path, stateDir), "hub.json"); + const daemonFilePath = path.join(agentDeviceStateDir(path, stateDir), "daemon.json"); + + const platformAvailability = Effect.fn("LocalDeviceHost.platformAvailability")(function* ( + platform: DevicePlatform, + ): Effect.fn.Return { + const sdk = yield* currentSdk; + const hasXcrun = platform === "ios" && hostPlatform === "darwin" && isCommandAvailable("xcrun"); + const reason = platformReason(platform, hostPlatform, sdk, hasXcrun); + return reason === null ? { platform, available: true } : { platform, available: false, reason }; + }); + + const readDaemonFile = fs + .readFileString(daemonFilePath) + .pipe(Effect.flatMap(decodeDaemonFile), Effect.option); + + const summary: DeviceHost.DeviceHost["Service"]["summary"] = Effect.gen(function* () { + const [platforms, hubInstalled, agentDeviceInstalled] = yield* Effect.all([ + Effect.all([platformAvailability("ios"), platformAvailability("android")]), + isDeviceToolInstalled(stateDir, "hub").pipe(services, withPath), + isDeviceToolInstalled(stateDir, "agent").pipe(services, withPath), + ]); + const running = yield* Ref.get(runningRef); + const hubAlive = running + ? yield* running.hub.child.isRunning.pipe(Effect.orElseSucceed(() => false)) + : false; + const daemon = running?.agentDevice ? yield* readDaemonFile : Option.none(); + const agentAlive = + Option.isSome(daemon) && daemon.value.pid ? yield* isProcessAlive(daemon.value.pid) : false; + const tools = yield* deviceToolVersions(stateDir, { + ...(hubAlive ? { hub: DEVICE_HUB_VERSION } : {}), + ...(agentAlive && Option.isSome(daemon) && daemon.value.version + ? { agent: daemon.value.version } + : {}), + }).pipe(services, withPath, Effect.option); + return { + platforms, + hubInstalled, + agentDeviceInstalled, + tools: Option.getOrUndefined(tools), + toolInspectionError: Option.isNone(tools) + ? "Could not read which device tool versions are installed." + : undefined, + }; + }); + + const stopHub = (hub: HubProcess | undefined) => + hub ? Scope.close(hub.scope, Exit.void).pipe(Effect.ignore) : Effect.void; + + const commandLineOf = (pid: number) => + runner + .run({ + command: "ps", + args: ["-o", "command=", "-p", String(pid)], + timeout: Duration.seconds(5), + timeoutBehavior: "timedOutResult", + }) + .pipe( + Effect.map((result) => result.stdout), + Effect.orElseSucceed(() => ""), + ); + + // A previous server's hub, stopped only if the recorded pid still runs the recorded entry: + // the pid may have been reused by anything since. + const reapStaleHub = Effect.gen(function* () { + const previous = yield* fs + .readFileString(hubStatePath) + .pipe(Effect.flatMap(decodeHubStateFile), Effect.option); + if (Option.isNone(previous)) return; + const { pid, port, entryPath } = previous.value; + if (hostPlatform !== "win32" && (yield* isProcessAlive(pid))) { + if ((yield* commandLineOf(pid)).includes(entryPath)) { + yield* Effect.logWarning("Stopping a device hub left behind by a previous server", { + pid, + port, + }); + yield* Effect.sync(() => { + try { + process.kill(pid, "SIGTERM"); + } catch {} + }); + } + } + yield* fs.remove(hubStatePath, { force: true }).pipe(Effect.ignore); + }).pipe(Effect.ignoreCause); + + const recordHub = (hub: HubProcess, hubTool: DeviceToolPaths) => + encodeHubStateFile({ + pid: Number(hub.child.pid), + port: Number(new URL(hub.origin).port), + entryPath: hubTool.entryPath, + }).pipe( + Effect.flatMap((contents) => writeFileStringAtomically({ filePath: hubStatePath, contents })), + services, + withPath, + Effect.ignore, + ); + + const waitForReady = (origin: string) => + Effect.gen(function* () { + const deadline = (yield* Clock.currentTimeMillis) + HUB_READY_TIMEOUT_MS; + while (true) { + const ok = yield* httpClient + .pipe(HttpClient.withScope) + .get(`${origin}/readyz`) + .pipe( + Effect.flatMap((response) => + response.arrayBuffer.pipe(Effect.as(response.status === 200)), + ), + Effect.scoped, + Effect.timeout(Duration.seconds(2)), + Effect.orElseSucceed(() => false), + ); + if (ok) return; + if ((yield* Clock.currentTimeMillis) > deadline) { + return yield* new DeviceHost.DeviceHostError({ + step: "waiting for the device hub to answer", + detail: `The device hub did not become ready within ${HUB_READY_TIMEOUT_MS / 1000} seconds.`, + }); + } + yield* Effect.sleep(Duration.millis(200)); + } + }); + + const hubEnvironment = (base: NodeJS.ProcessEnv): NodeJS.ProcessEnv => ({ + ...base, + FORCE_COLOR: "0", + NO_COLOR: "1", + // The hub writes `server-.json` there; keep it out of the shared temp folder. + EXPO_DEVICE_HUB_STATE_DIR: path.join(deviceStateDir(path, stateDir), "hub-state"), + }); + + const spawnHub = Effect.fn("LocalDeviceHost.spawnHub")(function* (hubTool: DeviceToolPaths) { + yield* reapStaleHub; + yield* fs + .makeDirectory(deviceStateDir(path, stateDir), { recursive: true }) + .pipe(Effect.ignore); + const reserved = yield* net + .reserveLoopbackPort("127.0.0.1") + .pipe(Effect.mapError(hostError("reserving a port for the device hub"))); + const scope = yield* Scope.make("sequential"); + // Not recorded yet: stop it on failure or interruption from the spawn on. + return yield* Effect.gen(function* () { + const node = yield* leaseDeviceNode(stateDir).pipe( + Effect.provideService(Scope.Scope, scope), + services, + withPath, + Effect.mapError(installFailure("preparing Node.js")), + ); + const child = yield* spawner + .spawn( + ChildProcess.make(node.node, hubArguments(hubTool.entryPath, reserved, hostPlatform), { + cwd: deviceStateDir(path, stateDir), + detached: false, + shell: false, + stdout: "pipe", + stderr: "pipe", + env: hubEnvironment(yield* currentHostEnvironment), + }), + ) + .pipe( + Effect.provideService(Scope.Scope, scope), + Effect.mapError(hostError("starting the device hub")), + ); + const startedAtMillis = yield* Clock.currentTimeMillis; + const started = yield* Deferred.make<{ readonly port: number; readonly token: string }>(); + yield* Effect.forkIn(observeHubOutput(child, started), scope); + const announced = yield* Deferred.await(started).pipe( + Effect.timeoutOption(Duration.millis(HUB_READY_TIMEOUT_MS)), + ); + if (Option.isNone(announced)) { + return yield* new DeviceHost.DeviceHostError({ + step: "waiting for the device hub to start", + detail: "The device hub did not print its startup line. Update it in Settings → Devices.", + }); + } + const origin = `http://127.0.0.1:${announced.value.port}`; + yield* waitForReady(origin); + const hub: HubProcess = { + child, + scope, + origin, + token: announced.value.token, + startedAtMillis, + nodePath: node.node, + }; + yield* recordHub(hub, hubTool); + yield* Effect.logInfo("Device hub started", { + pid: Number(child.pid), + port: announced.value.port, + }); + return hub; + }).pipe(Effect.onError(() => Scope.close(scope, Exit.void).pipe(Effect.ignore))); + }); + + const observeHubOutput = ( + child: ChildProcessSpawner.ChildProcessHandle, + started: Deferred.Deferred<{ readonly port: number; readonly token: string }>, + ) => + child.all.pipe( + Stream.decodeText(), + Stream.splitLines, + Stream.runForEach((line) => + Effect.gen(function* () { + const startup = parseHubStartupLine(line); + if (startup) { + yield* Deferred.succeed(started, startup); + return; + } + const text = line.trim(); + if (text.length > 0) { + yield* Effect.logDebug("Device hub output", { + pid: Number(child.pid), + output: redactHubLine(text), + }); + } + }), + ), + Effect.ignoreCause, + ); + + // Waits for the hub's lifetime and starts a replacement with backoff; a span would nest across restarts. + const superviseHub: (hub: HubProcess, hubTool: DeviceToolPaths) => Effect.Effect = + Effect.fnUntraced( + function* (hub: HubProcess, hubTool: DeviceToolPaths) { + yield* Effect.result(hub.child.exitCode); + const running = yield* Ref.get(runningRef); + if (running?.hub.child.pid !== hub.child.pid) return; + const uptime = (yield* Clock.currentTimeMillis) - hub.startedAtMillis; + const delay = yield* Ref.modify(restartDelayRef, (current) => { + if (uptime >= HUB_RESTART_STABLE_UPTIME_MS) return [0, 0]; + const next = current === 0 ? 1_000 : Math.min(current * 2, HUB_RESTART_MAX_DELAY_MS); + return [current, next]; + }); + yield* Effect.logWarning("Device hub exited; restarting", { + pid: Number(hub.child.pid), + delayMs: delay, + }); + yield* stopHub(hub); + yield* Effect.sleep(Duration.millis(delay)); + yield* startLock.withPermits(1)( + Effect.gen(function* () { + const current = yield* Ref.get(runningRef); + if (current?.hub.child.pid !== hub.child.pid) return; + const replacement = yield* spawnHub(hubTool); + yield* Ref.set(runningRef, { ...current, hub: replacement }); + yield* Effect.forkDetach(superviseHub(replacement, hubTool)); + }), + ); + }, + Effect.catchCause((cause) => Effect.logWarning("Device hub supervisor failed", { cause })), + ); + + /** Every process's command line, for keeping tool versions a running process uses. */ + const runningCommandLines = runner + .run({ + command: "ps", + args: ["-ax", "-o", "command="], + timeout: Duration.seconds(10), + timeoutBehavior: "timedOutResult", + maxOutputBytes: 16 * 1024 * 1024, + }) + .pipe( + Effect.map((result) => (result.code === 0 && !result.timedOut ? result.stdout : null)), + Effect.orElseSucceed(() => null), + ); + + const pruneTool = (tool: "hub" | "agent") => + hostPlatform === "win32" + ? Effect.void + : pruneDeviceTool(stateDir, tool, runningCommandLines).pipe( + services, + withPath, + Effect.ignore, + ); + + const daemonEnvironment = Effect.gen(function* () { + return { + ...(yield* currentHostEnvironment), + AGENT_DEVICE_STATE_DIR: agentDeviceStateDir(path, stateDir), + AGENT_DEVICE_DAEMON_SERVER_MODE: "http", + // No five-minute idle timeout: the server owns shutdown. + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: "0", + AGENT_DEVICE_NO_UPDATE_NOTIFIER: "1", + FORCE_COLOR: "0", + NO_COLOR: "1", + } satisfies NodeJS.ProcessEnv; + }); + + const startAgentDeviceDaemon = Effect.fn("LocalDeviceHost.startAgentDeviceDaemon")(function* ( + entryPath: string, + nodePath: string, + ) { + yield* fs + .makeDirectory(agentDeviceStateDir(path, stateDir), { recursive: true }) + .pipe(Effect.ignore); + const toEndpoint = ( + file: typeof AgentDeviceDaemonFile.Type, + ): DeviceHost.AgentDeviceEndpoint => ({ + baseUrl: `http://127.0.0.1:${file.httpPort}`, + token: file.token, + entryPath, + }); + const existing = yield* readDaemonFile; + if (Option.isSome(existing) && existing.value.version !== AGENT_DEVICE_VERSION) { + // An earlier server's daemon on another pin: stop it with its own CLI and start ours. + yield* stopAgentDeviceDaemon; + } else if (Option.isSome(existing)) { + const alive = yield* httpClient + .pipe(HttpClient.withScope) + .get(`http://127.0.0.1:${existing.value.httpPort}/health`) + .pipe( + Effect.timeout(Duration.seconds(2)), + Effect.flatMap((response) => + response.arrayBuffer.pipe(Effect.as(response.status === 200)), + ), + Effect.scoped, + Effect.orElseSucceed(() => false), + ); + if (alive) return toEndpoint(existing.value); + yield* fs.remove(daemonFilePath, { force: true }).pipe(Effect.ignore); + } + // agent-device has no `daemon start`; `devices` is the cheapest command that starts and awaits it. + const runResult = yield* runner + .run({ + command: nodePath, + args: [entryPath, "devices", "--json"], + env: yield* daemonEnvironment, + timeout: Duration.millis(DAEMON_READY_TIMEOUT_MS), + timeoutBehavior: "timedOutResult", + }) + .pipe(Effect.result); + // Discovery may fail after startup; a published endpoint is enough. + const started = yield* readDaemonFile; + if (Option.isSome(started)) return toEndpoint(started.value); + if (runResult._tag === "Failure") { + return yield* hostError("starting agent tools")(runResult.failure); + } + if (runResult.success.timedOut) { + return yield* new DeviceHost.DeviceHostError({ + step: "starting agent tools", + detail: `Agent tools did not start within ${DAEMON_READY_TIMEOUT_MS / 1000} seconds.`, + }); + } + if (runResult.success.code !== 0) { + return yield* new DeviceHost.DeviceHostError({ + step: "starting agent tools", + detail: `agent-device could not start (exit code ${runResult.success.code}). Its log is in ${agentDeviceStateDir(path, stateDir)}.`, + }); + } + const deadline = (yield* Clock.currentTimeMillis) + DAEMON_READY_TIMEOUT_MS; + while (true) { + const file = yield* readDaemonFile; + if (Option.isSome(file)) return toEndpoint(file.value); + if ((yield* Clock.currentTimeMillis) > deadline) { + return yield* new DeviceHost.DeviceHostError({ + step: "starting agent tools", + detail: `Agent tools did not start within ${DAEMON_READY_TIMEOUT_MS / 1000} seconds.`, + }); + } + yield* Effect.sleep(Duration.millis(DAEMON_POLL_MS)); + } + }); + + // The daemon may predate this process (an earlier server was killed, possibly on an + // older pin), so it is stopped with the CLI of its recorded version when that is installed. + const stopAgentDeviceDaemon = Effect.gen(function* () { + if (!(yield* fs.exists(daemonFilePath).pipe(Effect.orElseSucceed(() => false)))) return; + const recorded = yield* readDaemonFile; + if ( + Option.isSome(recorded) && + recorded.value.pid && + !(yield* isProcessAlive(recorded.value.pid)) + ) { + return yield* fs.remove(daemonFilePath, { force: true }); + } + const entryPath = yield* installedAgentDeviceEntry(stateDir, [ + ...(Option.isSome(recorded) && recorded.value.version ? [recorded.value.version] : []), + AGENT_DEVICE_VERSION, + ]).pipe(withPath, services); + if (!entryPath) return; + yield* Effect.scoped( + Effect.gen(function* () { + const node = yield* leaseDeviceNode(stateDir).pipe(services, withPath); + yield* runner.run({ + command: node.node, + args: [entryPath, "daemon", "stop", "--state-dir", agentDeviceStateDir(path, stateDir)], + env: { ...(yield* currentHostEnvironment), AGENT_DEVICE_NO_UPDATE_NOTIFIER: "1" }, + timeout: Duration.seconds(10), + timeoutBehavior: "timedOutResult", + }); + }), + ); + }).pipe(Effect.ignore); + + const serveSimHelpers = Effect.fn("LocalDeviceHost.serveSimHelpers")(function* ( + hubTool: DeviceToolPaths, + ) { + const serveSimDist = path.join( + hubTool.installDir, + "node_modules", + "expo-device-hub", + "vendor", + "serve-sim", + "dist", + ); + const candidate = { + serveSimAxSettings: path.join(serveSimDist, "simax", "serve-sim-ax-settings"), + serveSimCli: path.join(serveSimDist, "serve-sim.js"), + }; + const [axExists, cliExists] = yield* Effect.all([ + fs.exists(candidate.serveSimAxSettings).pipe(Effect.orElseSucceed(() => false)), + fs.exists(candidate.serveSimCli).pipe(Effect.orElseSucceed(() => false)), + ]); + return { + serveSimAxSettings: axExists ? candidate.serveSimAxSettings : null, + serveSimCli: cliExists ? candidate.serveSimCli : null, + }; + }); + + const ensureHubReady = Effect.fn("LocalDeviceHost.ensureHubReady")(function* ( + onPhase: (phase: DeviceHost.DeviceHostPhase, detail?: string) => Effect.Effect, + ) { + const running = yield* Ref.get(runningRef); + // A dead hub's record stays until a replacement is published: the agent + // daemon outlives the hub, and close needs its endpoint if the restart fails. + if (running) { + const alive = yield* running.hub.child.isRunning.pipe(Effect.orElseSucceed(() => false)); + if (alive) return running; + } + if (!(yield* isDeviceToolInstalled(stateDir, "hub").pipe(services, withPath))) { + yield* onPhase("installing", `Installing the device hub ${DEVICE_HUB_VERSION}…`); + } + const { paths: hubTool } = yield* Effect.scoped( + ensureDeviceTool(stateDir, "hub").pipe( + services, + withPath, + Effect.mapError(installFailure("installing device support")), + ), + ); + yield* onPhase("starting"); + const hub = yield* spawnHub(hubTool); + const next = yield* Effect.gen(function* () { + const nextRunning: RunningHost = { + hub, + agentDevice: running?.agentDevice ?? null, + helpers: yield* serveSimHelpers(hubTool), + }; + yield* Ref.set(runningRef, nextRunning); + return nextRunning; + }).pipe(Effect.onError(() => stopHub(hub))); + if (running) yield* stopHub(running.hub); + yield* Ref.set(restartDelayRef, 0); + yield* Effect.forkDetach(superviseHub(hub, hubTool)); + yield* pruneTool("hub").pipe(Effect.forkDetach); + return next; + }); + + const run: DeviceHost.DeviceHostReady["run"] = (command, args, options) => + currentSdk.pipe( + Effect.flatMap((sdk) => + runner.run({ + command: + command === "emulator" && sdk.root + ? path.join( + sdk.root, + "emulator", + hostPlatform === "win32" ? "emulator.exe" : "emulator", + ) + : command, + args, + env: deviceHostEnvironment(environment, sdk.root, hostPlatform, path), + timeout: Duration.millis(options?.timeoutMs ?? 20_000), + timeoutBehavior: "timedOutResult", + ...(options?.stdin === undefined ? {} : { stdin: options.stdin }), + ...(options?.maxOutputBytes === undefined + ? {} + : { maxOutputBytes: options.maxOutputBytes }), + }), + ), + Effect.map((result) => ({ + stdout: result.stdout, + stderr: result.stderr, + code: result.timedOut ? 124 : Number(result.code ?? 127), + })), + Effect.catch((cause) => Effect.succeed({ stdout: "", stderr: String(cause), code: 127 })), + ); + + const toReady = (running: RunningHost): DeviceHost.DeviceHostReady => ({ + hub: { origin: running.hub.origin, token: running.hub.token }, + devices: makeDeviceHubOperations({ + origin: running.hub.origin, + token: running.hub.token, + http: httpClient, + run, + platformAvailable: (platform) => + platformAvailability(platform).pipe(Effect.map((availability) => availability.available)), + }), + nodePath: running.hub.nodePath, + run, + helpers: running.helpers, + ...(running.agentDevice + ? { releaseAgentSession: releaseAgentDeviceSession(httpClient, running.agentDevice) } + : {}), + }); + + const ensureReady: DeviceHost.DeviceHost["Service"]["ensureReady"] = (onPhase) => + startLock.withPermits(1)(ensureHubReady(onPhase).pipe(Effect.map(toReady))); + + const ensureAgentReady: DeviceHost.DeviceHost["Service"]["ensureAgentReady"] = (onPhase) => + startLock.withPermits(1)( + Effect.gen(function* () { + const running = yield* ensureHubReady(onPhase); + if (!(yield* isDeviceToolInstalled(stateDir, "agent").pipe(services, withPath))) { + yield* onPhase("installing", `Installing agent tools ${AGENT_DEVICE_VERSION}…`); + } + const { paths: agentTool } = yield* Effect.scoped( + ensureDeviceTool(stateDir, "agent").pipe( + services, + withPath, + Effect.mapError(installFailure("installing agent tools")), + ), + ); + if (!running.agentDevice) yield* onPhase("starting"); + const agentDevice = yield* startAgentDeviceDaemon( + agentTool.entryPath, + running.hub.nodePath, + ); + const next = { ...running, agentDevice }; + yield* Ref.set(runningRef, next); + yield* pruneTool("agent").pipe(Effect.forkDetach); + return { ...toReady(next), agentDevice }; + }), + ); + + const current: DeviceHost.DeviceHost["Service"]["current"] = Ref.get(runningRef).pipe( + Effect.map((running) => (running ? toReady(running) : null)), + ); + + const releaseAgentSession: DeviceHost.DeviceHost["Service"]["releaseAgentSession"] = (session) => + Effect.gen(function* () { + const running = yield* Ref.get(runningRef); + const endpoint = + running?.agentDevice ?? + (yield* readDaemonFile.pipe( + Effect.map((file) => + Option.isSome(file) + ? { baseUrl: `http://127.0.0.1:${file.value.httpPort}`, token: file.value.token } + : null, + ), + )); + if (!endpoint) return; + yield* releaseAgentDeviceSession(httpClient, endpoint)(session); + }).pipe( + Effect.catch((cause) => + Effect.logWarning("Could not end the agent-device session", { session, cause }), + ), + ); + + const stopAgent: DeviceHost.DeviceHost["Service"]["stopAgent"] = startLock.withPermits(1)( + Effect.gen(function* () { + yield* stopAgentDeviceDaemon; + yield* Ref.update(runningRef, (running) => + running ? { ...running, agentDevice: null } : running, + ); + }), + ); + + const stop: DeviceHost.DeviceHost["Service"]["stop"] = startLock.withPermits(1)( + Effect.gen(function* () { + const running = yield* Ref.getAndSet(runningRef, null); + // Hub cleanup waits for its exit; it must not strand the detached agent daemon. + yield* Effect.all( + [ + stopHub(running?.hub).pipe( + Effect.andThen(fs.remove(hubStatePath, { force: true }).pipe(Effect.ignore)), + ), + stopAgentDeviceDaemon, + ], + { concurrency: "unbounded", discard: true }, + ); + }), + ); + + /** + * Installs the pinned version (a repair when its files went missing). The + * hub always starts on the pinned version; a daemon an earlier server left + * running on an older pin is restarted on this one. + */ + const updateTool: DeviceHost.DeviceHost["Service"]["updateTool"] = (tool) => + Effect.gen(function* () { + yield* Effect.scoped( + ensureDeviceTool(stateDir, tool).pipe( + services, + withPath, + Effect.mapError( + installFailure(tool === "hub" ? "updating the device hub" : "updating agent tools"), + ), + ), + ); + if (tool !== "agent") return; + const running = yield* Ref.get(runningRef); + if (!running?.agentDevice) return; + const daemon = yield* readDaemonFile; + if (Option.isSome(daemon) && daemon.value.version === AGENT_DEVICE_VERSION) return; + yield* stopAgent; + yield* ensureAgentReady(() => Effect.void); + }); + + yield* Effect.addFinalizer(() => stop); + + return DeviceHost.DeviceHost.of({ + summary, + platformAvailability, + ensureReady, + ensureAgentReady, + updateTool, + current, + releaseAgentSession, + stopAgent, + stop, + }); +}); + +export const layer = Layer.effect(DeviceHost.DeviceHost, make()); diff --git a/apps/server/src/device/deviceToolVersions.json b/apps/server/src/device/deviceToolVersions.json new file mode 100644 index 000000000..ce61669b2 --- /dev/null +++ b/apps/server/src/device/deviceToolVersions.json @@ -0,0 +1,4 @@ +{ + "agent-device": "0.21.36", + "expo-device-hub": "0.22.1" +} diff --git a/apps/server/src/device/mjpegStillFrame.ts b/apps/server/src/device/mjpegStillFrame.ts new file mode 100644 index 000000000..7cbe30075 --- /dev/null +++ b/apps/server/src/device/mjpegStillFrame.ts @@ -0,0 +1,121 @@ +/** + * Browsers paint a multipart/x-mixed-replace part only once the next part begins, and the hub + * writes a part only when the screen changes. Left alone, an `` viewer never shows a still + * screen (clients such as the iOS app then give up waiting for the first frame) and always hides + * the last frame of a change. Re-sending the last part after a short quiet period makes it paint. + */ +import * as Arr from "effect/Array"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Option from "effect/Option"; +import * as Stream from "effect/Stream"; + +const STILL_FRAME_DELAY = Duration.millis(100); + +const HEADER_END = Buffer.from("\r\n\r\n"); +const MAX_HEADER_BYTES = 16 * 1024; +// Simulator frames are far smaller; a larger claim is unknown framing, not something to retain. +const MAX_PART_BYTES = 16 * 1024 * 1024; + +type Part = Arr.NonEmptyReadonlyArray; + +const partLength = (headers: string) => { + if (!headers.startsWith("--")) return undefined; + const length = Number(/^content-length:\s*(\d+)\s*$/im.exec(headers)?.[1]); + return length <= MAX_PART_BYTES ? length : undefined; +}; + +const endsWithCrlf = (part: Part) => { + const last = Arr.lastNonEmpty(part); + const cr = last.length > 1 ? last.at(-2) : part.at(-2)?.at(-1); + return cr === 0x0d && last.at(-1) === 0x0a; +}; + +/** + * Follows the hub's part framing (`--frame`, `Content-Length`, CRLF CRLF, body, CRLF; each part + * is one write) and keeps the last complete part as views into the forwarded chunks, so frame + * bytes are never copied. Any other framing turns tracking off and the rest passes through. + */ +const partTracker = () => { + let pieces: Array = []; + let size = 0; + let partEnd: number | null = null; + let tracking = true; + return { + /** The last complete part, while the stream sits at a part boundary. */ + still: null as Part | null, + feed(chunk: Uint8Array) { + if (!tracking) return; + let rest = Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength); + let part: Part | undefined; + while (rest.length > 0) { + if (partEnd === null) { + // A header split across chunks is re-scanned per chunk. Only a header-sized + // prefix of the new chunk joins the scan, so frame bytes are never copied. + const head = + size === 0 + ? rest + : Buffer.concat([...pieces, rest.subarray(0, MAX_HEADER_BYTES + HEADER_END.length)]); + const headerEnd = head.indexOf(HEADER_END); + if (headerEnd < 0 && head.length <= MAX_HEADER_BYTES) { + pieces.push(rest); + size += rest.length; + break; + } + const length = + headerEnd < 0 || headerEnd > MAX_HEADER_BYTES + ? undefined + : partLength(head.subarray(0, headerEnd).toString("latin1")); + if (length === undefined) { + tracking = false; + break; + } + partEnd = headerEnd + HEADER_END.length + length + 2; + } + const taken = rest.subarray(0, partEnd - size); + pieces.push(taken); + size += taken.length; + rest = rest.subarray(taken.length); + if (size < partEnd) break; + const complete = pieces; + if (!Arr.isArrayNonEmpty(complete) || !endsWithCrlf(complete)) { + tracking = false; + break; + } + part = complete; + pieces = []; + size = 0; + partEnd = null; + } + if (!tracking) pieces = []; + this.still = tracking && size === 0 ? (part ?? this.still) : null; + }, + }; +}; + +/** Re-sends the latest frame once when the stream goes quiet between frames. */ +export const repeatStillFrame = (body: Stream.Stream) => + Stream.transformPull(body, (pull, scope) => + Effect.sync(() => { + const tracker = partTracker(); + let pending: Fiber.Fiber, Effect.Error> | null = + null; + const forward = (chunks: Effect.Success) => { + pending = null; + for (const chunk of chunks) tracker.feed(chunk); + return chunks; + }; + return Effect.gen(function* () { + const still = tracker.still; + if (still === null && pending === null) return forward(yield* pull); + pending ??= yield* Effect.forkIn(pull, scope); + if (still === null) return forward(yield* Fiber.join(pending)); + // Timing out the join leaves the pull running, so a frame arriving mid-wait is kept. + const next = yield* Fiber.join(pending).pipe(Effect.timeoutOption(STILL_FRAME_DELAY)); + if (Option.isSome(next)) return forward(next.value); + tracker.still = null; + return still; + }); + }), + ); diff --git a/apps/server/src/mcp/McpDeviceServer.ts b/apps/server/src/mcp/McpDeviceServer.ts new file mode 100644 index 000000000..f5b2d9d21 --- /dev/null +++ b/apps/server/src/mcp/McpDeviceServer.ts @@ -0,0 +1,229 @@ +/** + * The device tools' endpoint, `threadlines_device` at `/mcp/device` + * (docs/design/devices.md). + * + * Its own MCP server, like McpCommandsServer: a provider lists it only for + * sessions started with `deviceTools`, and Codex approves calls per server. + * The same bearer credentials as `/mcp` (McpSessionRegistry); a credential + * reaches this endpoint only if it carries the `device` grant, and each call + * checks again that "Use devices" and "Agents can use devices" are still on. + * + * Hand-registered rather than a toolkit so `device_screenshot` can answer + * with an MCP image block instead of base64 the model cannot see. + */ +import * as Cause from "effect/Cause"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import type * as Types from "effect/Types"; +import { McpProtocol, McpSchema, McpServer, Tool } from "effect/ai"; +import { HttpRouter, HttpServerRequest, type HttpServerResponse } from "effect/http"; + +import { DeviceService } from "../device/DeviceService.ts"; +import { deviceClose, deviceList, deviceOpen, deviceScreenshot } from "./deviceToolHandlers.ts"; +import { + DEVICE_CLOSE_TOOL_NAME, + DEVICE_LIST_TOOL_NAME, + DEVICE_OPEN_TOOL_NAME, + DEVICE_SCREENSHOT_TOOL_NAME, + DEVICE_TOOLS, + DeviceToolListInput, +} from "./deviceTools.ts"; +import { + DeviceToolCloseInput, + DeviceToolOpenInput, + DeviceToolTargetInput, +} from "@threadlines/contracts"; +import { + MCP_DEVICE_ROUTE_PATH, + normalizeMcpHttpResponse, + readBearerToken, + unauthorizedMcpResponse, +} from "./McpHttpServer.ts"; +import { type McpInvocationScope, mcpSessionRegistry } from "./McpSessionRegistry.ts"; + +/** Which runtime a device tool call came from: the full credential scope. */ +export class McpDeviceInvocation extends Context.Service()( + "@threadlines/server/mcp/McpDeviceInvocation", +) {} + +const authenticate = Effect.succeed( + Effect.fn("McpDeviceServer.authenticate")(function* ( + handler: Effect.Effect< + HttpServerResponse.HttpServerResponse, + Types.unhandled, + McpDeviceInvocation + >, + ) { + const request = yield* HttpServerRequest.HttpServerRequest; + const scope = yield* mcpSessionRegistry.resolve(readBearerToken(request)); + if (scope === null || !scope.device) { + return unauthorizedMcpResponse; + } + return yield* handler.pipe( + Effect.provideService(McpDeviceInvocation, scope), + Effect.map(normalizeMcpHttpResponse), + ); + }), +); + +const AuthenticationLive = HttpRouter.middleware<{ + provides: McpDeviceInvocation; +}>()(authenticate).layer; + +const describe = (tool: Tool.Any) => + new McpSchema.Tool({ + name: tool.name, + description: Tool.getDescription(tool), + inputSchema: Tool.getJsonSchema(tool), + annotations: { + ...Context.getOption(tool.annotations, Tool.Title).pipe( + Option.map((title) => ({ title })), + Option.getOrUndefined, + ), + readOnlyHint: Context.get(tool.annotations, Tool.Readonly), + destructiveHint: Context.get(tool.annotations, Tool.Destructive), + idempotentHint: Context.get(tool.annotations, Tool.Idempotent), + openWorldHint: Context.get(tool.annotations, Tool.OpenWorld), + }, + }); + +const textResult = (text: string, isError: boolean, structured?: Record) => + new McpSchema.CallToolResult({ + isError, + ...(structured !== undefined ? { structuredContent: structured } : {}), + content: [{ type: "text", text }], + }); + +/** A failure the model can read: the error's own words. Interrupts and defects stay what they are. */ +const failureResult = (cause: Cause.Cause) => { + if (Cause.hasInterrupts(cause) || cause.reasons.some(Cause.isDieReason)) { + return Effect.failCause(cause).pipe(Effect.orDie); + } + const error = cause.reasons.find(Cause.isFailReason)?.error; + const message = + typeof error === "object" && error !== null && "message" in error + ? String(error.message) + : "The device tool failed."; + return Effect.succeed(textResult(message, true)); +}; + +const screenshotResult = (shot: { + readonly device: { readonly name: string; readonly version: string }; + readonly png: Uint8Array; + readonly width: number; + readonly height: number; +}) => + new McpSchema.CallToolResult({ + isError: false, + structuredContent: { width: shot.width, height: shot.height }, + content: [ + { + type: "text", + text: `Screenshot of ${shot.device.name} (${shot.device.version}), ${shot.width}x${shot.height} pixels.`, + }, + { type: "image", data: shot.png, mimeType: "image/png" }, + ], + }); + +const registerDeviceTools = Effect.fn("McpDeviceServer.registerDeviceTools")(function* () { + const server = yield* McpServer.McpServer; + const devices = yield* DeviceService; + const decodeList = Schema.decodeUnknownEffect(DeviceToolListInput); + const decodeOpen = Schema.decodeUnknownEffect(DeviceToolOpenInput); + const decodeTarget = Schema.decodeUnknownEffect(DeviceToolTargetInput); + const decodeClose = Schema.decodeUnknownEffect(DeviceToolCloseInput); + + // The caller comes off the request's own fiber: a hand-registered handler + // does not run under the router middleware that provides it. + const callerOf = (fiber: { readonly context: Context.Context }) => + Context.getUnsafe(fiber.context as Context.Context, McpDeviceInvocation); + + const badArguments = (error: unknown) => + textResult( + `The arguments do not fit the tool's schema: ${error instanceof Error ? error.message : String(error)}`, + true, + ); + + const handlers: Record< + string, + ( + caller: McpInvocationScope, + payload: unknown, + ) => Effect.Effect + > = { + [DEVICE_LIST_TOOL_NAME]: (caller, payload) => + decodeList(payload ?? {}).pipe( + Effect.matchEffect({ + onFailure: (error) => Effect.succeed(badArguments(error)), + onSuccess: (input) => + deviceList(caller, input).pipe( + Effect.map((result) => + textResult(JSON.stringify(result, null, 2), false, { ...result }), + ), + Effect.catchCause(failureResult), + ), + }), + ), + [DEVICE_OPEN_TOOL_NAME]: (caller, payload) => + decodeOpen(payload ?? {}).pipe( + Effect.matchEffect({ + onFailure: (error) => Effect.succeed(badArguments(error)), + onSuccess: (input) => + deviceOpen(caller, input).pipe( + Effect.map((result) => textResult(result.quickStart, false, { ...result })), + Effect.catchCause(failureResult), + ), + }), + ), + [DEVICE_SCREENSHOT_TOOL_NAME]: (caller, payload) => + decodeTarget(payload ?? {}).pipe( + Effect.matchEffect({ + onFailure: (error) => Effect.succeed(badArguments(error)), + onSuccess: (input) => + deviceScreenshot(caller, input).pipe( + Effect.map(screenshotResult), + Effect.catchCause(failureResult), + ), + }), + ), + [DEVICE_CLOSE_TOOL_NAME]: (caller, payload) => + decodeClose(payload ?? {}).pipe( + Effect.matchEffect({ + onFailure: (error) => Effect.succeed(badArguments(error)), + onSuccess: (input) => + deviceClose(caller, input).pipe( + Effect.as(textResult("Closed.", false, {})), + Effect.catchCause(failureResult), + ), + }), + ), + }; + + for (const tool of DEVICE_TOOLS) { + const handle = handlers[tool.name]!; + yield* server.addTool({ + tool: describe(tool), + annotations: tool.annotations, + handle: (payload) => + Effect.withFiber((fiber) => + handle(callerOf(fiber), payload).pipe(Effect.provideService(DeviceService, devices)), + ), + }); + } +}); + +export const layer = Layer.fresh( + Layer.effectDiscard(registerDeviceTools()).pipe( + Layer.provideMerge( + McpServer.layerHttp({ + name: "threadlines-device", + version: "1", + path: MCP_DEVICE_ROUTE_PATH, + protocols: [McpProtocol.v2025_06_18], + }).pipe(Layer.provide(AuthenticationLive)), + ), + ), +); diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index d4f7289ca..739c7fa44 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -79,6 +79,15 @@ export function mcpCommandsEndpointUrl(port: number): string { return `http://127.0.0.1:${port}${MCP_COMMANDS_ROUTE_PATH}`; } +/** The device tools' endpoint (McpDeviceServer, docs/design/devices.md): its + * own server, so a provider lists them only for sessions with `deviceTools`. */ +export const MCP_DEVICE_ROUTE_PATH = "/mcp/device"; + +/** The device tools' address, on loopback for the same reason. */ +export function mcpDeviceEndpointUrl(port: number): string { + return `http://127.0.0.1:${port}${MCP_DEVICE_ROUTE_PATH}`; +} + /** * The same endpoint, in the shape Codex takes it. * diff --git a/apps/server/src/mcp/McpSessionRegistry.ts b/apps/server/src/mcp/McpSessionRegistry.ts index 99f3a86ae..80f37a533 100644 --- a/apps/server/src/mcp/McpSessionRegistry.ts +++ b/apps/server/src/mcp/McpSessionRegistry.ts @@ -54,6 +54,8 @@ export interface McpInvocationScope { readonly pages: boolean; /** May ask the user to run commands at `/mcp/commands` (McpCommandsServer). */ readonly commands: boolean; + /** May use the device tools at `/mcp/device` (McpDeviceServer). */ + readonly device: boolean; /** Room tools this credential may call at `/mcp/room`. Empty: none. */ readonly roomTools: ReadonlySet; /** @@ -80,6 +82,8 @@ export interface McpCredentialRequest { readonly pages?: boolean | undefined; /** `ask_user_to_run`. Never granted to a side runtime, whatever is asked. */ readonly commands?: boolean | undefined; + /** The device tools. Never granted to a side runtime, whatever is asked. */ + readonly device?: boolean | undefined; /** * A side runtime's kind. A side key without one is treated as a review, * the narrowest. @@ -156,6 +160,7 @@ export const makeMcpSessionRegistry = (): McpSessionRegistryShape => { browser: request.browser && sideKind === undefined, pages: request.pages === true && sideKind === undefined, commands: request.commands === true && sideKind === undefined, + device: request.device === true && sideKind === undefined, roomTools: new Set( request.room ? roomToolsFor(sideKind, { diff --git a/apps/server/src/mcp/alertAssistantsToolHandlers.test.ts b/apps/server/src/mcp/alertAssistantsToolHandlers.test.ts index 34cde7710..06f9aae49 100644 --- a/apps/server/src/mcp/alertAssistantsToolHandlers.test.ts +++ b/apps/server/src/mcp/alertAssistantsToolHandlers.test.ts @@ -47,6 +47,7 @@ const caller = (side?: "ask"): McpInvocationScope => ({ browser: true, pages: true, commands: false, + device: false, roomTools: new Set(roomToolsFor(side, { guideOnly: side === undefined })), ...(side !== undefined ? { side: { sideTurnId: SideTurnId.make("side-1"), kind: side } } : {}), }); diff --git a/apps/server/src/mcp/deviceToolHandlers.ts b/apps/server/src/mcp/deviceToolHandlers.ts new file mode 100644 index 000000000..eb5eb634c --- /dev/null +++ b/apps/server/src/mcp/deviceToolHandlers.ts @@ -0,0 +1,201 @@ +/** + * What the device tools do (docs/design/devices.md), for McpDeviceServer. + * Ported from T3 Code's device toolkit handlers, with one host per server. + */ +import { + type DeviceError, + type DeviceId, + type DevicePlatform, + type DeviceServiceState, + type DeviceSummary, + type DeviceToolCloseInput, + type DeviceToolListResult, + type DeviceToolOpenInput, + type DeviceToolOpenResult, + type DeviceToolTargetInput, + DeviceToolUnavailableError, + type ThreadId, +} from "@threadlines/contracts"; +import * as Effect from "effect/Effect"; + +import { DeviceService, type DeviceScreenshot } from "../device/DeviceService.ts"; +import type { McpInvocationScope } from "./McpSessionRegistry.ts"; + +const shellWord = (value: string) => + /^[a-zA-Z0-9_./:-]+$/.test(value) ? value : "'" + value.replaceAll("'", "'\"'\"'") + "'"; + +/** How to drive the device with its bound launcher: returned by `device_open`, never kept in a prompt. */ +export function agentDeviceQuickStart(device: DeviceSummary, command: string): string { + const executable = shellWord(command); + const run = (args: string) => `${executable} ${args}`; + return [ + `The user is watching ${device.name} (${device.version}) in the Device tab.`, + `Drive it with ${executable}. It is bound to this device and session, so it needs no --platform, --udid, --serial, --config or --session flags. Use this exact path; login shells may reset PATH.`, + // Screenshots and input need the session `open` starts, and an iOS snapshot also needs its app. + // device_open does not run open itself: on iOS that can start a minutes-long XCTest runner build. + "Every app launch, deep link, tap, text entry, alert, setting, screenshot and recording on this device goes through it. Run open (apps lists them) before snapshots, CLI screenshots or input; it starts the session they need. Use device_screenshot just to look.", + ` ${run("open --relaunch")} # restarts the app; Expo dev clients connect to their Metro, --metro-port picks it`, + ` ${run("open ")}`, + ` ${run("snapshot -i")} # accessibility tree with @eN refs`, + ` ${run("press @e3 --settle")}`, + ` ${run('fill @e5 "text" --settle')}`, + ` ${run("alert accept")}`, + ` ${run("settings appearance dark")} # also text-size, permission grant `, + ` ${run("screenshot /tmp/shot.png")} # ${device.platform === "ios" ? "iOS saves 1x points; add --pixel-density (3 on most iPhones) for device resolution, " : ""}or call device_screenshot to look`, + ` ${run("record start --scope device /tmp/run.mp4")} # then record stop`, + ` ${run("install ")}`, + `Prefer snapshot refs over coordinates. Run ${executable} help for flags and ${executable} help workflow for guides.`, + "Do not use simctl, idb, adb or ssh for those actions. They stay fine for builds, installs, code sync, Metro, port forwards, logs, test runners and what this command cannot do; come back to it for every UI action and screenshot so the user can watch.", + "Each device_open returns its own command; opening another device does not retarget this one.", + "The command is removed when the device closes or the thread is put away. If it is missing, call device_open again.", + device.platform === "ios" + ? "First use builds an XCTest runner and can take a couple of minutes; later commands are fast." + : "The Android snapshot helper installs itself on first use.", + ].join("\n"); +} + +const NO_ACCESS = + "Agents can't use devices on this computer right now. Ask the user to turn on Use devices and Agents can use devices in Settings → Devices; it applies to agent sessions started after that, such as in a new thread."; + +/** The caller may use the device tools: its credential has the grant and the settings still allow it. */ +const requireAccess = (caller: McpInvocationScope) => + Effect.gen(function* () { + if (!caller.device) return yield* new DeviceToolUnavailableError({ reason: NO_ACCESS }); + const state = yield* (yield* DeviceService).state; + if (!state.enabled || !state.agentAccessEnabled) { + return yield* new DeviceToolUnavailableError({ reason: NO_ACCESS }); + } + return state; + }); + +/** The device an agent means: the one it named, else the booted or first one of its platform. */ +export const pickDevice = ( + state: Pick, + input: { + readonly deviceId?: DeviceId | undefined; + readonly platform?: DevicePlatform | undefined; + }, +): Effect.Effect => { + if (input.deviceId !== undefined) { + const found = state.devices.find((device) => device.id === input.deviceId); + if (found) return Effect.succeed(found); + return Effect.fail( + new DeviceToolUnavailableError({ + reason: [ + `No device ${input.deviceId} on this computer.`, + ...(state.hostStatus === "failed" && state.hostStatusDetail + ? [`Device support failed to start: ${state.hostStatusDetail}`] + : []), + "Call device_list for current ids.", + ].join(" "), + }), + ); + } + const candidates = state.devices.filter( + (device) => input.platform === undefined || device.platform === input.platform, + ); + if (candidates.length === 0) { + return Effect.fail( + new DeviceToolUnavailableError({ + reason: + state.hostStatus === "failed" && state.hostStatusDetail + ? state.hostStatusDetail + : input.platform === undefined + ? "No simulators or emulators were found. Call device_list to see why." + : `No ${input.platform === "ios" ? "iOS" : "Android"} devices were found. Call device_list to see why.`, + }), + ); + } + if ( + input.platform === undefined && + new Set(candidates.map((device) => device.platform)).size > 1 + ) { + return Effect.fail( + new DeviceToolUnavailableError({ + reason: "Both iOS and Android devices are available; pass platform or deviceId.", + }), + ); + } + return Effect.succeed(candidates.find((device) => device.booted) ?? candidates[0]!); +}; + +const toolError = (error: DeviceError | DeviceToolUnavailableError) => error; + +export const deviceList = ( + caller: McpInvocationScope, + input: { readonly refresh?: boolean | undefined }, +) => + Effect.gen(function* () { + yield* requireAccess(caller); + const devices = yield* DeviceService; + const state = yield* devices.list({ refresh: input.refresh !== false }); + return { + hostStatus: state.hostStatus, + ...(state.hostStatusDetail ? { hostStatusDetail: state.hostStatusDetail } : {}), + platforms: state.platforms, + devices: state.devices, + open: state.sessions + .filter((session) => session.threadId === caller.threadId) + .map((session) => ({ deviceId: session.deviceId })), + } satisfies DeviceToolListResult; + }).pipe(Effect.mapError(toolError)); + +export const deviceOpen = (caller: McpInvocationScope, input: DeviceToolOpenInput) => + Effect.gen(function* () { + yield* requireAccess(caller); + const devices = yield* DeviceService; + const listed = yield* devices.list({ refresh: true }); + const target = yield* pickDevice(listed, input); + const opened = yield* devices.agentOpen({ + threadId: caller.threadId, + deviceId: target.id, + platform: target.platform, + }); + return { + device: opened.device, + agentDevice: { command: opened.command, session: opened.agentSession }, + quickStart: agentDeviceQuickStart(opened.device, opened.command), + } satisfies DeviceToolOpenResult; + }).pipe(Effect.mapError(toolError)); + +/** The session a screenshot or close means: the one named, else the last one this thread opened. */ +const sessionFor = (threadId: ThreadId, deviceId: DeviceId | undefined) => + Effect.gen(function* () { + const sessions = yield* (yield* DeviceService).sessionsForThread(threadId); + const session = + deviceId === undefined + ? sessions.at(-1) + : sessions.find((candidate) => candidate.deviceId === deviceId); + if (!session) { + return yield* new DeviceToolUnavailableError({ + reason: + deviceId === undefined + ? "No device is open in this thread. Call device_open first." + : `Device ${deviceId} is not open in this thread. Call device_open first.`, + }); + } + return session; + }); + +export const deviceScreenshot = ( + caller: McpInvocationScope, + input: DeviceToolTargetInput, +): Effect.Effect => + Effect.gen(function* () { + yield* requireAccess(caller); + const session = yield* sessionFor(caller.threadId, input.deviceId); + return yield* (yield* DeviceService).screenshot(session.deviceId); + }).pipe(Effect.mapError(toolError)); + +export const deviceClose = (caller: McpInvocationScope, input: DeviceToolCloseInput) => + Effect.gen(function* () { + yield* requireAccess(caller); + const devices = yield* DeviceService; + if (input.deviceId !== undefined) yield* sessionFor(caller.threadId, input.deviceId); + yield* devices.close({ + threadId: caller.threadId, + ...(input.deviceId === undefined ? {} : { deviceId: input.deviceId }), + ...(input.shutdown === undefined ? {} : { shutdown: input.shutdown }), + }); + return {}; + }).pipe(Effect.mapError(toolError)); diff --git a/apps/server/src/mcp/deviceTools.ts b/apps/server/src/mcp/deviceTools.ts new file mode 100644 index 000000000..a1ac01e2c --- /dev/null +++ b/apps/server/src/mcp/deviceTools.ts @@ -0,0 +1,93 @@ +/** + * The device tools (docs/design/devices.md), served at `/mcp/device` by + * McpDeviceServer: list, open, screenshot and close a simulator or emulator + * on this server's machine. Driving the device happens through the + * `agent-device` launcher `device_open` returns, not through more tools. + * + * Descriptions from T3 Code's device toolkit, without its host selection: a + * Threadlines server has one device host, its own machine. + */ +import { + DeviceToolCloseInput, + DeviceToolListResult, + DeviceToolOpenInput, + DeviceToolOpenResult, + DeviceToolScreenshotResult, + DeviceToolTargetInput, +} from "@threadlines/contracts"; +import * as Schema from "effect/Schema"; +import { Tool } from "effect/ai"; + +export const DEVICE_MCP_SERVER_NAME = "threadlines_device"; + +export const DEVICE_LIST_TOOL_NAME = "device_list"; +export const DEVICE_OPEN_TOOL_NAME = "device_open"; +export const DEVICE_SCREENSHOT_TOOL_NAME = "device_screenshot"; +export const DEVICE_CLOSE_TOOL_NAME = "device_close"; + +/** The tools as Claude names them, for its allow list. */ +export const CLAUDE_DEVICE_TOOL_IDS = [ + DEVICE_LIST_TOOL_NAME, + DEVICE_OPEN_TOOL_NAME, + DEVICE_SCREENSHOT_TOOL_NAME, + DEVICE_CLOSE_TOOL_NAME, +].map((name) => `mcp__${DEVICE_MCP_SERVER_NAME}__${name}`); + +// An empty struct serializes as `anyOf [object, array]`, which some providers +// reject and then drop every tool on the server with it. +export const DeviceToolListInput = Schema.Struct({ + refresh: Schema.optional( + Schema.Boolean.annotate({ + description: "Ask the device hub again instead of answering from the last listing.", + }), + ), +}); + +export const DeviceListTool = Tool.make(DEVICE_LIST_TOOL_NAME, { + description: + "List iOS Simulators and Android Emulators on this computer, what it can run, and what is open in this thread's Device tab. Call before `device_open` when you do not know a device id.", + parameters: DeviceToolListInput, + success: DeviceToolListResult, +}) + .annotate(Tool.Title, "List devices") + .annotate(Tool.Readonly, true) + .annotate(Tool.Destructive, false) + .annotate(Tool.Idempotent, true) + .annotate(Tool.OpenWorld, false); + +export const DeviceOpenTool = Tool.make(DEVICE_OPEN_TOOL_NAME, { + description: + "Open a simulator or emulator for this thread. Pass `deviceId` from `device_list`. Boots it if needed and shows it in the user's Device tab. Returns the `agent-device` command bound to that device. Drive the device with that command.", + parameters: DeviceToolOpenInput, + success: DeviceToolOpenResult, +}) + .annotate(Tool.Title, "Open device") + .annotate(Tool.Readonly, false) + .annotate(Tool.Destructive, false) + .annotate(Tool.Idempotent, true) + .annotate(Tool.OpenWorld, true); + +export const DeviceScreenshotTool = Tool.make(DEVICE_SCREENSHOT_TOOL_NAME, { + description: + "Get a PNG of an open device's screen. For taps and text, use the `agent-device` command.", + parameters: DeviceToolTargetInput, + success: DeviceToolScreenshotResult, +}) + .annotate(Tool.Title, "Screenshot device") + .annotate(Tool.Readonly, true) + .annotate(Tool.Destructive, false) + .annotate(Tool.Idempotent, true) + .annotate(Tool.OpenWorld, true); + +export const DeviceCloseTool = Tool.make(DEVICE_CLOSE_TOOL_NAME, { + description: "Remove a device from this thread's Device tab. `shutdown=true` also turns it off.", + parameters: DeviceToolCloseInput, + success: Schema.Struct({}), +}) + .annotate(Tool.Title, "Close device") + .annotate(Tool.Readonly, false) + .annotate(Tool.Destructive, true) + .annotate(Tool.Idempotent, true) + .annotate(Tool.OpenWorld, true); + +export const DEVICE_TOOLS = [DeviceListTool, DeviceOpenTool, DeviceScreenshotTool, DeviceCloseTool]; diff --git a/apps/server/src/mcp/projectNoteToolHandlers.test.ts b/apps/server/src/mcp/projectNoteToolHandlers.test.ts index 5088292ca..b3d61de41 100644 --- a/apps/server/src/mcp/projectNoteToolHandlers.test.ts +++ b/apps/server/src/mcp/projectNoteToolHandlers.test.ts @@ -104,6 +104,7 @@ const caller = (thread: OrchestrationThread, side?: "ask"): McpInvocationScope = browser: true, pages: true, commands: false, + device: false, roomTools: new Set(roomToolsFor(side)), ...(side !== undefined ? { side: { sideTurnId: SideTurnId.make("side-1"), kind: side } } : {}), }); diff --git a/apps/server/src/mcp/roomToolHandlers.test.ts b/apps/server/src/mcp/roomToolHandlers.test.ts index d55da1769..eda0aa0c8 100644 --- a/apps/server/src/mcp/roomToolHandlers.test.ts +++ b/apps/server/src/mcp/roomToolHandlers.test.ts @@ -102,6 +102,7 @@ const mainCaller = ( browser: true, pages: true, commands: false, + device: false, roomTools: new Set(tools), }); diff --git a/apps/server/src/mcp/secretRequestToolHandlers.test.ts b/apps/server/src/mcp/secretRequestToolHandlers.test.ts index d69220576..53f291615 100644 --- a/apps/server/src/mcp/secretRequestToolHandlers.test.ts +++ b/apps/server/src/mcp/secretRequestToolHandlers.test.ts @@ -39,6 +39,7 @@ const mainCaller = (): McpInvocationScope => ({ browser: true, pages: true, commands: false, + device: false, roomTools: new Set(roomToolsFor(undefined)), }); diff --git a/apps/server/src/mcp/settingsChangeToolHandlers.test.ts b/apps/server/src/mcp/settingsChangeToolHandlers.test.ts index 4c1a90e85..a582fac0f 100644 --- a/apps/server/src/mcp/settingsChangeToolHandlers.test.ts +++ b/apps/server/src/mcp/settingsChangeToolHandlers.test.ts @@ -52,6 +52,7 @@ const mainCaller = (): McpInvocationScope => ({ browser: true, pages: true, commands: false, + device: false, roomTools: new Set(roomToolsFor(undefined)), }); diff --git a/apps/server/src/mcp/threadToolHandlers.test.ts b/apps/server/src/mcp/threadToolHandlers.test.ts index 4942e632f..944e67147 100644 --- a/apps/server/src/mcp/threadToolHandlers.test.ts +++ b/apps/server/src/mcp/threadToolHandlers.test.ts @@ -192,6 +192,7 @@ const mainCaller = (): McpInvocationScope => ({ browser: true, pages: true, commands: false, + device: false, roomTools: new Set(roomToolsFor(undefined)), }); diff --git a/apps/server/src/pages/AgentPages.test.ts b/apps/server/src/pages/AgentPages.test.ts index 6ef53377f..05d5b9383 100644 --- a/apps/server/src/pages/AgentPages.test.ts +++ b/apps/server/src/pages/AgentPages.test.ts @@ -53,6 +53,7 @@ const caller = (overrides: Partial = {}): McpInvocationScope browser: true, pages: true, commands: false, + device: false, roomTools: new Set(), ...overrides, }); diff --git a/apps/server/src/provider/acpRegistry/AcpRegistryInstaller.ts b/apps/server/src/provider/acpRegistry/AcpRegistryInstaller.ts index dc854c41e..cd0f0d89c 100644 --- a/apps/server/src/provider/acpRegistry/AcpRegistryInstaller.ts +++ b/apps/server/src/provider/acpRegistry/AcpRegistryInstaller.ts @@ -263,7 +263,7 @@ const SHIM_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u; * configuration, say). * - `update-notifier`: npm's last lines are about this install. */ -const npmPinnedFlags = (ignoreScripts: boolean) => [ +export const npmPinnedFlags = (ignoreScripts: boolean) => [ "--global=false", "--package-lock=true", "--no-audit", diff --git a/apps/server/src/scheduled/ScheduledTasks.test.ts b/apps/server/src/scheduled/ScheduledTasks.test.ts index 64941f90f..3960b89e0 100644 --- a/apps/server/src/scheduled/ScheduledTasks.test.ts +++ b/apps/server/src/scheduled/ScheduledTasks.test.ts @@ -1346,6 +1346,7 @@ it.layer(NodeServices.layer)("schedule tools", (it) => { browser: true, pages: false, commands: false, + device: false, roomTools: new Set(roomToolsFor(undefined, { scheduledRun })), }); diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 1aecc2894..8308e51fb 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -184,6 +184,8 @@ import * as PreviewTunnel from "./preview/PreviewTunnel.ts"; import { AgentPages } from "./pages/AgentPages.ts"; import { McpApps } from "./mcpApps/McpApps.ts"; import { CommandRuns } from "./commandRuns/CommandRuns.ts"; +import { DeviceChannels } from "./device/DeviceChannels.ts"; +import { DeviceService } from "./device/DeviceService.ts"; import { AgentSettingsChanges } from "./settingsChanges/AgentSettingsChanges.ts"; import { AgentSecretRequests } from "./secrets/AgentSecretRequests.ts"; import { layer as SecretScrubberLive } from "./secrets/SecretScrubber.ts"; @@ -798,6 +800,9 @@ const buildAppUnderTest = (options?: { Layer.mock(AgentSettingsChanges)({}), // So are agents' requests for keys (AgentSecretRequests.test.ts). Layer.mock(AgentSecretRequests)({ destinations: {} as never }), + // Devices are tested on their own (device/*.test.ts); the routes only need them built. + Layer.mock(DeviceService)({}), + Layer.mock(DeviceChannels)({ releaseConnection: () => Effect.void }), // The watched keys' registry, empty: the routes' doors read it. SecretScrubberLive, Layer.mock(CommandRuns)({ diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 66f28cc7e..2ca612a9c 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -107,6 +107,10 @@ import * as McpHttpServer from "./mcp/McpHttpServer.ts"; import * as McpRoomServer from "./mcp/McpRoomServer.ts"; import * as McpPagesServer from "./mcp/McpPagesServer.ts"; import * as McpCommandsServer from "./mcp/McpCommandsServer.ts"; +import * as McpDeviceServer from "./mcp/McpDeviceServer.ts"; +import * as DeviceChannels from "./device/DeviceChannels.ts"; +import * as DeviceService from "./device/DeviceService.ts"; +import * as DeviceThreadReactor from "./device/DeviceThreadReactor.ts"; import * as McpConnectServer from "./outsideAgents/McpConnectServer.ts"; import { AssistantMcpServerLive } from "./assistants/AssistantMcpServer.ts"; import { OtherComputersLive } from "./outsideAgents/OtherComputers.ts"; @@ -632,6 +636,14 @@ const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe( // Answers to agents' requests for keys; it settles at startup what a // previous run left mid-save and sets saved files' expiry timers. AgentSecretRequests.layer.pipe(Layer.provide(ProcessRunnerLive)), + // Simulators and emulators (docs/design/devices.md): the service, the + // Device tab's channels to the hub, and the reactor that lets a thread + // put away let go of its devices (and sweeps claims at startup). + DeviceThreadReactor.layer.pipe( + Layer.provideMerge(DeviceChannels.layer), + Layer.provideMerge(DeviceService.layer), + Layer.provide(ProcessRunnerLive), + ), // The calls' table also serves thread deletion's cleanup. McpApps.layer.pipe(Layer.provideMerge(McpAppCallRepositoryLive)), // Outside agents' starts and sends by request id (`/mcp/connect`); @@ -773,6 +785,7 @@ export const makeRoutesLayer = Layer.mergeAll( McpRoomServer.layer, McpPagesServer.layer, McpCommandsServer.layer, + McpDeviceServer.layer, // Outside agents' tools (docs/design/outside-agents.md). McpConnectServer.layer, // The same tools for assistants, over the relay (docs/design/assistants.md). diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 58ebac1e5..3fad00147 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -70,6 +70,7 @@ import { WsMovesRpcGroup, WsThemeShowcaseRpcGroup, WsThemeVsCodeRpcGroup, + WsDeviceRpcGroup, WsThemesRpcGroup, WsOwnerRequiredError, OutsideAgentError, @@ -119,6 +120,8 @@ import { import { LocalServers } from "./preview/LocalServers.ts"; import { PreviewAutomationBroker } from "./preview/PreviewAutomationBroker.ts"; import { PreviewTunnel } from "./preview/PreviewTunnel.ts"; +import { DeviceChannels } from "./device/DeviceChannels.ts"; +import { DeviceService } from "./device/DeviceService.ts"; import { ProjectFaviconResolver } from "./project/Services/ProjectFaviconResolver.ts"; import { PullRequestService } from "./pullRequest/PullRequestService.ts"; import { PullRequestObservations } from "./pullRequest/PullRequestObservations.ts"; @@ -833,6 +836,51 @@ const ThemeVsCodeRpcLayer = WsThemeVsCodeRpcGroup.toLayer( }), ); +/** + * Devices (docs/design/devices.md): its own RPC group and layer + * (WsDeviceRpcGroup), merged in makeWsRpcLayer. Open to every paired device, + * as terminals are. Channels belong to this connection (`viewerId`) and close + * with it. + */ +const makeDeviceRpcLayer = (viewerId: string) => + WsDeviceRpcGroup.toLayer( + Effect.gen(function* () { + const devices = yield* DeviceService; + const channels = yield* DeviceChannels; + const aggregate = { "rpc.aggregate": "devices" } as const; + return WsDeviceRpcGroup.of({ + [WS_METHODS.deviceList]: (input) => + observeRpcEffect(WS_METHODS.deviceList, devices.list(input), aggregate), + [WS_METHODS.deviceUpdateTool]: (input) => + observeRpcEffect(WS_METHODS.deviceUpdateTool, devices.updateTool(input.tool), aggregate), + [WS_METHODS.deviceOpen]: (input) => + observeRpcEffect(WS_METHODS.deviceOpen, devices.open(input, "user"), aggregate), + [WS_METHODS.deviceClose]: (input) => + observeRpcEffect(WS_METHODS.deviceClose, devices.close(input), aggregate), + [WS_METHODS.deviceShutdown]: (input) => + observeRpcEffect(WS_METHODS.deviceShutdown, devices.shutdown(input), aggregate), + [WS_METHODS.deviceDetail]: (input) => + observeRpcEffect(WS_METHODS.deviceDetail, devices.detail(input), aggregate), + [WS_METHODS.deviceAction]: (input) => + observeRpcEffect(WS_METHODS.deviceAction, devices.action(input), aggregate), + [WS_METHODS.subscribeDeviceState]: (_input) => + observeRpcStream(WS_METHODS.subscribeDeviceState, devices.stream, aggregate), + [WS_METHODS.deviceChannelOpen]: (input) => + // Scoped to the subscription: the client letting go, or its socket + // dropping, closes the request or socket to the hub too. + observeRpcStreamEffect( + WS_METHODS.deviceChannelOpen, + channels.open(viewerId, input), + aggregate, + ), + // A channel's messages and credits are its traffic itself: left out of + // the RPC metrics and traces they would drown. + [WS_METHODS.deviceChannelSend]: (input) => channels.send(viewerId, input), + [WS_METHODS.deviceChannelCredit]: (input) => channels.credit(viewerId, input), + }); + }), + ); + /** Every RPC the WebSocket serves, for one connection. */ const makeWsRpcLayer = ( currentSession: { @@ -852,6 +900,7 @@ const makeWsRpcLayer = ( ThemeShowcaseRpcLayer, MovesRpcLayer, ThemeVsCodeRpcLayer, + makeDeviceRpcLayer(viewerId), ); const makeCoreRpcLayer = ( @@ -3675,6 +3724,7 @@ export const websocketRpcRouteLayer = Layer.unwrap( const threadBootstrap = yield* ThreadBootstrap; const terminalManager = yield* TerminalManager; const pushAlerts = yield* PushAlerts; + const deviceChannels = yield* DeviceChannels; return HttpRouter.add( "GET", "/ws", @@ -3770,6 +3820,7 @@ export const websocketRpcRouteLayer = Layer.unwrap( .pipe( Effect.andThen(terminalManager.releaseViewer(viewerId)), Effect.andThen(pushAlerts.releaseViewer(viewerId)), + Effect.andThen(deviceChannels.releaseConnection(viewerId)), ), ); }).pipe(Effect.scoped, Effect.catchTag("AuthError", respondToAuthError)), From b4b8a2e0fb59534bd13f82bf9f5bd581c3d5db29 Mon Sep 17 00:00:00 2001 From: Badcuban <108198679+badcuban@users.noreply.github.com> Date: Fri, 20 Nov 2026 01:23:11 -0500 Subject: [PATCH 04/49] feat(server): device tools reach every provider Sessions started with deviceTools (Use devices and Agents can use devices on, thread not fenced) get the threadlines_device server in Codex, Claude, ACP, Pi, Muse and OpenCode, plus a short instruction block everywhere but ACP. Nothing changes when the settings are off. --- .../Layers/ProviderCommandReactor.ts | 8 ++++++ .../src/provider/Layers/ClaudeAdapter.ts | 26 +++++++++++++++++++ .../src/provider/Layers/CodexAdapter.ts | 1 + .../provider/Layers/CodexSessionRuntime.ts | 22 ++++++++++++++++ .../server/src/provider/Layers/MuseAdapter.ts | 11 ++++++++ .../src/provider/Layers/OpenCodeAdapter.ts | 19 ++++++++++++++ apps/server/src/provider/Layers/PiAdapter.ts | 14 ++++++++++ apps/server/src/provider/acp/AcpAdapter.ts | 15 +++++++++++ .../src/provider/agentContextBudget.json | 2 +- .../src/provider/agentContextBudget.test.ts | 3 ++- .../server/src/provider/codexAppServerArgs.ts | 12 ++++++--- .../server/src/provider/deviceInstructions.ts | 26 +++++++++++++++++++ .../provider/opencode/OpenCodeRules.test.ts | 19 ++++++++++++++ .../src/provider/opencode/OpenCodeRules.ts | 17 ++++++++++++ .../provider/opencode/OpenCodeToolItems.ts | 10 ++++--- 15 files changed, 196 insertions(+), 9 deletions(-) create mode 100644 apps/server/src/provider/deviceInstructions.ts diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts index a2d41311c..dff8c246c 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts @@ -1722,6 +1722,14 @@ const make = Effect.gen(function* () { ? { artifacts: true } : {}), ...(toolSettings?.enableCommandRequests !== false ? { commandRequests: true } : {}), + // "Use devices" and "Agents can use devices", both off by default + // (docs/design/devices.md). Drivers leave them off a side runtime, and a + // fenced thread gets none: they act on this machine's simulators. + ...(!fencedOff && + toolSettings?.enableDevices === true && + toolSettings.agentDeviceAccess === true + ? { deviceTools: true } + : {}), runtimeMode: desiredRuntimeMode, }); diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.ts b/apps/server/src/provider/Layers/ClaudeAdapter.ts index 89d4ece63..3357298d5 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.ts @@ -159,6 +159,7 @@ import { ServerConfig } from "../../config.ts"; import { BROWSER_MCP_SERVER_NAME, mcpCommandsEndpointUrl, + mcpDeviceEndpointUrl, mcpEndpointUrl, mcpPagesEndpointUrl, mcpRoomEndpointUrl, @@ -168,6 +169,8 @@ import { CLAUDE_COMMAND_TOOL_IDS, COMMANDS_MCP_SERVER_NAME, } from "../../mcp/commandTools.ts"; +import { CLAUDE_DEVICE_TOOL_IDS, DEVICE_MCP_SERVER_NAME } from "../../mcp/deviceTools.ts"; +import { buildDeviceInstructions } from "../deviceInstructions.ts"; import { mcpSessionRegistry } from "../../mcp/McpSessionRegistry.ts"; import { outsideWorkLookup } from "../outsideWorkLookup.ts"; import { ROOM_TOOL_CALL_TIMEOUT } from "../../mcp/roomToolTimeouts.ts"; @@ -1008,6 +1011,8 @@ export function buildClaudeSystemPromptAppend(input: { readonly pageAssetsDir: string | undefined; readonly artifacts: boolean; readonly commandRequests: boolean; + /** The device tools (ProviderSessionStartInput.deviceTools). */ + readonly deviceTools: boolean; /** The room endpoint's tools this session may call, when it is attached. */ readonly room: | { readonly tools: ReadonlyArray; readonly schedules: boolean } @@ -1031,6 +1036,7 @@ export function buildClaudeSystemPromptAppend(input: { : []), ...(input.artifacts ? [CLAUDE_ARTIFACT_INSTRUCTIONS] : []), ...(input.commandRequests ? [buildCommandRequestInstructions({})] : []), + ...(input.deviceTools ? [buildDeviceInstructions()] : []), ...(input.room?.schedules === true ? [buildScheduleInstructions({ toolPrefix: roomPrefix, claude: true })] : []), @@ -9613,6 +9619,8 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( ); // `ask_user_to_run`: a command card the user runs from the chat. const commandRequests = input.commandRequests === true && !lockdown; + // The device tools: simulators and emulators on this computer. + const deviceTools = input.deviceTools === true && !lockdown; const credential = !lockdown || roomGrant.attach ? yield* mcpSessionRegistry.credentialFor({ @@ -9621,6 +9629,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( room: roomGrant.attach, pages: agentPages, commands: commandRequests, + device: deviceTools, sideKind, scheduledRun: input.scheduledRunTools === true, guideOnly: roomGrant.guideOnly, @@ -9687,6 +9696,19 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( }, } : {}), + ...(credential !== undefined && deviceTools + ? { + [DEVICE_MCP_SERVER_NAME]: { + type: "http" as const, + url: mcpDeviceEndpointUrl(serverConfig.port), + headers: { Authorization: `Bearer ${credential.token}` }, + // Like the others: Claude Code waits for the server before + // the first turn, and each tool's own mark decides whether + // it is in the prompt (toolLoading.ts). + alwaysLoad: true, + }, + } + : {}), }, ...(apiModelId ? { model: apiModelId } : {}), pathToClaudeCodeExecutable: claudeBinaryPath, @@ -9698,6 +9720,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( pageAssetsDir: agentPages ? pageAssets : undefined, artifacts, commandRequests: credential !== undefined && commandRequests, + deviceTools: credential !== undefined && deviceTools, room: roomServer !== undefined ? { @@ -9739,6 +9762,9 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( ...(credential !== undefined && agentPages ? CLAUDE_PAGE_TOOL_IDS : []), // Asking the user to run a command only shows a card; they decide. ...(credential !== undefined && commandRequests ? CLAUDE_COMMAND_TOOL_IDS : []), + // The user turned on "Agents can use devices"; driving one goes + // through the agent-device command, which asks like any command. + ...(credential !== undefined && deviceTools ? CLAUDE_DEVICE_TOOL_IDS : []), ], ...(effectiveEffort ? { diff --git a/apps/server/src/provider/Layers/CodexAdapter.ts b/apps/server/src/provider/Layers/CodexAdapter.ts index f5b587833..1db5ccff1 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.ts @@ -3437,6 +3437,7 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( ...(input.commandRequests === true && sideAnswer === undefined ? { commandRequests: true } : {}), + ...(input.deviceTools === true && sideAnswer === undefined ? { deviceTools: true } : {}), runtimeMode: input.runtimeMode, ...(input.modelSelection?.instanceId === boundInstanceId ? { model: input.modelSelection.model } diff --git a/apps/server/src/provider/Layers/CodexSessionRuntime.ts b/apps/server/src/provider/Layers/CodexSessionRuntime.ts index cf86efda2..237e2f35d 100644 --- a/apps/server/src/provider/Layers/CodexSessionRuntime.ts +++ b/apps/server/src/provider/Layers/CodexSessionRuntime.ts @@ -2,6 +2,7 @@ import { parseSessionKey } from "@threadlines/shared/threadParticipants"; import { BROWSER_MCP_SERVER_NAME, mcpCommandsEndpointUrl, + mcpDeviceEndpointUrl, mcpEndpointUrl, mcpPagesEndpointUrl, mcpRoomEndpointUrl, @@ -10,6 +11,8 @@ import { buildCommandRequestInstructions, COMMANDS_MCP_SERVER_NAME, } from "../../mcp/commandTools.ts"; +import { DEVICE_MCP_SERVER_NAME } from "../../mcp/deviceTools.ts"; +import { buildDeviceInstructions } from "../deviceInstructions.ts"; import { buildAgentPageInstructions, CODEX_PAGE_RIVAL, @@ -255,6 +258,8 @@ export interface CodexSessionRuntimeOptions { readonly agentPages?: boolean; /** Attach `ask_user_to_run` (ProviderSessionStartInput.commandRequests). Never for a side runtime. */ readonly commandRequests?: boolean; + /** Attach the device tools (ProviderSessionStartInput.deviceTools). Never for a side runtime. */ + readonly deviceTools?: boolean; /** * The project's brief (ProviderSessionStartInput.projectInstructions). The * provider thread keeps it as its developer instructions, set at every @@ -748,6 +753,8 @@ function buildCodexCollaborationMode(input: { readonly agentPageAssetsDir?: string; /** The session has `ask_user_to_run`. */ readonly commandRequests?: boolean; + /** The session has the device tools. */ + readonly deviceTools?: boolean; /** The thread's own runtime has the room tools, schedule tools among them. */ readonly scheduleTools?: boolean; /** The thread's own runtime has the room endpoint, so the app guide. */ @@ -779,6 +786,7 @@ function buildCodexCollaborationMode(input: { ] : []), ...(input.commandRequests ? [buildCommandRequestInstructions({})] : []), + ...(input.deviceTools ? [buildDeviceInstructions()] : []), ...(input.scheduleTools ? [buildScheduleInstructions({ toolPrefix: `mcp__${ROOM_MCP_SERVER_NAME}__` })] : []), @@ -809,6 +817,8 @@ export function buildTurnStartParams(input: { readonly agentPageAssetsDir?: string; /** The session has `ask_user_to_run`. */ readonly commandRequests?: boolean; + /** The session has the device tools. */ + readonly deviceTools?: boolean; /** The thread's own runtime has the room tools, schedule tools among them. */ readonly scheduleTools?: boolean; /** The thread's own runtime has the room endpoint, so the app guide. */ @@ -845,6 +855,7 @@ export function buildTurnStartParams(input: { ? { agentPageAssetsDir: input.agentPageAssetsDir } : {}), ...(input.commandRequests ? { commandRequests: true } : {}), + ...(input.deviceTools ? { deviceTools: true } : {}), ...(input.scheduleTools && !input.lockdown ? { scheduleTools: true } : {}), ...(input.guide && !input.lockdown ? { guide: true } : {}), }); @@ -2104,6 +2115,7 @@ const makeCodexSessionRuntimeIn = ( const roomGrant = roomEndpointGrant(options, lockdown?.kind); const agentPages = options.agentPages === true && lockdown === undefined; const commandRequests = options.commandRequests === true && lockdown === undefined; + const deviceTools = options.deviceTools === true && lockdown === undefined; const credential = lockdown === undefined || roomGrant.attach ? yield* mcpSessionRegistry.credentialFor({ @@ -2112,6 +2124,7 @@ const makeCodexSessionRuntimeIn = ( room: roomGrant.attach, pages: agentPages, commands: commandRequests, + device: deviceTools, ...(lockdown !== undefined ? { sideKind: lockdown.kind } : {}), scheduledRun: options.scheduledRunTools === true, guideOnly: roomGrant.guideOnly, @@ -2162,6 +2175,14 @@ const makeCodexSessionRuntimeIn = ( }, } : {}), + ...(deviceTools + ? { + device: { + url: mcpDeviceEndpointUrl(options.serverPort), + serverName: DEVICE_MCP_SERVER_NAME, + }, + } + : {}), }), env, ); @@ -3123,6 +3144,7 @@ const makeCodexSessionRuntimeIn = ( ...((yield* runsInManagedWorktree) ? { managedWorktree: true } : {}), ...(agentPageAssetsDir !== undefined ? { agentPageAssetsDir } : {}), ...(commandRequests ? { commandRequests: true } : {}), + ...(deviceTools ? { deviceTools: true } : {}), ...(roomTools && options.scheduleTools === true && lockdown === undefined ? { scheduleTools: true } : {}), diff --git a/apps/server/src/provider/Layers/MuseAdapter.ts b/apps/server/src/provider/Layers/MuseAdapter.ts index a3fa55c39..921f07b68 100644 --- a/apps/server/src/provider/Layers/MuseAdapter.ts +++ b/apps/server/src/provider/Layers/MuseAdapter.ts @@ -84,6 +84,7 @@ import { ServerConfig } from "../../config.ts"; import { BROWSER_MCP_SERVER_NAME, mcpCommandsEndpointUrl, + mcpDeviceEndpointUrl, mcpEndpointUrl, mcpPagesEndpointUrl, mcpRoomEndpointUrl, @@ -92,6 +93,8 @@ import { buildCommandRequestInstructions, COMMANDS_MCP_SERVER_NAME, } from "../../mcp/commandTools.ts"; +import { DEVICE_MCP_SERVER_NAME } from "../../mcp/deviceTools.ts"; +import { buildDeviceInstructions } from "../deviceInstructions.ts"; import { mcpSessionRegistry } from "../../mcp/McpSessionRegistry.ts"; import { buildAgentPageInstructions, PAGES_MCP_SERVER_NAME } from "../../mcp/pageTools.ts"; import { buildGuideInstructions } from "../../mcp/guideTools.ts"; @@ -314,6 +317,7 @@ interface SessionContext { readonly scheduleTools: boolean; readonly agentPageAssetsDir: string | undefined; readonly commandRequests: boolean; + readonly deviceTools: boolean; readonly managedWorktree: boolean; readonly mcpToken: string; museSessionId: string; @@ -1800,6 +1804,9 @@ export function makeMuseAdapter(options: MuseAdapterOptions) { ...(ctx.commandRequests ? { [COMMANDS_MCP_SERVER_NAME]: server(mcpCommandsEndpointUrl(serverConfig.port)) } : {}), + ...(ctx.deviceTools + ? { [DEVICE_MCP_SERVER_NAME]: server(mcpDeviceEndpointUrl(serverConfig.port)) } + : {}), }, }; }; @@ -1812,6 +1819,7 @@ export function makeMuseAdapter(options: MuseAdapterOptions) { : []), // Muse's own names for outside tools are not ours to guess; the bare name is unambiguous. ...(ctx.commandRequests ? [buildCommandRequestInstructions({ toolPrefix: "" })] : []), + ...(ctx.deviceTools ? [buildDeviceInstructions("")] : []), ...(ctx.scheduleTools ? [buildScheduleInstructions({ toolPrefix: "" })] : []), ...(ctx.roomEndpoint ? [buildGuideInstructions({ toolPrefix: "" })] : []), FILE_LINK_INSTRUCTIONS, @@ -2212,6 +2220,7 @@ export function makeMuseAdapter(options: MuseAdapterOptions) { const roomTools = input.roomTools === true; const roomGrant = roomEndpointGrant(input, undefined); const commandRequests = input.commandRequests === true; + const deviceTools = input.deviceTools === true; const agentPageAssetsDir = input.agentPages === true ? yield* ensurePageAssetsDir(parseSessionKey(input.threadId).threadId).pipe( @@ -2225,6 +2234,7 @@ export function makeMuseAdapter(options: MuseAdapterOptions) { room: roomGrant.attach, pages: agentPageAssetsDir !== undefined, commands: commandRequests, + device: deviceTools, scheduledRun: input.scheduledRunTools === true, guideOnly: roomGrant.guideOnly, }); @@ -2260,6 +2270,7 @@ export function makeMuseAdapter(options: MuseAdapterOptions) { scheduleTools: roomTools && input.scheduleTools === true, agentPageAssetsDir, commandRequests, + deviceTools, managedWorktree, mcpToken: credential.token, museSessionId: "", diff --git a/apps/server/src/provider/Layers/OpenCodeAdapter.ts b/apps/server/src/provider/Layers/OpenCodeAdapter.ts index 4b4043907..dd5c13243 100644 --- a/apps/server/src/provider/Layers/OpenCodeAdapter.ts +++ b/apps/server/src/provider/Layers/OpenCodeAdapter.ts @@ -74,11 +74,13 @@ import { ServerConfig } from "../../config.ts"; import { mcpEndpointUrl, mcpCommandsEndpointUrl, + mcpDeviceEndpointUrl, mcpPagesEndpointUrl, mcpRoomEndpointUrl, } from "../../mcp/McpHttpServer.ts"; import { buildAgentPageInstructions } from "../../mcp/pageTools.ts"; import { buildCommandRequestInstructions } from "../../mcp/commandTools.ts"; +import { buildDeviceInstructions } from "../deviceInstructions.ts"; import { buildGuideInstructions } from "../../mcp/guideTools.ts"; import { roomEndpointGrant } from "../../mcp/roomToolAccess.ts"; import { buildScheduleInstructions } from "../../mcp/scheduleTools.ts"; @@ -123,6 +125,7 @@ import { openCodeBrowserServerName, openCodeRoomServerName, openCodeCommandsServerName, + openCodeDeviceServerName, openCodePagesServerName, openCodeSessionGrantRules, openCodeSessionRules, @@ -260,6 +263,8 @@ interface SessionContext { readonly agentPageAssetsDir: string | undefined; /** The session has `ask_user_to_run`. */ readonly commandRequests: boolean; + /** The session has the device tools. */ + readonly deviceTools: boolean; /** The project's brief (ProviderSessionStartInput.projectInstructions), last in the instructions entry. */ readonly projectInstructions: string | undefined; readonly runtimeMode: RuntimeMode; @@ -600,6 +605,7 @@ export function makeOpenCodeAdapter(options: OpenCodeAdapterOptions) { roomTools: ctx.roomEndpoint, agentPages: ctx.agentPageAssetsDir !== undefined, commandRequests: ctx.commandRequests, + deviceTools: ctx.deviceTools, }); }); @@ -1993,6 +1999,14 @@ export function makeOpenCodeAdapter(options: OpenCodeAdapterOptions) { }, ] : []), + ...(ctx.deviceTools + ? [ + { + name: openCodeDeviceServerName(ctx.toolKey), + url: mcpDeviceEndpointUrl(serverConfig.port), + }, + ] + : []), ]; for (const server of servers) { yield* runOpenCode("mcp.add", (signal) => @@ -2044,6 +2058,9 @@ export function makeOpenCodeAdapter(options: OpenCodeAdapterOptions) { }), ] : []), + ...(ctx.deviceTools + ? [buildDeviceInstructions(`${openCodeDeviceServerName(ctx.toolKey)}_`)] + : []), ...(ctx.scheduleTools ? [buildScheduleInstructions({ toolPrefix: `${openCodeRoomServerName(ctx.toolKey)}_` })] : []), @@ -2294,6 +2311,7 @@ export function makeOpenCodeAdapter(options: OpenCodeAdapterOptions) { scheduleTools: roomTools && input.scheduleTools === true, agentPageAssetsDir, commandRequests: input.commandRequests === true, + deviceTools: input.deviceTools === true, projectInstructions: input.projectInstructions, runtimeMode: input.runtimeMode, agent: BUILD_AGENT, @@ -2335,6 +2353,7 @@ export function makeOpenCodeAdapter(options: OpenCodeAdapterOptions) { room: roomGrant.attach, pages: agentPageAssetsDir !== undefined, commands: input.commandRequests === true, + device: input.deviceTools === true, scheduledRun: input.scheduledRunTools === true, guideOnly: roomGrant.guideOnly, // OpenCode registers tool servers per folder: every session in diff --git a/apps/server/src/provider/Layers/PiAdapter.ts b/apps/server/src/provider/Layers/PiAdapter.ts index 8090cab0e..b390afaf7 100644 --- a/apps/server/src/provider/Layers/PiAdapter.ts +++ b/apps/server/src/provider/Layers/PiAdapter.ts @@ -78,6 +78,7 @@ import { ServerConfig } from "../../config.ts"; import { BROWSER_MCP_SERVER_NAME, mcpCommandsEndpointUrl, + mcpDeviceEndpointUrl, mcpEndpointUrl, mcpPagesEndpointUrl, mcpRoomEndpointUrl, @@ -86,6 +87,8 @@ import { buildCommandRequestInstructions, COMMANDS_MCP_SERVER_NAME, } from "../../mcp/commandTools.ts"; +import { DEVICE_MCP_SERVER_NAME } from "../../mcp/deviceTools.ts"; +import { buildDeviceInstructions } from "../deviceInstructions.ts"; import { mcpSessionRegistry } from "../../mcp/McpSessionRegistry.ts"; import { buildAgentPageInstructions, PAGES_MCP_SERVER_NAME } from "../../mcp/pageTools.ts"; import { ROOM_MCP_SERVER_NAME, roomEndpointGrant } from "../../mcp/roomToolAccess.ts"; @@ -308,6 +311,7 @@ interface SessionContext { readonly roomEndpoint: boolean; readonly agentPageAssetsDir: string | undefined; readonly commandRequests: boolean; + readonly deviceTools: boolean; readonly managedWorktree: boolean; readonly mcpToken: string; readonly launchArgs: ReadonlyArray; @@ -491,6 +495,7 @@ export function makePiAdapter(options: PiAdapterOptions) { const instructionsFor = (input: { readonly agentPageAssetsDir: string | undefined; readonly commandRequests: boolean; + readonly deviceTools: boolean; readonly scheduleTools: boolean; readonly roomEndpoint: boolean; readonly managedWorktree: boolean; @@ -503,6 +508,7 @@ export function makePiAdapter(options: PiAdapterOptions) { ...(input.commandRequests ? [buildCommandRequestInstructions({ toolPrefix: `mcp__${COMMANDS_MCP_SERVER_NAME}__` })] : []), + ...(input.deviceTools ? [buildDeviceInstructions(`mcp__${DEVICE_MCP_SERVER_NAME}__`)] : []), ...(input.scheduleTools ? [buildScheduleInstructions({ toolPrefix: `mcp__${ROOM_MCP_SERVER_NAME}__` })] : []), @@ -517,6 +523,7 @@ export function makePiAdapter(options: PiAdapterOptions) { readonly roomEndpoint: boolean; readonly agentPageAssetsDir: string | undefined; readonly commandRequests: boolean; + readonly deviceTools: boolean; }): ReadonlyArray => [ { name: BROWSER_MCP_SERVER_NAME, url: mcpEndpointUrl(serverConfig.port) }, ...(ctx.roomEndpoint @@ -528,6 +535,9 @@ export function makePiAdapter(options: PiAdapterOptions) { ...(ctx.commandRequests ? [{ name: COMMANDS_MCP_SERVER_NAME, url: mcpCommandsEndpointUrl(serverConfig.port) }] : []), + ...(ctx.deviceTools + ? [{ name: DEVICE_MCP_SERVER_NAME, url: mcpDeviceEndpointUrl(serverConfig.port) }] + : []), ]; // ── Turns ──────────────────────────────────────────────────────────── @@ -1734,6 +1744,7 @@ export function makePiAdapter(options: PiAdapterOptions) { const roomTools = input.roomTools === true; const roomGrant = roomEndpointGrant(input, undefined); const commandRequests = input.commandRequests === true; + const deviceTools = input.deviceTools === true; const agentPageAssetsDir = input.agentPages === true ? yield* ensurePageAssetsDir(parseSessionKey(input.threadId).threadId).pipe( @@ -1747,6 +1758,7 @@ export function makePiAdapter(options: PiAdapterOptions) { room: roomGrant.attach, pages: agentPageAssetsDir !== undefined, commands: commandRequests, + device: deviceTools, scheduledRun: input.scheduledRunTools === true, guideOnly: roomGrant.guideOnly, }); @@ -1763,6 +1775,7 @@ export function makePiAdapter(options: PiAdapterOptions) { instructionsFor({ agentPageAssetsDir, commandRequests, + deviceTools, scheduleTools: roomTools && input.scheduleTools === true, roomEndpoint: roomGrant.attach, managedWorktree, @@ -1808,6 +1821,7 @@ export function makePiAdapter(options: PiAdapterOptions) { roomEndpoint: roomGrant.attach, agentPageAssetsDir, commandRequests, + deviceTools, managedWorktree, mcpToken: credential.token, launchArgs: launchArgs.args, diff --git a/apps/server/src/provider/acp/AcpAdapter.ts b/apps/server/src/provider/acp/AcpAdapter.ts index d1caff225..3d5097d07 100644 --- a/apps/server/src/provider/acp/AcpAdapter.ts +++ b/apps/server/src/provider/acp/AcpAdapter.ts @@ -67,11 +67,13 @@ import { BROWSER_MCP_SERVER_NAME, mcpEndpointUrl, mcpCommandsEndpointUrl, + mcpDeviceEndpointUrl, mcpPagesEndpointUrl, mcpRoomEndpointUrl, } from "../../mcp/McpHttpServer.ts"; import { PAGES_MCP_SERVER_NAME } from "../../mcp/pageTools.ts"; import { COMMANDS_MCP_SERVER_NAME } from "../../mcp/commandTools.ts"; +import { DEVICE_MCP_SERVER_NAME } from "../../mcp/deviceTools.ts"; import { mcpSessionRegistry } from "../../mcp/McpSessionRegistry.ts"; import { ROOM_MCP_SERVER_NAME, @@ -1269,6 +1271,8 @@ export function makeAcpAdapter( const agentPagesRequested = input.agentPages === true && lockdown === undefined; // `ask_user_to_run` likewise: its description carries the rules. const commandRequestsRequested = input.commandRequests === true && lockdown === undefined; + // The device tools likewise. + const deviceToolsRequested = input.deviceTools === true && lockdown === undefined; const credential = reachesHost && (lockdown === undefined || roomGrant.attach) ? yield* mcpSessionRegistry.credentialFor({ @@ -1277,6 +1281,7 @@ export function makeAcpAdapter( room: roomGrant.attach, pages: agentPagesRequested, commands: commandRequestsRequested, + device: deviceToolsRequested, ...(sideKind !== undefined ? { sideKind } : {}), scheduledRun: lockdown === undefined && input.scheduledRunTools === true, guideOnly: roomGrant.guideOnly, @@ -1335,6 +1340,16 @@ export function makeAcpAdapter( }, ] : []), + ...(deviceToolsRequested + ? [ + { + type: "http" as const, + name: DEVICE_MCP_SERVER_NAME, + url: mcpDeviceEndpointUrl(serverConfig.port), + headers: authorization, + }, + ] + : []), ]; // A side runtime's process is the descriptor's locked-down one; what diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index a64b3e13c..b755bce5e 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { - "claudeSystemAppendBytes": 7234, + "claudeSystemAppendBytes": 7945, "upFrontTools": [ "app_guide", "ask_user_to_run", diff --git a/apps/server/src/provider/agentContextBudget.test.ts b/apps/server/src/provider/agentContextBudget.test.ts index 834b31413..ca8dea255 100644 --- a/apps/server/src/provider/agentContextBudget.test.ts +++ b/apps/server/src/provider/agentContextBudget.test.ts @@ -89,7 +89,7 @@ it("what a Claude thread is sent on every request matches the committed budget", const onDemand = threadTools.filter((tool) => !isLoadedUpFront(tool)); const measured = { // Every optional section on, as in a thread with pages, artifacts, - // command requests, schedules and the room tools, in a + // command requests, devices, schedules and the room tools, in a // Threadlines-managed worktree. claudeSystemAppendBytes: utf8Bytes( onLinux(() => @@ -97,6 +97,7 @@ it("what a Claude thread is sent on every request matches the committed budget", pageAssetsDir: "/tmp/threadlines-page-assets/thread", artifacts: true, commandRequests: true, + deviceTools: true, room: { tools: roomToolsFor(undefined), schedules: true }, lockdown: false, managedWorktree: true, diff --git a/apps/server/src/provider/codexAppServerArgs.ts b/apps/server/src/provider/codexAppServerArgs.ts index e6d1fb99e..c39db102f 100644 --- a/apps/server/src/provider/codexAppServerArgs.ts +++ b/apps/server/src/provider/codexAppServerArgs.ts @@ -122,6 +122,7 @@ export function codexAppServerArgs(servers?: { readonly room?: CodexRoomServer; readonly pages?: { readonly url: string; readonly serverName: string }; readonly commands?: { readonly url: string; readonly serverName: string }; + readonly device?: { readonly url: string; readonly serverName: string }; }): ReadonlyArray { return [ ...CODEX_APP_SERVER_ARGS, @@ -131,14 +132,17 @@ export function codexAppServerArgs(servers?: { ...(servers?.room !== undefined ? codexRoomServerArgs(servers.room) : []), ...(servers?.pages !== undefined ? codexPagesServerArgs(servers.pages) : []), ...(servers?.commands !== undefined ? codexPagesServerArgs(servers.commands) : []), + ...(servers?.device !== undefined ? codexPagesServerArgs(servers.device) : []), ]; } /** - * The page tools (McpPagesServer), and `ask_user_to_run` (McpCommandsServer). - * They never ask: a page shows only in the caller's own thread, and a command - * card only asks the user. That is why each has a server of its own, since - * Codex approves per server and the browser tools must still ask. + * The page tools (McpPagesServer), `ask_user_to_run` (McpCommandsServer) and + * the device tools (McpDeviceServer). They never ask: a page shows only in the + * caller's own thread, a command card only asks the user, and the user turned + * on "Agents can use devices" for the device tools. That is why each has a + * server of its own, since Codex approves per server and the browser tools + * must still ask. */ function codexPagesServerArgs(pages: { readonly url: string; diff --git a/apps/server/src/provider/deviceInstructions.ts b/apps/server/src/provider/deviceInstructions.ts new file mode 100644 index 000000000..f415058ba --- /dev/null +++ b/apps/server/src/provider/deviceInstructions.ts @@ -0,0 +1,26 @@ +/** + * The device tools, introduced to a model (docs/design/devices.md). + * + * Sent only to sessions started with `deviceTools`. An agent picks between + * these tools, `simctl`/`adb` and its own habits before it opens anything, so + * this says which to reach for. How to drive a device comes back from + * `device_open` itself: it costs nothing in threads that never open one and + * cannot drift from the pinned agent-device version. + * + * ACP providers get the tool descriptions only, like every other Threadlines + * tool there. + */ +import { DEVICE_MCP_SERVER_NAME } from "../mcp/deviceTools.ts"; + +/** + * `toolPrefix` is what precedes each device tool's own name for this + * provider: `mcp__threadlines_device__` for Claude and Codex, `` for Muse, + * `_` for OpenCode. + */ +export function buildDeviceInstructions( + toolPrefix: string = `mcp__${DEVICE_MCP_SERVER_NAME}__`, +): string { + return ` +The user can watch iOS Simulators and Android Emulators on this computer in the Device tab beside this chat. For any work on a simulator or emulator's screen (opening an app, tapping, typing, reading what is shown, taking a screenshot), call \`${toolPrefix}device_open\` and drive the device with the \`agent-device\` command it returns, so the user sees each step. \`${toolPrefix}device_list\` shows what can be opened, \`${toolPrefix}device_screenshot\` shows you the screen, and \`${toolPrefix}device_close\` ends your hold on a device. \`simctl\`, \`adb\` and build tools stay fine for builds, installs, logs and test runners. +`; +} diff --git a/apps/server/src/provider/opencode/OpenCodeRules.test.ts b/apps/server/src/provider/opencode/OpenCodeRules.test.ts index 716094df5..48c41c860 100644 --- a/apps/server/src/provider/opencode/OpenCodeRules.test.ts +++ b/apps/server/src/provider/opencode/OpenCodeRules.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "vite-plus/test"; import { type OpenCodeRule, openCodeBrowserServerName, + openCodeDeviceServerName, openCodePagesServerName, openCodeSessionGrantRules, openCodeSessionRules, @@ -114,6 +115,24 @@ describe("openCodeSessionRules", () => { expect(effect(rules, `${openCodePagesServerName(otherKey)}_show_page`, "*")).toBe("deny"); }); + it("lets only a session with devices reach its own thread's device tools", () => { + const rulesWith = (deviceTools: boolean) => [ + ...BUILD_AGENT, + ...openCodeSessionRules({ + runtimeMode: "approval-required", + agentRules: BUILD_AGENT, + toolKey: ownKey, + roomTools: false, + deviceTools, + }), + ]; + const own = `${openCodeDeviceServerName(ownKey)}_device_open`; + const other = `${openCodeDeviceServerName(otherKey)}_device_open`; + expect(effect(rulesWith(true), own, "*")).toBe("allow"); + expect(effect(rulesWith(true), other, "*")).toBe("deny"); + expect(effect(rulesWith(false), own, "*")).toBe("deny"); + }); + it("honours a user's deny of Threadlines' own tools", () => { const agent: ReadonlyArray = [ ...BUILD_AGENT, diff --git a/apps/server/src/provider/opencode/OpenCodeRules.ts b/apps/server/src/provider/opencode/OpenCodeRules.ts index adcb42dcd..0ea190090 100644 --- a/apps/server/src/provider/opencode/OpenCodeRules.ts +++ b/apps/server/src/provider/opencode/OpenCodeRules.ts @@ -35,6 +35,7 @@ const BROWSER_SERVER_PREFIX = "threadlines_b_"; const ROOM_SERVER_PREFIX = "threadlines_r_"; const PAGES_SERVER_PREFIX = "threadlines_p_"; const COMMANDS_SERVER_PREFIX = "threadlines_c_"; +const DEVICE_SERVER_PREFIX = "threadlines_d_"; /** Short, stable, tool-name-safe key for one thread's tool servers. */ export function openCodeThreadToolKey(threadKey: string): string { @@ -57,6 +58,10 @@ export function openCodeCommandsServerName(toolKey: string): string { return `${COMMANDS_SERVER_PREFIX}${toolKey}`; } +export function openCodeDeviceServerName(toolKey: string): string { + return `${DEVICE_SERVER_PREFIX}${toolKey}`; +} + /** OpenCode's own wildcard: `*` is any run of characters, `?` any one. */ function wildcardMatch(input: string, pattern: string): boolean { const escaped = pattern @@ -122,6 +127,8 @@ export function openCodeSessionRules(input: { readonly agentPages?: boolean; /** `ask_user_to_run`: it only shows the user a card, so it never asks. */ readonly commandRequests?: boolean; + /** The device tools: the user turned on "Agents can use devices", so they never ask. */ + readonly deviceTools?: boolean; }): ReadonlyArray { const mode = modeRules(input.runtimeMode); return [ @@ -130,6 +137,7 @@ export function openCodeSessionRules(input: { { action: `${ROOM_SERVER_PREFIX}*`, resource: "*", effect: "deny" }, { action: `${PAGES_SERVER_PREFIX}*`, resource: "*", effect: "deny" }, { action: `${COMMANDS_SERVER_PREFIX}*`, resource: "*", effect: "deny" }, + { action: `${DEVICE_SERVER_PREFIX}*`, resource: "*", effect: "deny" }, { action: `${openCodeBrowserServerName(input.toolKey)}_*`, resource: "*", @@ -162,6 +170,15 @@ export function openCodeSessionRules(input: { } satisfies OpenCodeRule, ] : []), + ...(input.deviceTools === true + ? [ + { + action: `${openCodeDeviceServerName(input.toolKey)}_*`, + resource: "*", + effect: "allow", + } satisfies OpenCodeRule, + ] + : []), ...agentRestrictions(input.agentRules, mode), ]; } diff --git a/apps/server/src/provider/opencode/OpenCodeToolItems.ts b/apps/server/src/provider/opencode/OpenCodeToolItems.ts index cf987034b..332c5ee1a 100644 --- a/apps/server/src/provider/opencode/OpenCodeToolItems.ts +++ b/apps/server/src/provider/opencode/OpenCodeToolItems.ts @@ -18,6 +18,7 @@ import { BROWSER_MCP_SERVER_NAME } from "../../mcp/McpHttpServer.ts"; import { ROOM_MCP_SERVER_NAME } from "../../mcp/roomToolAccess.ts"; import { PAGES_MCP_SERVER_NAME } from "../../mcp/pageTools.ts"; import { COMMANDS_MCP_SERVER_NAME } from "../../mcp/commandTools.ts"; +import { DEVICE_MCP_SERVER_NAME } from "../../mcp/deviceTools.ts"; const DETAIL_MAX_CHARS = 400; const OUTPUT_MAX_CHARS = 20_000; @@ -59,12 +60,13 @@ function truncate(value: string, max: number): string { /** * Threadlines' own tool servers are registered per thread under - * `threadlines_b_` / `threadlines_r_` / `threadlines_p_` / `threadlines_c_`; + * `threadlines_b_` / `threadlines_r_` / `threadlines_p_` / `threadlines_c_` / + * `threadlines_d_`; * the chat knows them by their * stable names, so that is what it is told. */ function threadlinesMcpTool(name: string): OpenCodeToolIdentity["mcp"] | undefined { - const match = /^threadlines_([brpc])_[0-9a-f]+_(.+)$/u.exec(name); + const match = /^threadlines_([brpcd])_[0-9a-f]+_(.+)$/u.exec(name); if (!match) return undefined; return { server: @@ -74,7 +76,9 @@ function threadlinesMcpTool(name: string): OpenCodeToolIdentity["mcp"] | undefin ? PAGES_MCP_SERVER_NAME : match[1] === "c" ? COMMANDS_MCP_SERVER_NAME - : ROOM_MCP_SERVER_NAME, + : match[1] === "d" + ? DEVICE_MCP_SERVER_NAME + : ROOM_MCP_SERVER_NAME, tool: match[2]!, }; } From 20ae62f2b02ac4701917e6971a3c41a65b82f9aa Mon Sep 17 00:00:00 2001 From: Badcuban <108198679+badcuban@users.noreply.github.com> Date: Fri, 20 Nov 2026 01:23:11 -0500 Subject: [PATCH 05/49] fix(server): preview tunnel refuses device tool ports A loopback port whose listener is the device hub, serve-sim, serve-emu, the agent-device daemon or one of their children is refused, whoever started it, and left out of the local servers list. Found by command line and process tree; nothing is ever signalled this way. --- apps/server/src/preview/LocalServers.ts | 42 +++--- apps/server/src/preview/PreviewTunnel.test.ts | 99 +++++++++++++ apps/server/src/preview/PreviewTunnel.ts | 21 ++- apps/server/src/preview/deviceToolPorts.ts | 132 ++++++++++++++++++ .../server/src/preview/parseListeningPorts.ts | 84 +++++++++-- packages/shared/src/processTree.ts | 31 +++- 6 files changed, 374 insertions(+), 35 deletions(-) create mode 100644 apps/server/src/preview/deviceToolPorts.ts diff --git a/apps/server/src/preview/LocalServers.ts b/apps/server/src/preview/LocalServers.ts index 8fdf65f36..4614bc744 100644 --- a/apps/server/src/preview/LocalServers.ts +++ b/apps/server/src/preview/LocalServers.ts @@ -9,15 +9,18 @@ */ import type { PreviewLocalServer } from "@threadlines/contracts"; -import { execFile } from "node:child_process"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; -import { parseListeningPorts, type ListeningPort } from "./parseListeningPorts.ts"; +import { findDeviceToolProcessIds } from "./deviceToolPorts.ts"; +import { + parseListeningPorts, + readListeningSockets, + type ListeningPort, +} from "./parseListeningPorts.ts"; import { probeHttpServer } from "./probeHttpServer.ts"; -const LSOF_TIMEOUT_MS = 5_000; /** * How long a probe result stands. Long enough that a stable server is not * re-fetched on every poll, short enough that restarting one on the same port @@ -68,26 +71,19 @@ export const make = Effect.gen(function* LocalServersMake() { scan: Effect.fn("LocalServers.scan")(function* () { // Never fails the caller: an empty list renders as "nothing running", // which is the truth whether lsof is missing or genuinely found nothing. - return yield* Effect.promise( - () => - new Promise>((resolve) => { - if (process.platform === "win32") { - resolve([]); - return; - } - execFile( - "lsof", - ["-iTCP", "-sTCP:LISTEN", "-P", "-n", "-F", "pcn"], - { timeout: LSOF_TIMEOUT_MS, maxBuffer: 4_000_000 }, - (_error, stdout) => { - // lsof exits non-zero when some descriptors are unreadable - // while still printing the ones it could read, so stdout is - // used even on error rather than discarding a good result. - void probeAll(parseListeningPorts(stdout ?? "")).then(resolve); - }, - ); - }), - ); + return yield* Effect.promise(async () => { + const [output, devicePids] = await Promise.all([ + readListeningSockets(), + findDeviceToolProcessIds(), + ]); + // Device tools' ports are never offered: the tunnel refuses them. When + // they can't be told apart the list keeps everything, and the tunnel + // still refuses. + const listeners = parseListeningPorts(output ?? "").filter( + (entry) => devicePids?.has(entry.pid) !== true, + ); + return await probeAll(listeners); + }); }), }); }).pipe(Effect.withSpan("LocalServers.make")); diff --git a/apps/server/src/preview/PreviewTunnel.test.ts b/apps/server/src/preview/PreviewTunnel.test.ts index 60009a9b7..21a53aeb5 100644 --- a/apps/server/src/preview/PreviewTunnel.test.ts +++ b/apps/server/src/preview/PreviewTunnel.test.ts @@ -3,7 +3,11 @@ import { PREVIEW_TUNNEL_WINDOW_BYTES, type PreviewTunnelEvent } from "@threadlin import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Stream from "effect/Stream"; +import { spawn } from "node:child_process"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import * as Net from "node:net"; +import { tmpdir } from "node:os"; +import * as Path from "node:path"; import { loopbackAddresses, makePreviewTunnel } from "./PreviewTunnel.ts"; @@ -46,6 +50,67 @@ const pullEvents = (stream: Stream.Stream) => }); }); +/** + * A stand-in device hub: a real process run from the hub's entry-point path, + * listening on one port, with a child of its own (an emulator, say) listening + * on another. Both exit when their stdin closes, so neither outlives the test. + */ +const HUB_CHILD = ` +const server = require("node:net").createServer(); +server.listen(0, "127.0.0.1", () => process.stdout.write(server.address().port + "\\n")); +process.stdin.on("end", () => process.exit(0)); +process.stdin.resume(); +`; +const HUB_SCRIPT = ` +import { spawn } from "node:child_process"; +import * as Net from "node:net"; +const server = Net.createServer(); +await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); +const child = spawn(process.execPath, ["-e", ${JSON.stringify(HUB_CHILD)}], { + stdio: ["pipe", "pipe", "inherit"], +}); +let heard = ""; +child.stdout.on("data", (chunk) => { + heard += chunk; + if (heard.includes("\\n")) { + const ports = { hub: server.address().port, child: Number.parseInt(heard, 10) }; + process.stdout.write(JSON.stringify(ports) + "\\n"); + } +}); +process.stdin.on("end", () => process.exit(0)); +process.stdin.resume(); +`; + +const startStandInHub = Effect.acquireRelease( + Effect.promise(async () => { + const root = await mkdtemp(Path.join(tmpdir(), "preview-tunnel-hub-")); + const entry = Path.join(root, "node_modules/expo-device-hub/dist/server/cli.mjs"); + await mkdir(Path.dirname(entry), { recursive: true }); + await writeFile(entry, HUB_SCRIPT); + const hub = spawn(process.execPath, [entry, "--port", "0"], { + stdio: ["pipe", "pipe", "inherit"], + }); + const ports = await new Promise<{ hub: number; child: number }>((resolve, reject) => { + let heard = ""; + hub.stdout.on("data", (chunk) => { + heard += chunk; + if (heard.includes("\n")) resolve(JSON.parse(heard)); + }); + hub.once("exit", (code) => reject(new Error(`The stand-in hub exited (${code}).`))); + }); + return { root, hub, ports }; + }), + ({ root, hub }) => + Effect.promise(async () => { + // Its child exits on its own once the hub's end of the pipe closes. + hub.stdin.end(); + if (hub.exitCode === null) { + await new Promise((resolve) => hub.once("exit", resolve)); + } + await rm(root, { recursive: true, force: true }); + }), +); + const decode = (event: PreviewTunnelEvent) => event._tag === "data" ? Buffer.from(event.data, "base64").toString("utf8") : `<${event._tag}>`; @@ -200,6 +265,40 @@ describe("PreviewTunnel", () => { ), ); + it.live.skipIf(process.platform === "win32")( + "refuses a device tool's ports and its children's, and opens a dev server's", + () => + Effect.scoped( + Effect.gen(function* () { + const { ports } = yield* startStandInHub; + const { port: devServer } = yield* listen(() => {}); + const tunnels = yield* makePreviewTunnel({ ownPort: -1 }); + + for (const [tunnelId, port] of [ + ["hub", ports.hub], + ["child", ports.child], + ] as const) { + const refused = yield* tunnels + .open(sessionId, { tunnelId, host: "localhost", port }) + .pipe(Effect.flip); + assert.strictEqual( + refused.detail, + "That port belongs to device support and can't be previewed.", + tunnelId, + ); + } + + const events = yield* tunnels.open(sessionId, { + tunnelId: "dev", + host: "localhost", + port: devServer, + }); + const next = yield* pullEvents(events); + assert.strictEqual((yield* next)._tag, "opened"); + }), + ), + ); + it.live("says plainly when nothing is listening", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/preview/PreviewTunnel.ts b/apps/server/src/preview/PreviewTunnel.ts index 05c0a3577..7e34ab428 100644 --- a/apps/server/src/preview/PreviewTunnel.ts +++ b/apps/server/src/preview/PreviewTunnel.ts @@ -8,7 +8,9 @@ * in contracts) and this opens it on our own loopback and passes the bytes. * * Every part is about staying a pipe and nothing more: only loopback, so no - * client gets a way into the network this machine sits on; bytes in the order + * client gets a way into the network this machine sits on; never a device + * tool's port, which takes actions from anyone on loopback (see + * `deviceToolPorts.ts`); bytes in the order * the page wrote them, however the calls carrying them were scheduled; and no * more in flight than the page is reading, so a stalled page holds the dev * server up instead of filling this process with its output. @@ -34,6 +36,7 @@ import * as Stream from "effect/Stream"; import * as Net from "node:net"; import { ServerConfig } from "../config.ts"; +import { currentDeviceToolPorts } from "./deviceToolPorts.ts"; /** Tunnels one session may hold open at once; a busy page uses a few dozen. */ export const PREVIEW_TUNNEL_MAX_PER_SESSION = 256; @@ -115,7 +118,8 @@ export interface PreviewTunnelShape { /** * Opens one connection and hands back what it sends. Scoped: the scope * closing -- the client unsubscribing, its socket dropping -- destroys it. - * Fails when the host is not loopback or nothing answers there. + * Fails when the host is not loopback, the port is a device tool's, or + * nothing answers there. */ readonly open: ( sessionId: string, @@ -300,6 +304,19 @@ const makeTunnels = (options: { readonly ownPort: number }): PreviewTunnelShape }), ); + // Asked afresh for every connection: a port that served a dev server a + // moment ago may be the device hub's now. + const devicePorts = yield* Effect.promise(() => currentDeviceToolPorts()); + if (devicePorts === null || devicePorts.has(input.port)) { + return yield* new PreviewTunnelError({ + reason: "notLoopback", + detail: + devicePorts === null + ? "Threadlines couldn't check what is listening on that port, so it didn't open it. Try again." + : "That port belongs to device support and can't be previewed.", + }); + } + let lastError: unknown = null; let socket: Net.Socket | null = null; for (const address of addresses) { diff --git a/apps/server/src/preview/deviceToolPorts.ts b/apps/server/src/preview/deviceToolPorts.ts new file mode 100644 index 000000000..dcd8696fa --- /dev/null +++ b/apps/server/src/preview/deviceToolPorts.ts @@ -0,0 +1,132 @@ +/** + * Which listeners on this machine belong to device tools, so a preview never + * reaches one. + * + * The device hub (and serve-sim/serve-emu inside it) and the agent-device + * daemon listen on loopback and accept actions there: touch input, app + * installs, shell commands. A client must reach them only through the device + * channel, never through the loopback tunnel (docs/design/devices.md, "The hub + * is never reachable from a client"). That holds for every copy on this + * machine, whoever started it -- this server, another Threadlines server, an + * older release -- so they are found by command line rather than from what + * this server spawned. Matching by name is for refusing only; nothing here + * ever signals a process it found this way. + * + * A process is a device tool's when its own command line matches, or one of + * its ancestors' does: an emulator the hub boots is the hub's child and + * listens on ports of its own. + */ +import { listProcessTreesAsync } from "@threadlines/shared/processTree"; +import { execFile } from "node:child_process"; + +import { parseListenerPortNumbers, readListeningSockets } from "./parseListeningPorts.ts"; + +const SCAN_TIMEOUT_MS = 5_000; + +/** + * The command lines of device tools, as an extended regular expression for + * `pgrep -f`: the hub's entry point, the agent-device daemon's, and + * serve-sim/serve-emu run on their own (their vendored entry points, or a bin + * by that name) in case a hub version spawns them as helpers. + */ +export const DEVICE_TOOL_COMMAND_PATTERN = [ + "expo-device-hub/dist/server/cli\\.mjs", + "agent-device/dist/src/internal/daemon\\.js", + "vendor/serve-(sim|emu)/", + "(^|[/ ])serve-(sim|emu)( |$)", +].join("|"); + +/** + * Pids whose command line matches DEVICE_TOOL_COMMAND_PATTERN. Empty when + * none does, or when this machine has no `pgrep` (a slim container, where no + * device tool runs either); null when the search failed or ran out of time. + */ +function matchingProcessIds(): Promise { + return new Promise((resolve) => { + execFile( + "pgrep", + ["-f", DEVICE_TOOL_COMMAND_PATTERN], + { encoding: "utf8", timeout: SCAN_TIMEOUT_MS }, + (error, stdout) => { + const code = (error as { code?: unknown } | null)?.code; + if (error !== null && error.killed !== true && (code === 1 || code === "ENOENT")) { + // 1: nothing matched. + resolve([]); + return; + } + if (error !== null) { + resolve(null); + return; + } + resolve( + stdout + .split("\n") + .map((line) => Number.parseInt(line.trim(), 10)) + .filter((pid) => Number.isInteger(pid) && pid > 0), + ); + }, + ); + }); +} + +/** + * Every process that belongs to a device tool: the matches and all their + * descendants. Null when that can't be known. Empty on Windows, where this + * check does not run. + */ +export async function findDeviceToolProcessIds(): Promise | null> { + if (process.platform === "win32") return new Set(); + const roots = await matchingProcessIds(); + if (roots === null) return null; + if (roots.length === 0) return new Set(); + const tree = await listProcessTreesAsync(roots, SCAN_TIMEOUT_MS); + return tree === null ? null : new Set(tree); +} + +/** Every port a device tool listens on, at any address. Null when that can't be known. */ +async function scanDeviceToolPorts(): Promise | null> { + const pids = await findDeviceToolProcessIds(); + if (pids === null) return null; + if (pids.size === 0) return new Set(); + const output = await readListeningSockets({ pids: [...pids] }); + return output === null ? null : new Set(parseListenerPortNumbers(output)); +} + +/** + * Runs `scan` for callers that each need an answer from after they asked. + * A caller arriving mid-scan waits for the next one, which every caller + * arriving meanwhile shares: a page opening dozens of connections at once + * costs two scans, not dozens, and none is answered from before it asked. + */ +function freshScans(scan: () => Promise): () => Promise { + let running: Promise | null = null; + let queued: Promise | null = null; + const start = (): Promise => { + const current: Promise = scan().finally(() => { + if (running === current) running = null; + }); + running = current; + return current; + }; + return () => { + if (running === null) return start(); + if (queued === null) { + const next = running.then( + () => undefined, + () => undefined, + ); + queued = next.then(() => { + queued = null; + return start(); + }); + } + return queued; + }; +} + +/** + * The ports device tools listen on right now, from a scan begun after the + * call. Never cached: a port that was a dev server's a moment ago may be the + * hub's now. Null when that can't be known, which callers treat as a refusal. + */ +export const currentDeviceToolPorts = freshScans(scanDeviceToolPorts); diff --git a/apps/server/src/preview/parseListeningPorts.ts b/apps/server/src/preview/parseListeningPorts.ts index 2556dfb9f..86ba7aa70 100644 --- a/apps/server/src/preview/parseListeningPorts.ts +++ b/apps/server/src/preview/parseListeningPorts.ts @@ -1,5 +1,5 @@ /** - * Parses `lsof -iTCP -sTCP:LISTEN -P -n -F pcn`. + * Lists and parses `lsof -iTCP -sTCP:LISTEN -P -n -F pcn`. * * `-F` emits one field per line behind a single-character tag, which is the * only lsof output worth depending on: the human-readable table changes shape @@ -7,6 +7,9 @@ * (command) describe the process, and every `n` (address) line after them * belongs to that process until the next `p`. */ +import { execFile } from "node:child_process"; + +const LSOF_TIMEOUT_MS = 5_000; export interface ListeningPort { port: number; @@ -40,6 +43,18 @@ function loopbackProbeHost(address: string): "127.0.0.1" | "::1" | null { } } +/** + * The port of an lsof `n` address, or null when it is not a listener's. IPv6 + * arrives bracketed, and lsof writes "->" for established connections; a + * listener has no peer. + */ +function listenerPort(address: string): number | null { + if (address.includes("->")) return null; + const port = Number.parseInt(address.slice(address.lastIndexOf(":") + 1), 10); + return Number.isNaN(port) || port <= 0 ? null : port; +} + +/** Listeners this machine's browser can reach, one per port, sorted by port. */ export function parseListeningPorts(lsofOutput: string): ListeningPort[] { const byPort = new Map(); let pid: number | null = null; @@ -63,14 +78,9 @@ export function parseListeningPorts(lsofOutput: string): ListeningPort[] { if (tag !== "n" || pid === null) { continue; } - // IPv6 arrives bracketed, and lsof writes "->" for established - // connections; a listener has no peer. - const probeHost = value.includes("->") ? null : loopbackProbeHost(value); - if (probeHost === null) { - continue; - } - const port = Number.parseInt(value.slice(value.lastIndexOf(":") + 1), 10); - if (Number.isNaN(port) || port <= 0) { + const port = listenerPort(value); + const probeHost = port === null ? null : loopbackProbeHost(value); + if (port === null || probeHost === null) { continue; } // The same port often appears twice, once per address family: one entry, @@ -85,3 +95,59 @@ export function parseListeningPorts(lsofOutput: string): ListeningPort[] { return [...byPort.values()].sort((a, b) => a.port - b.port); } + +/** + * Every port listened on in lsof output, at any address. For refusing + * ports: a listener bound only to 127.0.0.2 is not offered as a server, but a + * tunnel can still reach it. + */ +export function parseListenerPortNumbers(lsofOutput: string): number[] { + const ports = new Set(); + for (const line of lsofOutput.split("\n")) { + if (line[0] !== "n") continue; + const port = listenerPort(line.slice(1)); + if (port !== null) ports.add(port); + } + return [...ports].sort((a, b) => a - b); +} + +/** + * Runs `lsof -iTCP -sTCP:LISTEN -P -n -F pcn`, for every process or only + * `pids`, and hands back what it printed. Null when lsof is missing or ran out + * of time; empty on Windows, which has no lsof. + */ +export function readListeningSockets(options?: { + readonly pids?: ReadonlyArray; +}): Promise { + if (process.platform === "win32") return Promise.resolve(""); + const pids = options?.pids; + if (pids !== undefined && pids.length === 0) return Promise.resolve(""); + const args = [ + ...(pids === undefined ? [] : ["-a", "-p", pids.join(",")]), + "-iTCP", + "-sTCP:LISTEN", + "-P", + "-n", + "-F", + "pcn", + ]; + return new Promise((resolve) => { + execFile( + "lsof", + args, + { encoding: "utf8", timeout: LSOF_TIMEOUT_MS, maxBuffer: 4_000_000 }, + (error, stdout) => { + // lsof exits 1 when some descriptors are unreadable, or some of the + // given pids hold no listener, while still printing the ones it found, + // so its output stands then. Not when it never ran or was cut off: + // half a list read as a whole one would leave listeners out. + const exitedWithCode = typeof (error as { code?: unknown } | null)?.code === "number"; + if (error !== null && (!exitedWithCode || error.killed === true)) { + resolve(null); + return; + } + resolve(stdout); + }, + ); + }); +} diff --git a/packages/shared/src/processTree.ts b/packages/shared/src/processTree.ts index 40d5a25a1..95c69a022 100644 --- a/packages/shared/src/processTree.ts +++ b/packages/shared/src/processTree.ts @@ -32,13 +32,42 @@ export function listDescendantProcessIdsAsync( timeoutMs: number, ): Promise { if (process.platform === "win32") return Promise.resolve([]); + return readParentTableAsync(timeoutMs).then((table) => + table === null ? null : descendantsFromTable(table, rootPid), + ); +} + +/** + * Every process in the trees rooted at `rootPids`: the roots themselves and + * all their descendants, from one `ps` snapshot. Roots that have already + * exited are still listed. Null when the table couldn't be read; empty on + * Windows, which has no `ps`. + */ +export function listProcessTreesAsync( + rootPids: ReadonlyArray, + timeoutMs: number, +): Promise { + if (process.platform === "win32") return Promise.resolve([]); + if (rootPids.length === 0) return Promise.resolve([]); + return readParentTableAsync(timeoutMs).then((table) => { + if (table === null) return null; + const members = new Set(rootPids); + for (const root of rootPids) { + for (const pid of descendantsFromTable(table, root)) members.add(pid); + } + return [...members]; + }); +} + +/** A `ps -eo pid=,ppid=` table, or null when it couldn't be read in time. */ +function readParentTableAsync(timeoutMs: number): Promise { return new Promise((resolve) => { execFile( "ps", ["-eo", "pid=,ppid="], { encoding: "utf8", timeout: timeoutMs, maxBuffer: 16 * 1024 * 1024 }, (error, stdout) => { - resolve(error === null ? descendantsFromTable(stdout, rootPid) : null); + resolve(error === null ? stdout : null); }, ); }); From f0ada337492bb298af9b4135d21988284fff6c0f Mon Sep 17 00:00:00 2001 From: Badcuban <108198679+badcuban@users.noreply.github.com> Date: Fri, 20 Nov 2026 01:23:11 -0500 Subject: [PATCH 06/49] docs(devices): what the server build settled Managed Node.js for the tools, hub state folder, how sessions end, the fence on device tools, the agent-opened signal, settings reactions, and what the live check found about serve-sim's ports. --- docs/design/devices.md | 50 +++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/docs/design/devices.md b/docs/design/devices.md index fbbe9e7f8..c1afdd5fb 100644 --- a/docs/design/devices.md +++ b/docs/design/devices.md @@ -61,15 +61,24 @@ SSH hosts removed: devices". Reuse our pinned-npm installer (`provider/acpRegistry/AcpRegistryInstaller.ts`, `npmPinnedFlags`) and managed Node where they fit instead of copying T3's `npm install --prefix`. Install scripts stay on: the hub's native addons - need them. Runs with the server's Node; never `npx`. Keeps the last two - versions (`deviceToolMaintenance.ts`). + need them. npm and both tools run on Threadlines' managed Node.js + (`/tools/node`, shared with community agents, leased while a tool + runs), never `npx` and never the server's own runtime, which is Electron in + the desktop app. Both tools load only N-API addons, so a later Node.js + release runs an earlier install. Keeps the pinned version and the last + other one, never one a running process was started from + (`pruneDeviceTool`, in place of T3's `deviceToolMaintenance.ts`). - `DeviceHost.ts`: the seam. One implementation, `LocalDeviceHost.ts`. - `LocalDeviceHost.ts`: platform checks (macOS + `xcrun` for iOS; full Android SDK for Android), spawns the hub as a supervised child on a reserved loopback port with `--host 127.0.0.1`, restarts it with backoff, records `{pid, port, entryPath}` in `/device/hub.json` so the next server stops a hub this one left behind (checked by pid AND command line, never by - name), and the same for the agent-device daemon. + name). The hub's own state file goes to `/device/hub-state` + (`EXPO_DEVICE_HUB_STATE_DIR`), not the shared temp folder. The daemon + records itself in `/device/agent-device/daemon.json`; a healthy + daemon on the pinned version is reused, one on another pin is stopped + through its own CLI and replaced. - `DeviceHubOperations.ts`: list, boot, shutdown, screenshot through the hub. - `DeviceService.ts`: state (`DeviceServiceState`), sessions per thread, open/close/shutdown, `revision` for subscribers, stop the hub and daemon on @@ -81,7 +90,9 @@ SSH hosts removed: session of every claim whose thread was archived, deleted or moved while the server was down (sweep like `secrets/AgentSecretRequests.ts`), and keeps the rest. A reactor releases claims on `thread.archived`, `thread.move-started` - and `thread.deleted`. Closing never shuts a simulator down unless asked + and `thread.deleted`. Ending a session ends its agent-device session on the + daemon (`close` over the daemon's RPC, which quits the app it opened and + frees the device) and removes its launcher. Closing never shuts a simulator down unless asked (`shutdown: true` or the power button), the same as T3: a simulator the user booted is theirs. `bootedByThreadlines` is shown in the picker so the user can tell which ones Threadlines started. @@ -116,9 +127,12 @@ unauthenticated `/api`; serve-emu's action routes have no auth. So: agent-device daemon, found through the listening-port scan (`preview/parseListeningPorts.ts`, `LocalServers.ts`) by pid tree and by command line, so it also covers another server's tools. Refusing by name is - fine here; killing by name is not. The builder checks whether serve-sim's - per-device helpers listen on ports of their own (they would carry touch - input without the hub's token) and makes sure this check covers them. + fine here; killing by name is not. Checked live with 0.22.1: serve-sim's + per-device helper runs inside the hub process and opens no port of its own; + an emulator the hub boots is its child, so the pid tree covers its ports. + The check runs where `pgrep`, `ps` and `lsof` exist (macOS, Linux); on + Windows and in the slim Docker image, where no simulator runs, it is + skipped (preview/deviceToolPorts.ts). 4. Clients reach the hub only through the device channel below, which allows T3's stream and read paths and drops cookies and auth headers. **Not `/vendor/serve-sim/api` or `/vendor/serve-emu/api`**: serve-sim's body @@ -175,10 +189,15 @@ Owner and paired clients may both use devices (same rule as terminals). their descriptions. `device_open` returns the quick start text and a bound launcher path; `device_screenshot` returns an image block. - Session flag `ProviderSessionStartInput.deviceTools`, set in - `ProviderCommandReactor` when `enableDevices && agentDeviceAccess`. Like the - other per-session flags, it applies to sessions started after the switch. + `ProviderCommandReactor` when `enableDevices && agentDeviceAccess`, never for + a fenced thread (one another computer's agent or an outside agent started: + device tools act on this machine). Like the other per-session flags, it + applies to sessions started after the switch. Each tool call also checks + that both settings are still on, so turning either off stops agents at + once, not only in later sessions. - **No PATH changes.** `device_open` writes a launcher bound to that thread's agent-device session at `/device/bin///agent-device` + (a device id with characters unsafe in a path is encoded there) and returns its absolute path; the quick start tells the agent to use that exact path. This works the same in every provider, including OpenCode, whose one server per instance would never see a PATH change made for a later @@ -193,8 +212,10 @@ Owner and paired clients may both use devices (same rule as terminals). use the device tools and `agent-device` for simulator UI work, `simctl`/`adb` stay fine for builds, installs and logs. Codex, Claude, OpenCode, Pi, Muse. ACP providers get it through tool descriptions only (their existing rule). -- An agent's `device_open` publishes an activity the web uses to open the - Device tab on that device for anyone viewing the thread. +- An agent's `device_open` records its session with `openedBy: "agent"`. The + web watches `subscribeDeviceState` and opens the Device tab on that device + for anyone viewing the thread when such a session appears; no orchestration + activity is written. ### Web @@ -213,8 +234,11 @@ Owner and paired clients may both use devices (same rule as terminals). `ServerSettings.enableDevices` (default false) and `ServerSettings.agentDeviceAccess` (default false), plus patch fields. Turning -"Use devices" off stops the hub and daemon; simulators keep running. Turning -"Agents can use devices" off removes the tools from sessions started after. +"Use devices" off ends every device session and stops the hub and daemon; +simulators keep running. Turning "Agents can use devices" off stops the daemon +and makes the tools refuse at once; sessions started after do not get them. +Turning it on fetches agent-device in the background (nothing starts until an +agent asks), so the first `device_open` is quick. Both get app guide places (`devices.enable`, `devices.agentAccess`); agents may propose turning on "Use devices" through `settings_change` but agent access is user-only (`widensAgents`). From 27c9c01af7ca6f33eb80589f21ed149d377e2beb Mon Sep 17 00:00:00 2001 From: Badcuban <108198679+badcuban@users.noreply.github.com> Date: Fri, 20 Nov 2026 01:36:50 -0500 Subject: [PATCH 07/49] feat(web): device RPCs on the client and a devices capability The WS client and the environment API get a device group: the device calls, the state stream (followed again after a reconnect) and the device channel, which never resubscribes. A new optional capability, devices, lets clients hide the Device tab and its entry points from servers that lack it. client-runtime exports its device state helpers separately from the stream code, so the main bundle does not carry the stream. --- .../components/ChatView.browser-harness.tsx | 1 + apps/web/src/environmentApi.ts | 16 +++++++ apps/web/src/rpc/wsRpcClient.ts | 42 +++++++++++++++++ packages/client-runtime/package.json | 4 ++ packages/client-runtime/src/device/index.ts | 1 - packages/contracts/src/environment.ts | 6 +++ packages/contracts/src/ipc.ts | 45 +++++++++++++++++++ 7 files changed, 114 insertions(+), 1 deletion(-) diff --git a/apps/web/src/components/ChatView.browser-harness.tsx b/apps/web/src/components/ChatView.browser-harness.tsx index d11409a14..86a2a588b 100644 --- a/apps/web/src/components/ChatView.browser-harness.tsx +++ b/apps/web/src/components/ChatView.browser-harness.tsx @@ -402,6 +402,7 @@ export function createMockEnvironmentApi(input: { attachments: {} as EnvironmentApi["attachments"], previewAutomation: input.previewAutomation ?? ({} as EnvironmentApi["previewAutomation"]), previewLoopback: {} as EnvironmentApi["previewLoopback"], + device: {} as EnvironmentApi["device"], filesystem: { browse: input.browse, }, diff --git a/apps/web/src/environmentApi.ts b/apps/web/src/environmentApi.ts index 2f401b757..7a1d78251 100644 --- a/apps/web/src/environmentApi.ts +++ b/apps/web/src/environmentApi.ts @@ -41,6 +41,22 @@ export function createEnvironmentApi(rpcClient: WsRpcClient): EnvironmentApi { claim: (claim) => rpcClient.previewAutomation.claim(claim), active: (activity) => rpcClient.previewAutomation.active(activity), }, + device: { + list: rpcClient.device.list, + updateTool: rpcClient.device.updateTool, + open: rpcClient.device.open, + close: rpcClient.device.close, + shutdown: rpcClient.device.shutdown, + detail: rpcClient.device.detail, + action: rpcClient.device.action, + subscribeState: (listener) => rpcClient.device.subscribeState(listener), + channel: { + open: (input, listener, onClosed) => + rpcClient.device.channelOpen(input, listener, { onComplete: onClosed }), + send: rpcClient.device.channelSend, + credit: rpcClient.device.channelCredit, + }, + }, previewLoopback: { open: (input, listener, onClosed) => rpcClient.previewLoopback.open(input, listener, { onComplete: onClosed }), diff --git a/apps/web/src/rpc/wsRpcClient.ts b/apps/web/src/rpc/wsRpcClient.ts index 932859e50..c462bf261 100644 --- a/apps/web/src/rpc/wsRpcClient.ts +++ b/apps/web/src/rpc/wsRpcClient.ts @@ -447,6 +447,20 @@ export interface WsRpcClient { readonly search: RpcUnaryMethod; readonly read: RpcUnaryMethod; }; + /** Simulators and emulators (docs/design/devices.md), on servers reporting `devices`. */ + readonly device: { + readonly list: RpcUnaryMethod; + readonly updateTool: RpcUnaryMethod; + readonly open: RpcUnaryMethod; + readonly close: RpcUnaryMethod; + readonly shutdown: RpcUnaryMethod; + readonly detail: RpcUnaryMethod; + readonly action: RpcUnaryMethod; + readonly subscribeState: RpcStreamMethod; + readonly channelOpen: RpcInputStreamMethod; + readonly channelSend: RpcUnaryMethod; + readonly channelCredit: RpcUnaryMethod; + }; /** This computer's project briefs and notes (docs/design/project-brief-notes.md), on servers reporting `projectContext`. */ readonly projectContext: { readonly subscribe: RpcStreamMethod; @@ -1326,6 +1340,34 @@ export function createWsRpcClient(transport: WsTransport): WsRpcClient { search: (input) => transport.request((client) => client[WS_METHODS.themeVsCodeSearch](input)), read: (input) => transport.request((client) => client[WS_METHODS.themeVsCodeRead](input)), }, + device: { + list: (input) => transport.request((client) => client[WS_METHODS.deviceList](input)), + updateTool: (input) => + transport.request((client) => client[WS_METHODS.deviceUpdateTool](input)), + open: (input) => transport.request((client) => client[WS_METHODS.deviceOpen](input)), + close: (input) => transport.request((client) => client[WS_METHODS.deviceClose](input)), + shutdown: (input) => transport.request((client) => client[WS_METHODS.deviceShutdown](input)), + detail: (input) => transport.request((client) => client[WS_METHODS.deviceDetail](input)), + action: (input) => transport.request((client) => client[WS_METHODS.deviceAction](input)), + subscribeState: (listener, options) => + transport.subscribe((client) => client[WS_METHODS.subscribeDeviceState]({}), listener, { + ...options, + tag: WS_METHODS.subscribeDeviceState, + }), + // A channel carries one request or socket to the hub and cannot pick up + // where it left off: when the connection drops it ends, and the Device + // tab opens a fresh one with a decoder waiting for a keyframe. + channelOpen: (input, listener, options) => + transport.subscribe((client) => client[WS_METHODS.deviceChannelOpen](input), listener, { + tag: WS_METHODS.deviceChannelOpen, + resubscribe: false, + ...(options?.onComplete ? { onComplete: options.onComplete } : {}), + }), + channelSend: (input) => + transport.request((client) => client[WS_METHODS.deviceChannelSend](input)), + channelCredit: (input) => + transport.request((client) => client[WS_METHODS.deviceChannelCredit](input)), + }, projectContext: { subscribe: (listener, options) => transport.subscribe((client) => client[WS_METHODS.projectContextSubscribe]({}), listener, { diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index 22bb6214e..8a315fd7c 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -10,6 +10,10 @@ "./device": { "types": "./src/device/index.ts", "import": "./src/device/index.ts" + }, + "./deviceState": { + "types": "./src/state/deviceState.ts", + "import": "./src/state/deviceState.ts" } }, "scripts": { diff --git a/packages/client-runtime/src/device/index.ts b/packages/client-runtime/src/device/index.ts index ed57105bd..114851fb5 100644 --- a/packages/client-runtime/src/device/index.ts +++ b/packages/client-runtime/src/device/index.ts @@ -2,4 +2,3 @@ export * from "./frame.ts"; export * from "./hubTransport.ts"; export * from "./screenshot.ts"; export * from "./stream.ts"; -export * from "../state/deviceState.ts"; diff --git a/packages/contracts/src/environment.ts b/packages/contracts/src/environment.ts index 952d76bef..840009f0c 100644 --- a/packages/contracts/src/environment.ts +++ b/packages/contracts/src/environment.ts @@ -213,6 +213,12 @@ export const ExecutionEnvironmentCapabilities = Schema.Struct({ * one without it. */ projectContext: Schema.optionalKey(Schema.Boolean), + /** + * The Device tab and Settings → Devices (`device.*`, docs/design/devices.md). + * Older servers omit it; clients then hide the Device tab, its keybinding, + * its palette entry and the Devices settings for them. + */ + devices: Schema.optionalKey(Schema.Boolean), }); export type ExecutionEnvironmentCapabilities = typeof ExecutionEnvironmentCapabilities.Type; diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index f859e4fd6..950152fc1 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -222,6 +222,22 @@ import type { PreviewAutomationClientHost, PreviewAutomationResponse, } from "./previewAutomation.ts"; +import type { + DeviceActionInput, + DeviceChannelCredit, + DeviceChannelEvent, + DeviceChannelOpen, + DeviceChannelSend, + DeviceCloseInput, + DeviceDetail, + DeviceDetailInput, + DeviceListInput, + DeviceOpenInput, + DeviceServiceState, + DeviceSession, + DeviceShutdownInput, + DeviceUpdateToolInput, +} from "./device.ts"; import type { PreviewLocalServer, PreviewTunnelCredit, @@ -1891,6 +1907,35 @@ export interface EnvironmentApi { /** Only for servers that advertise `browserLoopbackTunnel`. */ active: (activity: PreviewAutomationActivity) => Promise; }; + /** + * Simulators and emulators on the environment's machine + * (docs/design/devices.md). Only for servers that advertise `devices`. + */ + device: { + list: (input: DeviceListInput) => Promise; + updateTool: (input: DeviceUpdateToolInput) => Promise; + open: (input: DeviceOpenInput) => Promise; + close: (input: DeviceCloseInput) => Promise; + shutdown: (input: DeviceShutdownInput) => Promise; + detail: (input: DeviceDetailInput) => Promise; + action: (input: DeviceActionInput) => Promise; + /** The state now and after every change, followed again after a reconnect. */ + subscribeState: (listener: (state: DeviceServiceState) => void) => () => void; + /** + * One request or socket to the device hub, open until the returned call. + * Never resubscribed: `onClosed` comes once when it ends any other way, + * with the failure when it failed. + */ + channel: { + open: ( + input: DeviceChannelOpen, + listener: (event: DeviceChannelEvent) => void, + onClosed: (failure?: unknown) => void, + ) => () => void; + send: (input: DeviceChannelSend) => Promise; + credit: (input: DeviceChannelCredit) => Promise; + }; + }; /** * The browser's way to the environment's own `localhost` from another * computer (`previewTunnel.ts`). Only for servers that advertise From f48125c1a92b96fd29f37b896adbcb4643f967fd Mon Sep 17 00:00:00 2001 From: Badcuban <108198679+badcuban@users.noreply.github.com> Date: Fri, 20 Nov 2026 01:36:50 -0500 Subject: [PATCH 08/49] feat(web): a Device tab with a live simulator or emulator The right panel gets a Device tab for project threads on computers with devices. On first use it explains what each step installs and installs nothing until Use devices is on. Then it lists the computer's simulators and emulators, opens one for the thread (starting it when off) and shows its screen in a flat frame sized to the device, with touch, keys and paste, a toolbar of hardware buttons, screenshot and power off, and a Tools drawer that only shows values the device read back. Only the device on screen streams: a hidden tab or another thread holds no channel. An agent opening a device brings the tab up on it for whoever is watching the thread. --- apps/web/src/components/ChatView.tsx | 5 +- .../components/chat/RightPanelTabStrip.tsx | 2 + .../components/device/DeviceLoadingView.tsx | 41 ++ .../components/device/DevicePanel.browser.tsx | 292 +++++++++ .../web/src/components/device/DevicePanel.tsx | 183 ++++++ .../src/components/device/DevicePicker.tsx | 165 +++++ .../web/src/components/device/DeviceSetup.tsx | 241 +++++++ .../components/device/DeviceStreamView.tsx | 188 ++++++ .../components/device/DeviceToolsPanel.tsx | 593 ++++++++++++++++++ .../src/components/device/DeviceWorkspace.tsx | 250 ++++++++ .../components/device/deviceFrameLayout.ts | 36 ++ apps/web/src/components/device/deviceState.ts | 311 +++++++++ .../components/device/useDeviceControls.ts | 83 +++ .../src/components/device/useDeviceStream.ts | 120 ++++ apps/web/src/diffRouteSearch.test.ts | 1 + apps/web/src/diffRouteSearch.ts | 19 +- apps/web/src/rightPanelTabs.ts | 33 +- .../routes/_chat.$environmentId.$threadId.tsx | 44 +- 18 files changed, 2596 insertions(+), 11 deletions(-) create mode 100644 apps/web/src/components/device/DeviceLoadingView.tsx create mode 100644 apps/web/src/components/device/DevicePanel.browser.tsx create mode 100644 apps/web/src/components/device/DevicePanel.tsx create mode 100644 apps/web/src/components/device/DevicePicker.tsx create mode 100644 apps/web/src/components/device/DeviceSetup.tsx create mode 100644 apps/web/src/components/device/DeviceStreamView.tsx create mode 100644 apps/web/src/components/device/DeviceToolsPanel.tsx create mode 100644 apps/web/src/components/device/DeviceWorkspace.tsx create mode 100644 apps/web/src/components/device/deviceFrameLayout.ts create mode 100644 apps/web/src/components/device/deviceState.ts create mode 100644 apps/web/src/components/device/useDeviceControls.ts create mode 100644 apps/web/src/components/device/useDeviceStream.ts diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 965897805..0d7594692 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -97,6 +97,7 @@ import { useRoomRecipientStore, } from "../rooms"; import { ELECTRON_HEADER_HEIGHT_CLASS } from "../desktopChrome"; +import { useDevicesSupported } from "./device/deviceState"; import { isElectron } from "../env"; import { ensureLocalApi, readLocalApi } from "../localApi"; import { parseDiffRouteSearch } from "../diffRouteSearch"; @@ -1603,13 +1604,15 @@ export default function ChatView(props: ChatViewProps) { // Which surfaces this thread's sidebar offers. Resolved from the same inputs // the route uses, so the header's entry points can never land on a tab the // strip would refuse to show. + const devicesSupported = useDevicesSupported(environmentId); const rightPanelAvailableTabs = useMemo( () => availableRightPanelTabs({ isGeneralChat: isGeneralChatThread, isDraft: routeKind === "draft" || !isServerThread, + devicesSupported, }), - [isGeneralChatThread, isServerThread, routeKind], + [devicesSupported, isGeneralChatThread, isServerThread, routeKind], ); /** * A picked element becomes a context attached to the message, not text in it. diff --git a/apps/web/src/components/chat/RightPanelTabStrip.tsx b/apps/web/src/components/chat/RightPanelTabStrip.tsx index c376f3c1c..dd41a604e 100644 --- a/apps/web/src/components/chat/RightPanelTabStrip.tsx +++ b/apps/web/src/components/chat/RightPanelTabStrip.tsx @@ -25,6 +25,7 @@ import { ImagesIcon, ListTreeIcon, PlusIcon, + SmartphoneIcon, XIcon, } from "lucide-react"; import { memo, useCallback, useEffect, useLayoutEffect, useRef, useState } from "react"; @@ -56,6 +57,7 @@ export const RIGHT_PANEL_TAB_ICONS: Readonly> = { agents: BotIcon, media: ImagesIcon, threads: ListTreeIcon, + device: SmartphoneIcon, }; /** Labelled while they fit, all icons once they do not. Never a mix: dropping diff --git a/apps/web/src/components/device/DeviceLoadingView.tsx b/apps/web/src/components/device/DeviceLoadingView.tsx new file mode 100644 index 000000000..c08ceb5e8 --- /dev/null +++ b/apps/web/src/components/device/DeviceLoadingView.tsx @@ -0,0 +1,41 @@ +import { SmartphoneIcon } from "lucide-react"; +import type { ReactNode } from "react"; + +import { Spinner } from "../ui/spinner"; + +/** A device that is opening, connecting or failed: what it is, and one line on where it stands. */ +export function DeviceLoadingView(props: { + readonly name: string; + readonly description?: string; + readonly message: string; + readonly error?: boolean; + readonly children?: ReactNode; +}) { + return ( +
+
+
+
+ ); +} diff --git a/apps/web/src/components/device/DevicePanel.browser.tsx b/apps/web/src/components/device/DevicePanel.browser.tsx new file mode 100644 index 000000000..e16c51a99 --- /dev/null +++ b/apps/web/src/components/device/DevicePanel.browser.tsx @@ -0,0 +1,292 @@ +import "../../index.css"; + +import { + type DeviceChannelEvent, + type DeviceChannelOpen, + type DeviceDetail, + type DeviceServiceState, + type DeviceSession, + type EnvironmentApi, + EnvironmentId, + ThreadId, +} from "@threadlines/contracts"; +import { page } from "vite-plus/test/browser"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; +import { render } from "vitest-browser-react"; + +import { + __resetEnvironmentApiOverridesForTests, + __setEnvironmentApiOverrideForTests, +} from "../../environmentApi"; +import { writePrimaryEnvironmentDescriptor } from "../../environments/primary/context"; +import { resetTestPage } from "../../lib/testPage.browser-harness"; +import { DevicePanel } from "./DevicePanel"; +import { onAgentOpenedDevice, resetDeviceStateForTests } from "./deviceState"; + +const ENVIRONMENT_ID = EnvironmentId.make("environment-devices"); +const THREAD_ID = ThreadId.make("thread-devices"); +const THREAD_KEY = `${ENVIRONMENT_ID}:${THREAD_ID}`; + +const IPHONE = { + id: "SIM-1", + platform: "ios", + name: "iPhone 17 Pro", + version: "iOS 27.2", + booted: false, + physical: false, +} as const; +const PIXEL = { + id: "emulator-5554", + platform: "android", + name: "Pixel 9", + version: "Android 16", + booted: true, + physical: false, +} as const; + +function deviceState(patch: Partial = {}): DeviceServiceState { + return { + enabled: true, + agentAccessEnabled: false, + hostStatus: "ready", + platforms: [ + { platform: "ios", available: true }, + { platform: "android", available: true }, + ], + incompletePlatforms: [], + tools: { + hub: { requiredVersion: "0.22.1", installedVersions: ["0.22.1"], runningVersion: "0.22.1" }, + agent: { requiredVersion: "0.9.0", installedVersions: [], runningVersion: null }, + }, + hubInstalled: true, + agentDeviceInstalled: false, + devices: [IPHONE, PIXEL], + sessions: [], + bootingDevices: [], + revision: 1, + ...patch, + }; +} + +const session = ( + deviceId: string, + platform: "ios" | "android", + openedBy: "user" | "agent", +): DeviceSession => ({ + threadId: THREAD_ID, + deviceId, + platform, + openedAt: "2026-11-20T00:00:00.000Z", + bootedByThreadlines: true, + openedBy, +}); + +const DETAIL: DeviceDetail = { + deviceId: IPHONE.id, + settings: { appearance: "light", textSize: "default", reduceMotion: false }, + foregroundApp: null, + readAt: "2026-11-20T00:00:00.000Z", +}; + +interface FakeChannel { + readonly input: DeviceChannelOpen; + readonly emit: (event: DeviceChannelEvent) => void; + stopped: boolean; +} + +/** A computer with devices, answering from `state` and holding every channel the tab opens. */ +function fakeComputer(initial: DeviceServiceState) { + let state = initial; + let listener: ((next: DeviceServiceState) => void) | null = null; + const channels: FakeChannel[] = []; + const device = { + list: vi.fn(async () => state), + updateTool: vi.fn(async () => state), + open: vi.fn(async (input: { deviceId: string; platform: "ios" | "android" }) => + session(input.deviceId, input.platform, "user"), + ), + close: vi.fn(async () => undefined), + shutdown: vi.fn(async () => undefined), + detail: vi.fn(async () => DETAIL), + action: vi.fn(async () => DETAIL), + subscribeState: (next: (value: DeviceServiceState) => void) => { + listener = next; + next(state); + return () => { + listener = null; + }; + }, + channel: { + open: (input: DeviceChannelOpen, emit: (event: DeviceChannelEvent) => void) => { + const channel: FakeChannel = { input, emit, stopped: false }; + channels.push(channel); + return () => { + channel.stopped = true; + }; + }, + send: vi.fn(async () => undefined), + credit: vi.fn(async () => undefined), + }, + }; + const settings = { update: vi.fn(async () => ({})) }; + __setEnvironmentApiOverrideForTests(ENVIRONMENT_ID, { + device, + settings, + } as unknown as EnvironmentApi); + return { + device, + settings, + channels, + push(next: DeviceServiceState) { + state = next; + listener?.(next); + }, + channel(path: string) { + return channels.find((channel) => channel.input.path === path && !channel.stopped); + }, + }; +} + +function base64(bytes: Uint8Array) { + let binary = ""; + for (const byte of bytes) binary += String.fromCharCode(byte); + return btoa(binary); +} + +/** A JPEG of a plain screen, wrapped as serve-sim's AVCC "seed" envelope. */ +async function seedEnvelope(): Promise { + const canvas = document.createElement("canvas"); + canvas.width = 118; + canvas.height = 256; + const context = canvas.getContext("2d")!; + context.fillStyle = "#3b82f6"; + context.fillRect(0, 0, canvas.width, canvas.height); + const blob = await new Promise((resolve) => + canvas.toBlob((value) => resolve(value!), "image/jpeg"), + ); + const jpeg = new Uint8Array(await blob.arrayBuffer()); + const envelope = new Uint8Array(5 + jpeg.length); + new DataView(envelope.buffer).setUint32(0, jpeg.length + 1); + envelope[4] = 4; + envelope.set(jpeg, 5); + return envelope; +} + +function Host(props: { visible: boolean }) { + return ( +
+ +
+ ); +} + +describe("DevicePanel", () => { + beforeEach(() => { + writePrimaryEnvironmentDescriptor({ + environmentId: ENVIRONMENT_ID, + label: "This computer", + capabilities: { devices: true }, + } as never); + }); + + afterEach(async () => { + __resetEnvironmentApiOverridesForTests(); + resetDeviceStateForTests(); + writePrimaryEnvironmentDescriptor(null); + await resetTestPage(); + }); + + it("explains setup and installs nothing until Use devices is turned on", async () => { + const computer = fakeComputer( + deviceState({ enabled: false, hostStatus: "disabled", devices: [], hubInstalled: false }), + ); + await render(); + + await expect.element(page.getByRole("heading", { name: "Set up devices" })).toBeVisible(); + await expect.element(page.getByText("expo-device-hub 0.22.1")).toBeVisible(); + expect(computer.device.list).not.toHaveBeenCalled(); + + await page.getByRole("switch", { name: "Use devices", exact: true }).click(); + await vi.waitFor(() => + expect(computer.settings.update).toHaveBeenCalledWith({ enableDevices: true }), + ); + await vi.waitFor(() => expect(computer.device.list).toHaveBeenCalledWith({ refresh: true })); + }); + + it("opens a device for the thread and shows its screen through the device channel", async () => { + const computer = fakeComputer(deviceState()); + const mounted = await render(); + + await page.getByRole("button", { name: "Start iPhone 17 Pro" }).click(); + await vi.waitFor(() => + expect(computer.device.open).toHaveBeenCalledWith({ + threadId: THREAD_ID, + deviceId: IPHONE.id, + platform: "ios", + }), + ); + computer.push( + deviceState({ + devices: [{ ...IPHONE, booted: true }, PIXEL], + sessions: [session(IPHONE.id, "ios", "user")], + revision: 2, + }), + ); + await expect.element(page.getByRole("toolbar", { name: "Device controls" })).toBeVisible(); + + // The helper is primed through its MJPEG stream, then the input socket opens. + await vi.waitFor(() => + expect(computer.channel("/vendor/serve-sim/helper/SIM-1/stream.mjpeg")).toBeDefined(), + ); + const prime = computer.channel("/vendor/serve-sim/helper/SIM-1/stream.mjpeg")!; + prime.emit({ _tag: "opened", status: 200, contentType: "multipart/x-mixed-replace" }); + prime.emit({ _tag: "data", data: base64(new Uint8Array([45, 45])) }); + await vi.waitFor(() => expect(computer.channel("/vendor/serve-sim/helper/ws")).toBeDefined()); + expect(computer.channel("/vendor/serve-sim/helper/ws")!.input).toMatchObject({ + _tag: "ws", + query: { device: "SIM-1" }, + }); + + const video = computer.channel("/vendor/serve-sim/helper/SIM-1/stream.avcc")!; + expect(video.input).toMatchObject({ _tag: "http", method: "GET" }); + video.emit({ _tag: "opened", status: 200, contentType: "application/octet-stream" }); + video.emit({ _tag: "data", data: base64(await seedEnvelope()) }); + + await expect.element(page.getByText("Connecting to the screen…")).not.toBeInTheDocument(); + const canvas = document.querySelector("[data-testid=device-frame] canvas"); + expect(canvas?.width).toBe(118); + await vi.waitFor(() => expect(computer.device.channel.credit).toHaveBeenCalled()); + + // Only the device on screen streams: hiding the tab drops every channel. + await mounted.rerender(); + await vi.waitFor(() => + expect(computer.channels.every((channel) => channel.stopped)).toBe(true), + ); + }); + + it("reports a device an agent opened, but not one already open when the tab loaded", async () => { + const computer = fakeComputer( + deviceState({ sessions: [session(PIXEL.id, "android", "agent")] }), + ); + const opened: string[] = []; + const stop = onAgentOpenedDevice((_environmentId, next) => opened.push(next.deviceId)); + await render(); + await expect.element(page.getByText("Open in this thread")).toBeVisible(); + expect(opened).toEqual([]); + + computer.push( + deviceState({ + devices: [{ ...IPHONE, booted: true }, PIXEL], + sessions: [session(PIXEL.id, "android", "agent"), session(IPHONE.id, "ios", "agent")], + revision: 2, + }), + ); + await vi.waitFor(() => expect(opened).toEqual([IPHONE.id])); + stop(); + }); +}); diff --git a/apps/web/src/components/device/DevicePanel.tsx b/apps/web/src/components/device/DevicePanel.tsx new file mode 100644 index 000000000..5f662a656 --- /dev/null +++ b/apps/web/src/components/device/DevicePanel.tsx @@ -0,0 +1,183 @@ +import type { DeviceSummary, EnvironmentId, ThreadId } from "@threadlines/contracts"; +import { threadDeviceSessions } from "@threadlines/client-runtime/deviceState"; +import { CircleAlertIcon, XIcon } from "lucide-react"; +import { useEffect, useRef, useState } from "react"; + +import { Button } from "../ui/button"; +import { DeviceLoadingView } from "./DeviceLoadingView"; +import { DevicePicker } from "./DevicePicker"; +import { DeviceSetup } from "./DeviceSetup"; +import { + closeDevice, + deviceErrorText, + openDevice, + refreshDevices, + selectThreadDevice, + shutdownDevice, + useDevicesSupported, + useDeviceState, + useSelectedThreadDevice, +} from "./deviceState"; +import { DeviceWorkspace } from "./DeviceWorkspace"; + +/** + * The Device tab (docs/design/devices.md): setup on first use, the device + * picker, and the open device. Stays mounted while its tab is in the strip; + * `visible` says whether it is the tab on screen, and only then does a + * device stream. + */ +export function DevicePanel(props: { + readonly environmentId: EnvironmentId; + readonly threadId: ThreadId; + readonly threadKey: string; + readonly visible: boolean; +}) { + const { environmentId, threadId, threadKey, visible } = props; + const supported = useDevicesSupported(environmentId); + const state = useDeviceState(environmentId, supported); + const selectedId = useSelectedThreadDevice(threadKey); + const [inSetup, setInSetup] = useState(false); + const [pending, setPending] = useState(null); + const [refreshing, setRefreshing] = useState(false); + const [error, setError] = useState(null); + + // Devices on and nothing started yet: list them, which starts the hub. Once + // per panel, and only while the tab is on screen. + const listed = useRef(false); + const enabled = state?.enabled === true; + const idle = state?.hostStatus === "idle"; + useEffect(() => { + if (!visible || !enabled || !idle || listed.current) return; + listed.current = true; + refreshDevices(environmentId).catch((cause: unknown) => setError(deviceErrorText(cause))); + }, [enabled, environmentId, idle, visible]); + + if (!supported) { + return ( + + This computer runs a Threadlines without devices. Update it to use the Device tab. + + ); + } + if (state === null) { + return Loading devices…; + } + if (!state.enabled || inSetup) { + return ( + setInSetup(true)} + onDone={() => setInSetup(false)} + /> + ); + } + + const sessions = threadDeviceSessions(state, threadId); + const activeSession = sessions.find((session) => session.deviceId === selectedId); + const activeDevice = activeSession + ? state.devices.find((device) => device.id === activeSession.deviceId) + : undefined; + + const run = async (work: () => Promise) => { + setError(null); + try { + await work(); + } catch (cause) { + setError(deviceErrorText(cause)); + } + }; + + const open = (device: DeviceSummary) => { + if (sessions.some((session) => session.deviceId === device.id)) { + selectThreadDevice(threadKey, device.id); + return; + } + setPending(device.id); + void run(async () => { + await openDevice(environmentId, { threadId, deviceId: device.id, platform: device.platform }); + selectThreadDevice(threadKey, device.id); + }).finally(() => setPending(null)); + }; + + const close = (shutdown: boolean) => { + if (!activeSession) return; + selectThreadDevice(threadKey, null); + void run(() => + closeDevice(environmentId, { threadId, deviceId: activeSession.deviceId, shutdown }), + ); + }; + + const errorLine = error ? ( +
+
+ ) : null; + + if (activeDevice && activeSession) { + return ( +
+ {errorLine} + selectThreadDevice(threadKey, null)} + onClose={() => close(false)} + onPowerOff={() => close(true)} + /> +
+ ); + } + + const pendingDevice = + pending === null ? undefined : state.devices.find((device) => device.id === pending); + return ( +
+ {errorLine} + {pendingDevice ? ( + + ) : ( + + void run(() => + shutdownDevice(environmentId, { deviceId: device.id, platform: device.platform }), + ) + } + onRefresh={() => { + setRefreshing(true); + void run(() => refreshDevices(environmentId)).finally(() => setRefreshing(false)); + }} + /> + )} +
+ ); +} + +function PanelMessage(props: { readonly children: string }) { + return

{props.children}

; +} diff --git a/apps/web/src/components/device/DevicePicker.tsx b/apps/web/src/components/device/DevicePicker.tsx new file mode 100644 index 000000000..d4f2913ad --- /dev/null +++ b/apps/web/src/components/device/DevicePicker.tsx @@ -0,0 +1,165 @@ +import type { DeviceServiceState, DeviceSession, DeviceSummary } from "@threadlines/contracts"; +import { + devicePlatformName, + groupDevicesByPlatform, + isDeviceHostBusy, +} from "@threadlines/client-runtime/deviceState"; +import { PowerIcon, RefreshCwIcon, SmartphoneIcon } from "lucide-react"; + +import { cn } from "~/lib/utils"; +import { SETTINGS_GROUP_ROW_CLASS, SettingsGroup } from "../settings/settingsLayout"; +import { Button } from "../ui/button"; +import { Spinner } from "../ui/spinner"; +import { HubStatus } from "./DeviceSetup"; + +/** + * The devices on the thread's computer, by platform, booted first. Picking + * one opens it for the thread (starting it when it is off); a running one + * can be powered off from its row. + */ +export function DevicePicker(props: { + readonly state: DeviceServiceState; + /** This thread's open devices. */ + readonly sessions: ReadonlyArray; + readonly pendingDeviceId: string | null; + readonly refreshing: boolean; + readonly onOpen: (device: DeviceSummary) => void; + readonly onPowerOff: (device: DeviceSummary) => void; + readonly onRefresh: () => void; +}) { + const { state } = props; + const groups = groupDevicesByPlatform(state.devices); + const unavailable = state.platforms.filter((platform) => !platform.available); + const busy = isDeviceHostBusy(state); + const booting = state.bootingDevices; + const heldIds = new Set(props.sessions.map((session) => session.deviceId)); + const startedByUs = new Set( + state.sessions.filter((session) => session.bootedByThreadlines).map((s) => s.deviceId), + ); + + return ( +
+
+ + {state.devices.length === 1 ? "1 device" : `${state.devices.length} devices`} + + +
+
+ {state.hostStatus !== "ready" && state.hostStatus !== "idle" ? ( + + + + ) : null} + {booting.length > 0 ? ( +

+ Starting {booting.map((device) => device.name).join(", ")}. This can take a minute. +

+ ) : null} + {groups.map((group) => ( +
+

+ {group.platform === "ios" ? "iOS Simulators" : "Android Emulators"} +

+ + {group.devices.map((device) => { + const held = heldIds.has(device.id); + const pending = props.pendingDeviceId === device.id; + return ( +
+ + {device.booted ? ( + + ) : null} +
+ ); + })} +
+
+ ))} + {groups.length === 0 && state.hostStatus === "ready" ? ( +

+ No simulators or emulators on this computer yet. +

+ ) : null} + {unavailable.length > 0 ? ( +
+ {unavailable.map((platform) => ( +

+ {devicePlatformName(platform.platform)}:{" "} + {platform.reason ?? + (platform.platform === "ios" + ? "needs a Mac with Xcode." + : "needs the Android SDK with the emulator.")} +

+ ))} +
+ ) : null} + {state.incompletePlatforms.length > 0 ? ( +

+ The {state.incompletePlatforms.map(devicePlatformName).join(" and ")} list may be + missing devices. Refresh to try again. +

+ ) : null} +
+
+ ); +} diff --git a/apps/web/src/components/device/DeviceSetup.tsx b/apps/web/src/components/device/DeviceSetup.tsx new file mode 100644 index 000000000..f9b2e5578 --- /dev/null +++ b/apps/web/src/components/device/DeviceSetup.tsx @@ -0,0 +1,241 @@ +import type { DevicePlatform, DeviceServiceState, EnvironmentId } from "@threadlines/contracts"; +import { + devicePlatformName, + devicePlatformStatus, + deviceToolInstallMessage, + isDeviceHostBusy, +} from "@threadlines/client-runtime/deviceState"; +import { CheckIcon, CircleAlertIcon } from "lucide-react"; +import { type ReactNode, useState } from "react"; + +import { cn } from "~/lib/utils"; +import { SETTINGS_GROUP_ROW_CLASS, SettingsGroup } from "../settings/settingsLayout"; +import { Button } from "../ui/button"; +import { Spinner } from "../ui/spinner"; +import { Switch } from "../ui/switch"; +import { deviceErrorText, refreshDevices, updateDeviceSettings } from "./deviceState"; + +/** + * The Device tab's first open: what each step installs, before anything is. + * Opening the tab downloads nothing; turning on "Use devices" lets the + * computer install the device hub when devices are first listed, and agent + * access installs agent-device. + */ +export function DeviceSetup(props: { + readonly environmentId: EnvironmentId; + readonly state: DeviceServiceState; + /** Setup has started; the panel keeps showing it until Done. */ + readonly onStarted: () => void; + readonly onDone: () => void; +}) { + const { environmentId, state } = props; + const [pending, setPending] = useState<"enable" | "agent" | "check" | null>(null); + const [error, setError] = useState(null); + const busy = isDeviceHostBusy(state); + const hubVersion = state.tools?.hub.requiredVersion; + const agentVersion = state.tools?.agent.requiredVersion; + + const run = async (kind: NonNullable, work: () => Promise) => { + setPending(kind); + setError(null); + try { + await work(); + } catch (cause) { + setError(deviceErrorText(cause)); + } finally { + setPending(null); + } + }; + + const enable = (on: boolean) => + run("enable", async () => { + if (on) props.onStarted(); + await updateDeviceSettings( + environmentId, + on ? { enableDevices: true } : { enableDevices: false, agentDeviceAccess: false }, + ); + // The listing is what starts the hub, installing it the first time. + if (on) await refreshDevices(environmentId); + }); + + return ( +
+
+
+

Set up devices

+

+ See and use iOS Simulators and Android Emulators on this computer. Nothing is installed + until you turn it on. +

+
+ + +
+

+ Installs the device hub + {hubVersion ? ( + <> + {" "} + (expo-device-hub {hubVersion}) + + ) : null}{" "} + on this computer. It shows each simulator's screen here and passes your taps and + typing to it. +

+ void enable(Boolean(checked))} + /> +
+ +
+ + + + +
+

+ Either platform is enough. +

+ +
+
+ + +
+

+ Agents can open, tap through and screenshot devices too. Installs{" "} + + agent-device{agentVersion ? ` ${agentVersion}` : ""} + {" "} + and applies to agent sessions started after you turn it on. +

+ + void run("agent", () => + updateDeviceSettings(environmentId, { agentDeviceAccess: Boolean(checked) }), + ) + } + /> +
+
+ + {error ? ( +

+ {error} +

+ ) : null} + +
+ +
+
+
+ ); +} + +function Step(props: { + readonly number: number; + readonly title: string; + readonly dimmed?: boolean; + readonly children: ReactNode; +}) { + return ( +
+

+ {props.number} {props.title} +

+ {props.children} +
+ ); +} + +/** Where the device hub stands, while it installs or starts, or once it failed. */ +export function HubStatus(props: { readonly state: DeviceServiceState }) { + const { state } = props; + if (!state.enabled) return null; + const line = + state.hostStatus === "installing" + ? (state.hostStatusDetail ?? deviceToolInstallMessage("expo-device-hub", state.tools?.hub)) + : state.hostStatus === "starting" + ? "Starting the device hub…" + : state.hostStatus === "failed" + ? (state.hostStatusDetail ?? "The device hub could not start.") + : state.hostStatus === "ready" + ? "The device hub is running." + : null; + if (line === null) return null; + const failed = state.hostStatus === "failed"; + return ( +
+ {failed ? ( +
+ ); +} + +function PlatformRow(props: { + readonly platform: DevicePlatform; + readonly state: DeviceServiceState; +}) { + const availability = props.state.platforms.find( + (candidate) => candidate.platform === props.platform, + ); + // A missing platform is known as soon as the computer has looked; a working + // one only once the hub has listed its devices. + const known = + props.state.enabled && + (props.state.hostStatus === "ready" || availability?.available === false); + const status = devicePlatformStatus(props.state, props.platform); + return ( +
+ {known && status.ready ? ( +
+ ); +} diff --git a/apps/web/src/components/device/DeviceStreamView.tsx b/apps/web/src/components/device/DeviceStreamView.tsx new file mode 100644 index 000000000..f2a5f7f63 --- /dev/null +++ b/apps/web/src/components/device/DeviceStreamView.tsx @@ -0,0 +1,188 @@ +import type { DevicePlatform } from "@threadlines/contracts"; +import { type CSSProperties, type PointerEvent, useRef, useState } from "react"; + +import { cn } from "~/lib/utils"; +import { Button } from "../ui/button"; +import { DeviceLoadingView } from "./DeviceLoadingView"; +import { deviceScreenAspect, deviceScreenRotation, fitDeviceFrame } from "./deviceFrameLayout"; +import type { DeviceStream, DeviceStreamState } from "./useDeviceStream"; + +/** The bezel around the screen, in px: the frame is drawn in CSS, not a model. */ +const BEZEL = 6; +/** The bezel and the frame's 1px hairline. */ +const FRAME_INSET = BEZEL + 1; + +/** + * The device's screen in a flat rounded frame sized to its aspect. Pointer + * input becomes touches, keys go to the device while the screen has focus, + * and paste types the clipboard (Android). The frame keeps the last picture + * while a dropped stream reconnects. + */ +export function DeviceStreamView(props: { + readonly platform: DevicePlatform; + readonly deviceName: string; + readonly deviceDescription: string; + readonly stream: DeviceStream; + readonly state: DeviceStreamState; +}) { + const { stream, state, platform } = props; + const { client } = stream; + const [host, setHost] = useState({ width: 0, height: 0 }); + const hostRef = (element: HTMLDivElement | null) => { + if (!element) return; + const update = () => { + const rect = element.getBoundingClientRect(); + setHost((current) => + current.width === rect.width && current.height === rect.height + ? current + : { width: rect.width, height: rect.height }, + ); + }; + update(); + const observer = new ResizeObserver(update); + observer.observe(element); + return () => observer.disconnect(); + }; + const mountCanvas = (node: HTMLDivElement | null) => { + if (!node) return; + node.append(stream.canvas); + return () => stream.canvas.remove(); + }; + + const aspect = deviceScreenAspect(platform, state.screen); + // The screen fits inside the host less its bezel on every side. + const screenBox = fitDeviceFrame( + aspect, + Math.max(0, host.width - 2 * FRAME_INSET), + Math.max(0, host.height - 2 * FRAME_INSET), + ); + const rotation = deviceScreenRotation(platform, state.screen); + const sideways = rotation === 90 || rotation === -90; + const mediaStyle: CSSProperties = sideways + ? { + width: screenBox.height, + height: screenBox.width, + left: (screenBox.width - screenBox.height) / 2, + top: (screenBox.height - screenBox.width) / 2, + transform: `rotate(${rotation}deg)`, + } + : { + width: screenBox.width, + height: screenBox.height, + ...(rotation ? { transform: `rotate(${rotation}deg)` } : {}), + }; + + const activePointer = useRef<{ id: number; x: number; y: number } | null>(null); + const normalizedPoint = (event: PointerEvent) => { + const rect = event.currentTarget.getBoundingClientRect(); + const x = (event.clientX - rect.left) / rect.width; + const y = (event.clientY - rect.top) / rect.height; + return { x: Math.min(1, Math.max(0, x)), y: Math.min(1, Math.max(0, y)) }; + }; + const endPointer = (event: PointerEvent) => { + const pointer = activePointer.current; + if (pointer?.id !== event.pointerId) return; + activePointer.current = null; + const { x, y } = event.type === "lostpointercapture" ? pointer : normalizedPoint(event); + client.sendTouch("end", x, y); + }; + + const streaming = state.status === "streaming"; + return ( +
{ + if (event.target !== event.currentTarget) return; + if (event.metaKey) return; + // Let paste chords fire the paste event. + if ( + (event.ctrlKey && (event.key === "v" || event.key === "V")) || + (event.shiftKey && event.key === "Insert") + ) { + return; + } + event.preventDefault(); + client.sendKey(event.nativeEvent, "down"); + }} + onKeyUp={(event) => { + if (event.target !== event.currentTarget) return; + client.sendKey(event.nativeEvent, "up"); + }} + onPaste={(event) => { + if (event.target !== event.currentTarget) return; + event.preventDefault(); + client.sendText(event.clipboardData.getData("text/plain")); + }} + > +
+
{ + if (event.button !== 0 || !state.input.connected || activePointer.current !== null) + return; + event.currentTarget.setPointerCapture(event.pointerId); + (event.currentTarget.closest("[role=application]") as HTMLElement | null)?.focus(); + const { x, y } = normalizedPoint(event); + activePointer.current = { id: event.pointerId, x, y }; + client.sendTouch("begin", x, y); + }} + onPointerMove={(event) => { + if (activePointer.current?.id !== event.pointerId) return; + const { x, y } = normalizedPoint(event); + activePointer.current = { id: event.pointerId, x, y }; + client.sendTouch("move", x, y); + }} + onPointerUp={endPointer} + onPointerCancel={endPointer} + onLostPointerCapture={endPointer} + > +
+ {!streaming ? ( +
+ + {state.status === "error" ? ( + + ) : null} + +
+ ) : null} +
+
+ {streaming && !state.input.connected ? ( +

+ {state.input.detail + ? `Touch is reconnecting (${state.input.detail}).` + : "Touch is reconnecting…"} +

+ ) : null} +
+ ); +} diff --git a/apps/web/src/components/device/DeviceToolsPanel.tsx b/apps/web/src/components/device/DeviceToolsPanel.tsx new file mode 100644 index 000000000..fa1f88473 --- /dev/null +++ b/apps/web/src/components/device/DeviceToolsPanel.tsx @@ -0,0 +1,593 @@ +import type { + DeviceColorFilter, + DeviceOrientation, + DevicePermission, + DeviceSummary, + DeviceTextSize, +} from "@threadlines/contracts"; +import { XIcon } from "lucide-react"; +import { type ReactNode, useState } from "react"; + +import { cn } from "~/lib/utils"; +import { Button } from "../ui/button"; +import { Input } from "../ui/input"; +import { SegmentedControl, SegmentedControlItem } from "../ui/segmented-control"; +import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select"; +import { Spinner } from "../ui/spinner"; +import { Switch } from "../ui/switch"; +import type { DeviceControls } from "./useDeviceControls"; + +const TEXT_SIZES: ReadonlyArray<{ value: DeviceTextSize; label: string }> = [ + { value: "small", label: "Small" }, + { value: "default", label: "Default" }, + { value: "large", label: "Large" }, + { value: "extra-large", label: "Extra large" }, +]; + +const COLOR_FILTERS: ReadonlyArray<{ value: DeviceColorFilter; label: string }> = [ + { value: "none", label: "None" }, + { value: "grayscale", label: "Grayscale" }, + { value: "red-green", label: "Red / green (protanopia)" }, + { value: "green-red", label: "Green / red (deuteranopia)" }, + { value: "blue-yellow", label: "Blue / yellow (tritanopia)" }, +]; + +const ORIENTATIONS: ReadonlyArray<{ value: DeviceOrientation; label: string }> = [ + { value: "portrait", label: "Portrait" }, + { value: "landscape_left", label: "Landscape left" }, + { value: "portrait_upside_down", label: "Upside down" }, + { value: "landscape_right", label: "Landscape right" }, +]; + +const IOS_PERMISSIONS: ReadonlyArray<{ value: DevicePermission; label: string }> = [ + { value: "camera", label: "Camera" }, + { value: "microphone", label: "Microphone" }, + { value: "photos", label: "Photos" }, + { value: "contacts", label: "Contacts" }, + { value: "calendar", label: "Calendar" }, + { value: "reminders", label: "Reminders" }, + { value: "location", label: "Location" }, + { value: "notifications", label: "Notifications" }, + { value: "motion", label: "Motion" }, + { value: "media-library", label: "Media library" }, + { value: "faceid", label: "Face ID" }, +]; + +const ANDROID_PERMISSIONS: ReadonlyArray<{ value: DevicePermission; label: string }> = [ + { value: "camera", label: "Camera" }, + { value: "microphone", label: "Microphone" }, + { value: "photos", label: "Photos" }, + { value: "contacts", label: "Contacts" }, + { value: "calendar", label: "Calendar" }, + { value: "location", label: "Location" }, + { value: "notifications", label: "Notifications" }, + { value: "motion", label: "Physical activity" }, +]; + +const LOCATION_PRESETS = [ + { label: "San Francisco", latitude: 37.7749, longitude: -122.4194 }, + { label: "New York", latitude: 40.7128, longitude: -74.006 }, + { label: "London", latitude: 51.5074, longitude: -0.1278 }, + { label: "Stockholm", latitude: 59.3293, longitude: 18.0686 }, + { label: "Tokyo", latitude: 35.6762, longitude: 139.6503 }, +] as const; + +/** + * The device's settings and test actions. Every value shown is one the device + * read back; a control the platform lacks is left out or disabled. + */ +export function DeviceToolsPanel(props: { + readonly controls: DeviceControls; + readonly device: DeviceSummary; + readonly onClose: () => void; + readonly className?: string; +}) { + const { device, controls } = props; + const { detail, pending, error, foregroundApp, disabled, act } = controls; + const settings = detail?.settings; + const ios = device.platform === "ios"; + + return ( +
+
+

Tools

+ {pending ? : null} + +
+
+ {error ? ( +

+ {error} +

+ ) : null} + {detail === null && !error ? ( +

+ Reading the device's settings… +

+ ) : null} + +
+ + {foregroundApp?.id ?? "None"} + + {foregroundApp ? ( +
+ + +
+ ) : null} + act({ type: "openUrl", url })} + /> + act({ type: "launchApp", appId })} + /> +
+ +
+
+ +
+ + void act({ type: "setAppearance", value })} + > + Light + Dark + + + + act({ type: "setTextSize", value })} + /> + + {ios ? ( + <> + + void act({ type: "setLiquidGlass", value })} + > + Clear + Tinted + + + + act({ type: "setColorFilter", value })} + /> + + + ) : ( + + act({ type: "setOrientation", value })} + /> + + )} + act({ type: "setToggle", setting: "reduceMotion", value })} + /> + {ios ? ( + <> + act({ type: "setToggle", setting: "increaseContrast", value })} + /> + + act({ type: "setToggle", setting: "reduceTransparency", value }) + } + /> + act({ type: "setToggle", setting: "showBorders", value })} + /> + act({ type: "setToggle", setting: "voiceOver", value })} + /> + + ) : ( + act({ type: "setToggle", setting: "networkEnabled", value })} + /> + )} +
+ + act({ type: "setLocation", latitude, longitude })} + onClear={() => act({ type: "clearLocation" })} + /> + + + act({ type: "setPermission", appId, permission, decision }) + } + /> + + {ios ? ( +
+ + foregroundApp + ? act({ type: "sendPush", appId: foregroundApp.id, payload }) + : Promise.resolve(false) + } + /> + {!foregroundApp ? ( +

+ Open an app first; the push goes to it. +

+ ) : null} +
+ ) : null} +
+
+ ); +} + +function Section(props: { readonly title: string; readonly children: ReactNode }) { + return ( +
+

{props.title}

+ {props.children} +
+ ); +} + +function Row(props: { readonly label: string; readonly children: ReactNode }) { + return ( +
+ {props.label} +
{props.children}
+
+ ); +} + +function SwitchRow(props: { + readonly label: string; + readonly checked: boolean | undefined; + readonly disabled: boolean; + readonly onChange: (value: boolean) => Promise; +}) { + return ( + + void props.onChange(checked)} + /> + + ); +} + +function ChoiceSelect(props: { + readonly label: string; + readonly value: V | null; + readonly options: ReadonlyArray<{ readonly value: V; readonly label: string }>; + readonly disabled: boolean; + readonly placeholder?: string; + readonly onChange: (value: V) => Promise; +}) { + const current = props.options.find((option) => option.value === props.value); + return ( + + ); +} + +function SubmitRow(props: { + readonly label: string; + readonly placeholder: string; + readonly action: string; + readonly disabled: boolean; + readonly onSubmit: (value: string) => Promise; +}) { + const [value, setValue] = useState(""); + return ( +
{ + event.preventDefault(); + const trimmed = value.trim(); + if (!trimmed) return; + void props.onSubmit(trimmed).then((sent) => { + if (sent) setValue(""); + }); + }} + > + setValue(event.target.value)} + /> + +
+ ); +} + +function LocationSection(props: { + readonly disabled: boolean; + readonly canClear: boolean; + readonly onSet: (latitude: number, longitude: number) => Promise; + readonly onClear: () => Promise; +}) { + const [latitude, setLatitude] = useState(""); + const [longitude, setLongitude] = useState(""); + const parsed = { latitude: Number(latitude), longitude: Number(longitude) }; + const valid = + latitude.trim() !== "" && + longitude.trim() !== "" && + Number.isFinite(parsed.latitude) && + Number.isFinite(parsed.longitude) && + Math.abs(parsed.latitude) <= 90 && + Math.abs(parsed.longitude) <= 180; + return ( +
+
+ setLatitude(event.target.value)} + /> + setLongitude(event.target.value)} + /> +
+
+ + value={null} + disabled={props.disabled} + onValueChange={(value) => { + const preset = LOCATION_PRESETS.find((candidate) => candidate.label === value); + if (!preset) return; + setLatitude(String(preset.latitude)); + setLongitude(String(preset.longitude)); + void props.onSet(preset.latitude, preset.longitude); + }} + > + + + City… + + + + {LOCATION_PRESETS.map((preset) => ( + + {preset.label} + + ))} + + + + {props.canClear ? ( + + ) : null} +
+
+ ); +} + +function PermissionsSection(props: { + readonly permissions: ReadonlyArray<{ value: DevicePermission; label: string }>; + readonly canReset: boolean; + readonly defaultAppId: string; + readonly disabled: boolean; + readonly onDecide: ( + appId: string, + permission: DevicePermission, + decision: "grant" | "revoke" | "reset", + ) => Promise; +}) { + const [appId, setAppId] = useState(""); + const [permission, setPermission] = useState("camera"); + const resolvedAppId = appId.trim() || props.defaultAppId; + const decide = (decision: "grant" | "revoke" | "reset") => + void props.onDecide(resolvedAppId, permission, decision); + return ( +
+ setAppId(event.target.value)} + /> +
+ { + setPermission(value); + return Promise.resolve(); + }} + /> + + + {props.canReset ? ( + + ) : null} +
+
+ ); +} diff --git a/apps/web/src/components/device/DeviceWorkspace.tsx b/apps/web/src/components/device/DeviceWorkspace.tsx new file mode 100644 index 000000000..556c24ae2 --- /dev/null +++ b/apps/web/src/components/device/DeviceWorkspace.tsx @@ -0,0 +1,250 @@ +// @effect-diagnostics globalTimers:off - The download link's object URL is released on a plain timer. +import type { DeviceSummary, EnvironmentId } from "@threadlines/contracts"; +import { captureDeviceScreenshot } from "@threadlines/client-runtime/device"; +import { + CameraIcon, + ChevronLeftIcon, + EllipsisIcon, + HomeIcon, + Layers2Icon, + PowerIcon, + RotateCwIcon, + SlidersHorizontalIcon, + SquareIcon, + Undo2Icon, + XIcon, +} from "lucide-react"; +import { type ReactNode, useRef, useState } from "react"; + +import { cn } from "~/lib/utils"; +import { Button } from "../ui/button"; +import { Menu, MenuItem, MenuPopup, MenuSeparator, MenuTrigger } from "../ui/menu"; +import { deviceErrorText } from "./deviceState"; +import { DeviceStreamView } from "./DeviceStreamView"; +import { DeviceToolsPanel } from "./DeviceToolsPanel"; +import { useDeviceControls } from "./useDeviceControls"; +import { useDeviceStream } from "./useDeviceStream"; + +const DOWNLOAD_URL_LIFETIME_MS = 60_000; + +/** + * One open device: a toolbar of its hardware buttons and actions, the live + * screen, and the Tools drawer. Mount it keyed by computer and device. + */ +export function DeviceWorkspace(props: { + readonly environmentId: EnvironmentId; + readonly device: DeviceSummary; + /** The tab is on screen; nothing streams while it is not. */ + readonly visible: boolean; + readonly bootedByThreadlines: boolean; + readonly onBack: () => void; + readonly onClose: () => void; + readonly onPowerOff: () => void; +}) { + const { device, environmentId, visible } = props; + const ios = device.platform === "ios"; + const { stream, state } = useDeviceStream({ + environmentId, + platform: device.platform, + deviceId: device.id, + active: visible, + }); + const [toolsOpen, setToolsOpen] = useState(false); + // Read while on screen, not only with Tools open: rotating Android goes through it too. + const controls = useDeviceControls({ environmentId, device, visible }); + const [screenshotPending, setScreenshotPending] = useState(false); + const [notice, setNotice] = useState(null); + // A capture always finishes, even if the panel hides meanwhile; the user asked for it. + const capturing = useRef(false); + + const inputReady = state.input.connected; + const press = (button: "home" | "back" | "recents" | "appSwitcher") => + stream.client.pressButton(button); + const saveScreenshot = () => { + if (capturing.current) return; + capturing.current = true; + setScreenshotPending(true); + setNotice(null); + captureDeviceScreenshot(stream.transport, { platform: device.platform, deviceId: device.id }) + .then((image) => { + const url = URL.createObjectURL(image); + const link = document.createElement("a"); + link.href = url; + link.download = `${device.name.replace(/[^a-z0-9-]/gi, "-")}-${Date.now()}.png`; + document.body.append(link); + link.click(); + link.remove(); + // The browser reads the blob after the click returns. + window.setTimeout(() => URL.revokeObjectURL(url), DOWNLOAD_URL_LIFETIME_MS); + }) + .catch((cause: unknown) => setNotice(deviceErrorText(cause))) + .finally(() => { + capturing.current = false; + setScreenshotPending(false); + }); + }; + + return ( +
+
+ + + +
+

{device.name}

+

+ {device.version} + {props.bootedByThreadlines ? " · started by Threadlines" : ""} +

+
+ press("home")}> + + + {ios ? ( + press("appSwitcher")} + > + + + ) : ( + <> + press("back")}> + + + press("recents")}> + + + + )} + {ios ? ( + + + + ) : ( + + + } + > + + + + {(["portrait", "landscape_left"] as const).map((value) => ( + void controls.act({ type: "setOrientation", value })} + > + {value === "portrait" ? "Portrait" : "Landscape"} + + ))} + + + )} + + + + setToolsOpen((open) => !open)} + > + + + + } + > + + + + + + Close device + + + + + Power off + + + +
+ {notice || (controls.error && !toolsOpen) ? ( +

+ {notice ?? controls.error} +

+ ) : null} + {state.mode === "stills" ? ( +

+ Showing still frames. The desktop app, https or the relay gives full video. +

+ ) : null} +
+
+ +
+ {toolsOpen ? ( + setToolsOpen(false)} + className={cn( + // Beside the screen when the panel is wide, floating over it when not. + "absolute inset-y-0 right-0 z-20 w-full max-w-72 border-l border-border shadow-lg", + "@[560px]:static @[560px]:w-72 @[560px]:shrink-0 @[560px]:shadow-none", + )} + /> + ) : null} +
+
+ ); +} + +function ToolbarButton(props: { + readonly label: string; + readonly disabled?: boolean; + readonly pressed?: boolean; + readonly onClick: () => void; + readonly children: ReactNode; +}) { + return ( + + ); +} diff --git a/apps/web/src/components/device/deviceFrameLayout.ts b/apps/web/src/components/device/deviceFrameLayout.ts new file mode 100644 index 000000000..771b39222 --- /dev/null +++ b/apps/web/src/components/device/deviceFrameLayout.ts @@ -0,0 +1,36 @@ +import type { DevicePlatform } from "@threadlines/contracts"; +import type { DeviceScreenSize } from "@threadlines/client-runtime/device"; + +/** The largest box of `aspect` (width / height) that fits `width` × `height`. */ +export function fitDeviceFrame(aspect: number, width: number, height: number) { + if (width <= 0 || height <= 0 || !Number.isFinite(aspect) || aspect <= 0) { + return { width: 0, height: 0 }; + } + const byHeight = { width: height * aspect, height }; + return byHeight.width <= width ? byHeight : { width, height: width / aspect }; +} + +/** The screen's width / height as shown, before the first frame a phone's usual shape. */ +export function deviceScreenAspect(platform: DevicePlatform, screen: DeviceScreenSize | null) { + if (!screen) return platform === "ios" ? 9 / 19.5 : 9 / 20; + const landscape = + screen.orientation === "landscape_left" || screen.orientation === "landscape_right"; + const long = Math.max(screen.width, screen.height); + const short = Math.min(screen.width, screen.height); + return landscape ? long / short : short / long; +} + +/** serve-sim sends portrait framebuffers even when iOS is landscape; the view turns them. */ +export function deviceScreenRotation(platform: DevicePlatform, screen: DeviceScreenSize | null) { + if (platform !== "ios" || !screen || screen.width > screen.height) return 0; + switch (screen.orientation) { + case "landscape_left": + return 90; + case "landscape_right": + return -90; + case "portrait_upside_down": + return 180; + default: + return 0; + } +} diff --git a/apps/web/src/components/device/deviceState.ts b/apps/web/src/components/device/deviceState.ts new file mode 100644 index 000000000..cf6034b8c --- /dev/null +++ b/apps/web/src/components/device/deviceState.ts @@ -0,0 +1,311 @@ +/** + * The device state of each computer on the web side (docs/design/devices.md): + * one `subscribeDeviceState` stream per computer, shared by every view that + * follows it (the Device tab, Settings → Devices, the thread route watching + * for an agent opening a device) and dropped when the last one leaves. The + * stream follows a reconnect on its own; the last state stays until then. + * + * Also the device a thread's Device tab shows, and the calls the tab makes, + * each naming the computer the device is on. + */ +import type { + DeviceActionInput, + DeviceCloseInput, + DeviceDetail, + DeviceOpenInput, + DeviceServiceState, + DeviceSession, + DeviceShutdownInput, + DeviceToolName, + EnvironmentApi, + EnvironmentId, + ServerSettingsPatch, +} from "@threadlines/contracts"; +import type { DeviceChannelApi } from "@threadlines/client-runtime/device"; +import { newAgentDeviceSessions } from "@threadlines/client-runtime/deviceState"; +import { useCallback, useEffect, useSyncExternalStore } from "react"; + +import { isEnvironmentApiAvailable, readEnvironmentApi } from "../../environmentApi"; +import { usePrimaryEnvironmentDescriptor } from "../../environments/primary"; +import { + readEnvironmentConnection, + readEnvironmentDescriptor, + subscribeEnvironmentConnections, + useSavedEnvironmentRuntimeStore, +} from "../../environments/runtime"; + +/** Whether a computer's server has devices (capability `devices`), as it last said. */ +export function environmentSupportsDevices(environmentId: EnvironmentId): boolean { + return readEnvironmentDescriptor(environmentId)?.capabilities.devices === true; +} + +/** `environmentSupportsDevices`, again whenever a computer's descriptor changes. */ +export function useDevicesSupported(environmentId: EnvironmentId | null): boolean { + const primaryDescriptor = usePrimaryEnvironmentDescriptor(); + const linked = useSavedEnvironmentRuntimeStore((state) => + environmentId === null + ? undefined + : state.byId[environmentId]?.descriptor?.capabilities.devices, + ); + if (environmentId === null) return false; + return primaryDescriptor?.environmentId === environmentId + ? primaryDescriptor.capabilities.devices === true + : linked === true; +} + +interface Followed { + refs: number; + /** What the subscription runs on: the connection, or a stand-in API in tests. */ + connection: object | null; + unsubscribe: (() => void) | null; +} + +const STAND_IN_CONNECTION = {}; +let stopWatchingConnections: (() => void) | null = null; + +const followed = new Map(); +let states: Readonly> = {}; +const listeners = new Set<() => void>(); +const agentOpenListeners = new Set< + (environmentId: EnvironmentId, session: DeviceSession) => void +>(); + +function emit() { + for (const listener of listeners) listener(); +} + +/** + * Takes a state from the stream (always: it is in order, and a restarted + * server starts its revision again) or from a call's answer (only when it is + * not older than the one held). + */ +function receive(environmentId: EnvironmentId, next: DeviceServiceState, fromStream: boolean) { + const previous = states[environmentId]; + if (!fromStream && previous !== undefined && next.revision < previous.revision) return; + states = { ...states, [environmentId]: next }; + emit(); + if (previous === undefined) return; + for (const threadId of new Set(next.sessions.map((session) => session.threadId))) { + for (const session of newAgentDeviceSessions(previous, next, threadId)) { + for (const listener of agentOpenListeners) listener(environmentId, session); + } + } +} + +/** Subscribes each followed computer on its current connection, once. */ +function syncConnections() { + for (const [environmentId, entry] of followed) { + const connection = + readEnvironmentConnection(environmentId) ?? + (isEnvironmentApiAvailable(environmentId) ? STAND_IN_CONNECTION : null); + if (entry.connection === connection) continue; + entry.unsubscribe?.(); + entry.unsubscribe = null; + entry.connection = connection; + const api = connection === null ? undefined : readEnvironmentApi(environmentId); + if (api === undefined) continue; + entry.unsubscribe = api.device.subscribeState((state) => { + // A late answer from a subscription already dropped is not this one's. + if (followed.get(environmentId) !== entry || entry.connection !== connection) return; + receive(environmentId, state, true); + }); + } +} + +function follow(environmentId: EnvironmentId): () => void { + let entry = followed.get(environmentId); + if (entry === undefined) { + entry = { refs: 0, connection: null, unsubscribe: null }; + followed.set(environmentId, entry); + } + stopWatchingConnections ??= subscribeEnvironmentConnections(syncConnections); + syncConnections(); + entry.refs += 1; + const held = entry; + let released = false; + return () => { + if (released) return; + released = true; + held.refs -= 1; + if (held.refs > 0) return; + held.unsubscribe?.(); + followed.delete(environmentId); + if (followed.size === 0) { + stopWatchingConnections?.(); + stopWatchingConnections = null; + } + }; +} + +function subscribeStates(listener: () => void) { + listeners.add(listener); + return () => { + listeners.delete(listener); + }; +} + +/** + * A computer's device state, followed while mounted. Null until its first + * state arrives, and always for a computer whose server lacks `devices` + * (pass `supported` false). + */ +export function useDeviceState( + environmentId: EnvironmentId | null, + supported: boolean, +): DeviceServiceState | null { + const following = supported && environmentId !== null; + useEffect( + () => (following && environmentId !== null ? follow(environmentId) : undefined), + [environmentId, following], + ); + const read = useCallback( + () => (environmentId === null ? undefined : states[environmentId]), + [environmentId], + ); + const state = useSyncExternalStore(subscribeStates, read, read); + return following ? (state ?? null) : null; +} + +/** Calls `listener` whenever an agent opens a device for a thread on a followed computer. */ +export function onAgentOpenedDevice( + listener: (environmentId: EnvironmentId, session: DeviceSession) => void, +): () => void { + agentOpenListeners.add(listener); + return () => { + agentOpenListeners.delete(listener); + }; +} + +// ── The device each thread's tab shows ───────────────────────────────── + +let selected: Readonly> = {}; +const selectionListeners = new Set<() => void>(); + +/** Points a thread's Device tab at a device, or back at the picker (null). */ +export function selectThreadDevice(threadKey: string, deviceId: string | null) { + if ((selected[threadKey] ?? null) === deviceId) return; + if (deviceId === null) { + const { [threadKey]: _dropped, ...rest } = selected; + selected = rest; + } else { + selected = { ...selected, [threadKey]: deviceId }; + } + for (const listener of selectionListeners) listener(); +} + +export function useSelectedThreadDevice(threadKey: string | null): string | null { + const read = useCallback( + () => (threadKey === null ? null : (selected[threadKey] ?? null)), + [threadKey], + ); + return useSyncExternalStore( + (listener) => { + selectionListeners.add(listener); + return () => { + selectionListeners.delete(listener); + }; + }, + read, + read, + ); +} + +// ── Calls ────────────────────────────────────────────────────────────── + +function deviceApi(environmentId: EnvironmentId): EnvironmentApi["device"] { + const api = readEnvironmentApi(environmentId); + if (api === undefined) { + throw new Error("That computer is offline. Try again when it is back."); + } + return api.device; +} + +/** What went wrong with a device call, in words the panel can show. */ +export function deviceErrorText(error: unknown): string { + const message = + error instanceof Error + ? error.message + : typeof (error as { message?: unknown } | null)?.message === "string" + ? (error as { message: string }).message + : ""; + return message.trim().length > 0 ? message : "That did not work. Try again."; +} + +export async function refreshDevices(environmentId: EnvironmentId): Promise { + receive(environmentId, await deviceApi(environmentId).list({ refresh: true }), false); +} + +export async function updateDeviceTool( + environmentId: EnvironmentId, + tool: DeviceToolName, +): Promise { + receive(environmentId, await deviceApi(environmentId).updateTool({ tool }), false); +} + +export function openDevice(environmentId: EnvironmentId, input: DeviceOpenInput) { + return deviceApi(environmentId).open(input); +} + +export function closeDevice(environmentId: EnvironmentId, input: DeviceCloseInput) { + return deviceApi(environmentId).close(input); +} + +export function shutdownDevice(environmentId: EnvironmentId, input: DeviceShutdownInput) { + return deviceApi(environmentId).shutdown(input); +} + +export function readDeviceDetail( + environmentId: EnvironmentId, + deviceId: string, +): Promise { + return deviceApi(environmentId).detail({ deviceId }); +} + +export function runDeviceAction( + environmentId: EnvironmentId, + input: DeviceActionInput, +): Promise { + return deviceApi(environmentId).action(input); +} + +/** "Use devices" and "Agents can use devices" on that computer, through its settings. */ +export async function updateDeviceSettings( + environmentId: EnvironmentId, + patch: Pick, +): Promise { + const api = readEnvironmentApi(environmentId); + if (api === undefined) { + throw new Error("That computer is offline. Try again when it is back."); + } + await api.settings.update(patch); +} + +/** + * The device channel of a computer, read again for every channel opened, so + * a channel opened after a reconnect goes over the new connection. + */ +export function deviceChannelApi(environmentId: EnvironmentId): DeviceChannelApi { + return { + open: (input, listener, onClosed) => { + const api = readEnvironmentApi(environmentId); + if (api === undefined) { + queueMicrotask(() => onClosed(new Error("That computer is offline."))); + return () => {}; + } + return api.device.channel.open(input, listener, onClosed); + }, + send: (input) => deviceApi(environmentId).channel.send(input), + credit: (input) => deviceApi(environmentId).channel.credit(input), + }; +} + +export function resetDeviceStateForTests() { + for (const entry of followed.values()) entry.unsubscribe?.(); + followed.clear(); + stopWatchingConnections?.(); + stopWatchingConnections = null; + states = {}; + selected = {}; + emit(); + for (const listener of selectionListeners) listener(); +} diff --git a/apps/web/src/components/device/useDeviceControls.ts b/apps/web/src/components/device/useDeviceControls.ts new file mode 100644 index 000000000..c67bceef9 --- /dev/null +++ b/apps/web/src/components/device/useDeviceControls.ts @@ -0,0 +1,83 @@ +import type { + DeviceActionInput, + DeviceDetail, + DeviceSummary, + EnvironmentId, +} from "@threadlines/contracts"; +import { useEffect, useRef, useState } from "react"; + +import { deviceErrorText, readDeviceDetail, runDeviceAction } from "./deviceState"; + +type ActionBody = DeviceActionInput extends infer A + ? A extends { readonly type: string } + ? Omit + : never + : never; + +/** + * The Tools drawer's values and actions for one device. Reads the values when + * it becomes visible; every action answers with the values read back, so the + * drawer only ever shows what the device says. Actions run one at a time. + * Mount it keyed by device. + */ +export function useDeviceControls(input: { + readonly environmentId: EnvironmentId; + readonly device: DeviceSummary; + readonly visible: boolean; +}) { + const { environmentId, device, visible } = input; + const [detail, setDetail] = useState(null); + const [error, setError] = useState(null); + const [pending, setPending] = useState(false); + // Hiding does not cancel a command on the device, so actions stay one at a + // time until each settles rather than letting a reopened drawer race it. + const busy = useRef(false); + + useEffect(() => { + if (!visible) return; + let current = true; + readDeviceDetail(environmentId, device.id).then( + (read) => { + if (!current) return; + setDetail(read); + setError(null); + }, + (cause: unknown) => { + if (current) setError(deviceErrorText(cause)); + }, + ); + return () => { + current = false; + }; + }, [device.id, environmentId, visible]); + + const act = async (body: ActionBody): Promise => { + if (busy.current || detail === null || !visible) return false; + busy.current = true; + setPending(true); + setError(null); + try { + setDetail( + await runDeviceAction(environmentId, { ...body, deviceId: device.id } as DeviceActionInput), + ); + return true; + } catch (cause) { + setError(deviceErrorText(cause)); + return false; + } finally { + busy.current = false; + setPending(false); + } + }; + + return { + detail, + pending, + error, + act, + disabled: pending || detail === null || !visible, + foregroundApp: detail?.foregroundApp ?? null, + }; +} + +export type DeviceControls = ReturnType; diff --git a/apps/web/src/components/device/useDeviceStream.ts b/apps/web/src/components/device/useDeviceStream.ts new file mode 100644 index 000000000..90f223f46 --- /dev/null +++ b/apps/web/src/components/device/useDeviceStream.ts @@ -0,0 +1,120 @@ +import type { DevicePlatform, EnvironmentId } from "@threadlines/contracts"; +import { + createCanvasFrameSink, + createDeviceHubTransport, + createDeviceStreamClient, + type DeviceHubTransport, + type DeviceScreenSize, + type DeviceStreamClient, + type DeviceStreamStatus, + type DeviceVideoMode, +} from "@threadlines/client-runtime/device"; +import { useEffect, useState, useSyncExternalStore } from "react"; + +import { deviceChannelApi } from "./deviceState"; + +export interface DeviceStreamState { + readonly status: DeviceStreamStatus; + readonly detail: string | undefined; + readonly screen: DeviceScreenSize | null; + readonly mode: DeviceVideoMode; + readonly input: { readonly connected: boolean; readonly detail?: string }; +} + +const CONNECTING: DeviceStreamState = { + status: "connecting", + detail: undefined, + screen: null, + mode: "video", + input: { connected: false }, +}; + +/** One device's stream for one view: its canvas, its client and what it last reported. */ +export interface DeviceStream { + /** The stream draws here; the view mounts it. */ + readonly canvas: HTMLCanvasElement; + readonly client: DeviceStreamClient; + readonly transport: DeviceHubTransport; + readonly subscribe: (listener: () => void) => () => void; + readonly getState: () => DeviceStreamState; + readonly start: () => void; + readonly stop: () => void; + /** Drops the channels and opens fresh ones, with a decoder waiting for a keyframe. */ + readonly reconnect: () => void; +} + +export function createDeviceStream(input: { + readonly environmentId: EnvironmentId; + readonly platform: DevicePlatform; + readonly deviceId: string; + readonly transport?: DeviceHubTransport; +}): DeviceStream { + const canvas = document.createElement("canvas"); + canvas.style.display = "block"; + canvas.style.width = "100%"; + canvas.style.height = "100%"; + const transport = + input.transport ?? createDeviceHubTransport(deviceChannelApi(input.environmentId)); + const listeners = new Set<() => void>(); + let state = CONNECTING; + const update = (patch: Partial) => { + state = { ...state, ...patch }; + for (const listener of listeners) listener(); + }; + const client = createDeviceStreamClient( + { platform: input.platform, deviceId: input.deviceId, transport }, + createCanvasFrameSink(canvas), + { + onStatus: (status, detail) => update({ status, detail }), + onScreen: (screen) => update({ screen }), + onInputConnected: (connected, detail) => + update({ input: detail === undefined ? { connected } : { connected, detail } }), + onVideoMode: (mode) => update({ mode }), + }, + ); + return { + canvas, + client, + transport, + subscribe: (listener) => { + listeners.add(listener); + return () => { + listeners.delete(listener); + }; + }, + getState: () => state, + start: () => { + update(CONNECTING); + client.start(); + }, + stop: () => client.stop(), + reconnect: () => { + client.stop(); + update(CONNECTING); + client.start(); + }, + }; +} + +/** + * A device's live screen and input while `active`: only the device on screen + * streams, so a hidden tab, another thread or a closed panel holds no channel. + * Each start opens fresh channels with a fresh decoder. Mount it keyed by the + * computer and device. + */ +export function useDeviceStream(input: { + readonly environmentId: EnvironmentId; + readonly platform: DevicePlatform; + readonly deviceId: string; + readonly active: boolean; +}): { readonly stream: DeviceStream; readonly state: DeviceStreamState } { + const { environmentId, platform, deviceId, active } = input; + const [stream] = useState(() => createDeviceStream({ environmentId, platform, deviceId })); + useEffect(() => { + if (!active) return; + stream.start(); + return () => stream.stop(); + }, [active, stream]); + const state = useSyncExternalStore(stream.subscribe, stream.getState, stream.getState); + return { stream, state }; +} diff --git a/apps/web/src/diffRouteSearch.test.ts b/apps/web/src/diffRouteSearch.test.ts index 3eaeddc6f..86035cccc 100644 --- a/apps/web/src/diffRouteSearch.test.ts +++ b/apps/web/src/diffRouteSearch.test.ts @@ -201,6 +201,7 @@ describe("closeRightPanelSearchParams", () => { agents: "0", media: "0", threads: "0", + device: "0", diffTurnId: undefined, diffFilePath: undefined, }); diff --git a/apps/web/src/diffRouteSearch.ts b/apps/web/src/diffRouteSearch.ts index aa2527e49..b1b12ef84 100644 --- a/apps/web/src/diffRouteSearch.ts +++ b/apps/web/src/diffRouteSearch.ts @@ -4,7 +4,7 @@ import { MessageId, TurnId } from "@threadlines/contracts"; * The right sidebar's tabs are filed under the params they have always used: * `sourceControl=1` is the Source tab, `diff=1` the Diff tab, `pullRequest=1` * the Pull request tab, `agents=1` the Agents tab, `media=1` the Media tab, - * `threads=1` the Threads tab. At most one reads as open, + * `threads=1` the Threads tab, `device=1` the Device tab. At most one reads as open, * and that one is the active tab; an explicit `0` means the sidebar is closed. * `pullRequestNumber` names which of the thread's pull requests the Pull * request tab shows, the way the diff params name what the Diff tab shows. @@ -18,6 +18,7 @@ export interface DiffRouteSearch { agents?: "1" | "0" | undefined; media?: "1" | "0" | undefined; threads?: "1" | "0" | undefined; + device?: "1" | "0" | undefined; diffTurnId?: TurnId | undefined; diffFilePath?: string | undefined; focusMessageId?: MessageId | undefined; @@ -85,6 +86,7 @@ type RightPanelSearchKey = | "agents" | "media" | "threads" + | "device" | "diffTurnId" | "diffFilePath"; @@ -97,6 +99,7 @@ interface ClearedRightPanelSearchParams { agents?: undefined; media?: undefined; threads?: undefined; + device?: undefined; diffTurnId?: undefined; diffFilePath?: undefined; } @@ -117,6 +120,7 @@ export function stripRightPanelSearchParams>( agents: _agents, media: _media, threads: _threads, + device: _device, diffTurnId: _diffTurnId, diffFilePath: _diffFilePath, ...rest @@ -131,6 +135,7 @@ export function stripRightPanelSearchParams>( agents: undefined, media: undefined, threads: undefined, + device: undefined, diffTurnId: undefined, diffFilePath: undefined, } as Omit & ClearedRightPanelSearchParams; @@ -148,6 +153,7 @@ export function closeRightPanelSearchParams>(p agents: "0" as const, media: "0" as const, threads: "0" as const, + device: "0" as const, }; } @@ -159,7 +165,8 @@ export function closeRightPanelSearchParams>(p export function preserveRightPanelSearchParamsForDraftNavigation>( params: T, ) { - const { sourceControl, pullRequest, agents, media, threads } = parseDiffRouteSearch(params); + const { sourceControl, pullRequest, agents, media, threads, device } = + parseDiffRouteSearch(params); const stripped = stripRightPanelSearchParams(params); return { ...stripped, @@ -168,6 +175,7 @@ export function preserveRightPanelSearchParamsForDraftNavigation): DiffRoute : !diff && isExplicitClosedValue(search.threads) ? "0" : undefined; + const device = + !diff && isDiffOpenValue(search.device) + ? "1" + : !diff && isExplicitClosedValue(search.device) + ? "0" + : undefined; const diffTurnIdRaw = diff && diffMode !== "workingTree" ? normalizeSearchString(search.diffTurnId) : undefined; const diffTurnId = diffTurnIdRaw ? TurnId.make(diffTurnIdRaw) : undefined; @@ -226,6 +240,7 @@ export function parseDiffRouteSearch(search: Record): DiffRoute ...(agents ? { agents } : {}), ...(media ? { media } : {}), ...(threads ? { threads } : {}), + ...(device ? { device } : {}), ...(diffTurnId ? { diffTurnId } : {}), ...(diffFilePath ? { diffFilePath } : {}), ...(focusMessageId ? { focusMessageId } : {}), diff --git a/apps/web/src/rightPanelTabs.ts b/apps/web/src/rightPanelTabs.ts index 1192a49eb..7765c9f9e 100644 --- a/apps/web/src/rightPanelTabs.ts +++ b/apps/web/src/rightPanelTabs.ts @@ -2,7 +2,7 @@ * The right sidebar's tab model. * * The sidebar is one browser-style panel with a tab strip: Source, Diff, - * Pull request, Agents, Media and Threads are surfaces you open into it, each + * Pull request, Agents, Media, Threads and Device are surfaces you open into it, each * at most once. Two things have to * agree about it — the route that renders the panel and the chat header that * toggles it — so the state lives here rather than in either of them. @@ -10,7 +10,7 @@ * Split of authority: * * - The URL owns the ACTIVE tab, mapped onto the params that already existed - * (`sourceControl=1`, `diff=1`, `agents=1`, `media=1`, `threads=1`). Deep links keep working and an + * (`sourceControl=1`, `diff=1`, `agents=1`, `media=1`, `threads=1`, `device=1`). Deep links keep working and an * explicit `0` still means closed. * - This module owns what the URL cannot say: which tabs are in the strip, * whether the sidebar is showing at all (it can be open on the launcher with @@ -35,7 +35,8 @@ export type RightPanelTab = | "pullRequest" | "agents" | "media" - | "threads"; + | "threads" + | "device"; /** Launcher order and `+` menu order. The strip itself is ordered by when each * tab was opened, like a browser's, so this is only how the fixed menus list @@ -47,6 +48,7 @@ export const RIGHT_PANEL_TAB_ORDER: ReadonlyArray = [ "agents", "media", "threads", + "device", ]; export interface RightPanelSurface { @@ -97,17 +99,27 @@ export const RIGHT_PANEL_SURFACES: Readonly { if (input.isGeneralChat) { return input.isDraft ? [] : ["agents"]; @@ -115,7 +127,9 @@ export function availableRightPanelTabs(input: { if (input.isDraft) { return ["sourceControl", "diff"]; } - return RIGHT_PANEL_TAB_ORDER; + return input.devicesSupported === true + ? RIGHT_PANEL_TAB_ORDER + : RIGHT_PANEL_TAB_ORDER.filter((tab) => tab !== "device"); } /** What the Diff tab is pointed at, as the search params that express it. */ @@ -353,6 +367,9 @@ export function activeRightPanelTabFromSearch(search: DiffRouteSearch): RightPan if (search.threads === "1") { return "threads"; } + if (search.device === "1") { + return "device"; + } if (search.sourceControl === "1") { return "sourceControl"; } @@ -370,7 +387,8 @@ export function isRightPanelClosedInSearch(search: DiffRouteSearch): boolean { search.pullRequest === "0" || search.agents === "0" || search.media === "0" || - search.threads === "0" + search.threads === "0" || + search.device === "0" ); } @@ -407,6 +425,7 @@ export function rightPanelTabSearchParams>( agents: tab === "agents" ? ("1" as const) : ("0" as const), media: tab === "media" ? ("1" as const) : ("0" as const), threads: tab === "threads" ? ("1" as const) : ("0" as const), + device: tab === "device" ? ("1" as const) : ("0" as const), }; } diff --git a/apps/web/src/routes/_chat.$environmentId.$threadId.tsx b/apps/web/src/routes/_chat.$environmentId.$threadId.tsx index b47ee97da..b173e7d26 100644 --- a/apps/web/src/routes/_chat.$environmentId.$threadId.tsx +++ b/apps/web/src/routes/_chat.$environmentId.$threadId.tsx @@ -34,6 +34,12 @@ import { type ThreadPullRequestLinks, } from "../components/chat/ThreadPullRequestLinkContext"; import { ChatRightPanel } from "../components/ChatRightPanel"; +import { + onAgentOpenedDevice, + selectThreadDevice, + useDevicesSupported, + useDeviceState, +} from "../components/device/deviceState"; import { useAgentsPanelSource } from "../agentsPanelStore"; import { preloadDiffPanel, schedulePreloadDiffPanel } from "../diffPanelPreload"; import { useMediaQuery } from "../hooks/useMediaQuery"; @@ -149,6 +155,9 @@ const DiffPanel = lazy(() => import("../components/DiffPanel")); const MediaPanel = lazy(() => import("../components/media/MediaPanel").then((module) => ({ default: module.MediaPanel })), ); +const DevicePanel = lazy(() => + import("../components/device/DevicePanel").then((module) => ({ default: module.DevicePanel })), +); const FileViewerOverlay = lazy(() => import("../components/file-viewer/FileViewerOverlay")); /** Mount the heavy viewer chunk only after the first open request. */ @@ -362,14 +371,16 @@ function ChatThreadRouteView() { [navigate, threadRef], ); + const devicesSupported = useDevicesSupported(threadRef?.environmentId ?? null); const availableTabs = useMemo( () => availableRightPanelTabs({ isGeneralChat: isGeneralChatThread, // A thread the server has not accepted yet has no turn, so no agents. isDraft: !serverThread, + devicesSupported, }), - [isGeneralChatThread, serverThread], + [devicesSupported, isGeneralChatThread, serverThread], ); // The thread's pull request, for the Pull request tab and the launcher row @@ -623,6 +634,23 @@ function ChatThreadRouteView() { const agentsOpen = activeTab === "agents"; const mediaOpen = activeTab === "media"; const threadsOpen = activeTab === "threads"; + const deviceOpen = activeTab === "device"; + // An agent opening a device brings the Device tab up on it for whoever is + // watching the thread (docs/design/devices.md). The device state is + // followed while a thread whose computer has devices is on screen. + const deviceTabAvailable = availableTabs.includes("device"); + useDeviceState(threadRef?.environmentId ?? null, deviceTabAvailable); + useEffect(() => { + if (!threadRef || !deviceTabAvailable || currentThreadKey === null) return; + return onAgentOpenedDevice((environmentId, session) => { + if (environmentId !== threadRef.environmentId || session.threadId !== threadRef.threadId) { + return; + } + selectThreadDevice(currentThreadKey, session.deviceId); + focusRightPanelTab(currentThreadKey, "device"); + navigateToTab("device"); + }); + }, [currentThreadKey, deviceTabAvailable, navigateToTab, threadRef]); // A published source outlives the navigation for a frame or two, so the rail // only trusts one that names the thread it is rendering. const agentsSource = @@ -1098,6 +1126,20 @@ function ChatThreadRouteView() { />
) : null} + {openTabs.includes("device") && serverThread && currentThreadKey !== null ? ( +
+ Loading devices…

} + > + +
+
+ ) : null} {openTabs.includes("sourceControl") ? (
Date: Fri, 20 Nov 2026 01:36:51 -0500 Subject: [PATCH 09/49] feat(web): Settings > Devices, the app guide and Open device Settings gets a Devices page with Use devices, Agents can use devices (user-only in the app guide) and the versions of the device hub and agent-device with Update. The app guide gets places for the page and its rows and a device-tab topic. The command palette gets Open device. --- apps/server/src/guide/topics.ts | 24 ++ .../src/components/CommandPaletteDialog.tsx | 34 ++- .../components/settings/DevicesSettings.tsx | 270 ++++++++++++++++++ .../settings/SettingsSidebarNav.test.ts | 1 + .../components/settings/settingsNavigation.ts | 6 + .../src/components/settings/settingsPaths.ts | 1 + .../settings/settingsPlaceAvailability.tsx | 12 + .../settings/settingsRowRegistry.tsx | 4 + apps/web/src/routeTree.gen.ts | 21 ++ apps/web/src/routes/settings.devices.tsx | 7 + packages/shared/src/appPlaces.ts | 49 ++++ 11 files changed, 428 insertions(+), 1 deletion(-) create mode 100644 apps/web/src/components/settings/DevicesSettings.tsx create mode 100644 apps/web/src/routes/settings.devices.tsx diff --git a/apps/server/src/guide/topics.ts b/apps/server/src/guide/topics.ts index 911656e47..1a901cb66 100644 --- a/apps/server/src/guide/topics.ts +++ b/apps/server/src/guide/topics.ts @@ -381,6 +381,30 @@ The project menu's Project brief and notes, the command palette and the lead cha Show device toolbar, in the browser's options menu (⋮), sets the page's size: Responsive fills the panel, and a Phone, Tablet, Laptop or Desktop size shrinks to fit it. New tabs open at {place:general.browserPageSize}, which the size menu can also set. Words typed in the browser's address bar are searched for, with the search engine set in {place:general.browserSearchEngine}. An address loads as it always did.`, + }, + { + id: "device-tab", + title: "The Device tab", + keywords: [ + "device", + "simulator", + "emulator", + "ios", + "android", + "iphone", + "xcode", + "android sdk", + "app preview", + "device_open", + "agent-device", + ], + body: `The Device tab in the right panel shows a live iOS Simulator or Android Emulator running on the thread's computer. You tap, swipe and type on its screen as on a phone. The toolbar above it has Home, the app switcher, Back and Recents on Android, rotate, screenshot and power off. The Tools drawer sets appearance, text size, accessibility options, location and permissions, opens a URL, launches or quits an app, shakes the device, sends a test push, and on Android sets the network. + +Open it from the right panel's + menu or its launcher, or with Open device in the command palette. It shows in project threads, not General Chats. + +iOS needs a Mac with Xcode, and Android needs the Android SDK with the emulator. Opening the tab downloads nothing: {place:devices.enable} has to be on first, and then the device hub installs the first time the tab or an agent asks. {place:devices.tools} shows the installed versions and an Update button when a newer one is needed. + +Agents get device tools only while {place:devices.agentAccess} is on, and only in sessions started after it was turned on. An agent that opens a device brings up the Device tab for anyone watching the thread. Closing a device leaves the simulator running; power off shuts it down.`, }, { id: "file-viewer", diff --git a/apps/web/src/components/CommandPaletteDialog.tsx b/apps/web/src/components/CommandPaletteDialog.tsx index b1bdbb8fd..30c1538be 100644 --- a/apps/web/src/components/CommandPaletteDialog.tsx +++ b/apps/web/src/components/CommandPaletteDialog.tsx @@ -73,6 +73,7 @@ import { PinOffIcon, PlugIcon, CameraIcon, + SmartphoneIcon, } from "lucide-react"; import { useCallback, @@ -154,7 +155,11 @@ import { import { useGitStatus } from "../lib/gitStatusState"; import { useScheduledTaskEnvironments } from "../lib/scheduledTasksState"; import { resolveThreadPullRequest, threadViewBranch } from "./pull-requests/pullRequests.logic"; -import { focusRightPanelTab, pullRequestTabSearchParams } from "../rightPanelTabs"; +import { + focusRightPanelTab, + pullRequestTabSearchParams, + rightPanelTabSearchParams, +} from "../rightPanelTabs"; import { cn, isMacPlatform, newCommandId, newProjectId, newThreadId } from "../lib/utils"; import { selectProjectsAcrossEnvironments, @@ -201,6 +206,7 @@ import { APP_VERSION } from "../branding"; import { useProjectSetupTargets } from "../environments/projectSetups/projectSetupTargets"; import { useMoveThreadsOffered, useThreadMoveOffered } from "../environments/links/threadMoves"; import { openMoveThreadsDialog } from "./MoveThreadsDialog"; +import { environmentSupportsDevices } from "./device/deviceState"; import { startAssistantCode, useOutsideAgentsSnapshot, @@ -2033,6 +2039,32 @@ export function OpenCommandPaletteDialog() { }); } + // A project thread on a computer whose server has devices: the same rule as + // the Device tab itself (availableRightPanelTabs). + if ( + activeThread && + activeThreadProject !== null && + activeThreadProject.kind !== "general-chat" && + environmentSupportsDevices(activeThread.environmentId) + ) { + const threadRef = scopeThreadRef(activeThread.environmentId, activeThread.id); + actionItems.push({ + kind: "action", + value: "action:thread-device", + searchTerms: ["device", "simulator", "emulator", "ios", "android", "phone"], + title: "Open device", + icon: , + run: async () => { + focusRightPanelTab(scopedThreadKey(threadRef), "device"); + await navigate({ + to: "/$environmentId/$threadId", + params: buildThreadRouteParams(threadRef), + search: (previous) => rightPanelTabSearchParams(previous, "device"), + }); + }, + }); + } + // Where find in chat can open: a saved chat on screen, off phones. if (chatFindAvailable) { actionItems.push({ diff --git a/apps/web/src/components/settings/DevicesSettings.tsx b/apps/web/src/components/settings/DevicesSettings.tsx new file mode 100644 index 000000000..1b8ea99c5 --- /dev/null +++ b/apps/web/src/components/settings/DevicesSettings.tsx @@ -0,0 +1,270 @@ +import type { + DevicePlatform, + DeviceServiceState, + DeviceToolName, + DeviceToolVersion, + EnvironmentId, +} from "@threadlines/contracts"; +import { + devicePlatformName, + devicePlatformStatus, + deviceToolInstallMessage, + deviceToolNeedsUpdate, + deviceToolShownVersion, +} from "@threadlines/client-runtime/deviceState"; +import { useState } from "react"; + +import { usePrimaryEnvironmentId } from "../../environments/primary/context"; +import { useSettings, useUpdateSettings } from "../../hooks/useSettings"; +import { + deviceErrorText, + updateDeviceTool, + useDevicesSupported, + useDeviceState, +} from "../device/deviceState"; +import { Button } from "../ui/button"; +import { Spinner } from "../ui/spinner"; +import { Switch } from "../ui/switch"; +import { + SettingsGroup, + SettingsPageContainer, + SettingsPageHeader, + SettingsRow, + SettingsSection, +} from "./settingsLayout"; +import { useSettingsPlaceShown } from "./settingsPlaceAvailability"; + +/** The two tools Threadlines installs for devices, as the Device tools section lists them. */ +const DEVICE_TOOLS: ReadonlyArray<{ + readonly tool: DeviceToolName; + readonly title: string; + /** The package's own name, as install lines say it. */ + readonly packageName: string; + readonly description: string; +}> = [ + { + tool: "hub", + title: "Device hub", + packageName: "expo-device-hub", + description: "expo-device-hub streams a device's screen and takes your taps and typing.", + }, + { + tool: "agent", + title: "agent-device", + packageName: "agent-device", + description: + "The command line agents drive devices with. It installs once agents can use devices.", + }, +]; + +const PLATFORMS: ReadonlyArray = ["ios", "android"]; + +/** + * Settings › Devices: whether the Device tab shows this computer's iOS + * Simulators and Android Emulators, whether agents can use them, and the + * versions of the tools Threadlines installs for them (docs/design/devices.md). + * Always the app's own computer. + */ +export function DevicesSettingsPanel() { + const environmentId = usePrimaryEnvironmentId(); + const supported = useDevicesSupported(environmentId); + const shown = useSettingsPlaceShown(); + const state = useDeviceState(environmentId, supported); + const settingOn = useSettings((settings) => settings.enableDevices); + // One answer for the whole page: the computer's device state once it has + // said, the setting until then. + const enabled = state?.enabled ?? settingOn; + + return ( + + + {shown("devices.simulators") ? ( + <> + + + + {state !== null && enabled + ? PLATFORMS.map((platform) => ( + + )) + : null} + + + {enabled ? ( + DEVICE_TOOLS.map((entry) => ( + + )) + ) : ( +

+ Nothing is installed until Use devices is on. +

+ )} +
+ + ) : ( + +

+ This computer's Threadlines doesn't have devices yet. Update it to use them. +

+
+ )} +
+ ); +} + +export function UseDevicesRow() { + const on = useSettings((settings) => settings.enableDevices); + const { updateSettings } = useUpdateSettings(); + return ( + + updateSettings( + checked + ? { enableDevices: true } + : { enableDevices: false, agentDeviceAccess: false }, + ) + } + aria-label="Use devices" + /> + } + /> + ); +} + +export function AgentDeviceAccessRow() { + const devicesOn = useSettings((settings) => settings.enableDevices); + const on = useSettings((settings) => settings.agentDeviceAccess); + const { updateSettings } = useUpdateSettings(); + return ( + updateSettings({ agentDeviceAccess: Boolean(checked) })} + aria-label="Agents can use devices" + /> + } + /> + ); +} + +/** One platform and whether it works here, in the words the Device tab uses. */ +function PlatformStatusRow({ + state, + platform, +}: { + readonly state: DeviceServiceState; + readonly platform: DevicePlatform; +}) { + const status = devicePlatformStatus(state, platform); + return ; +} + +/** A tool's version line: what is on disk, and the version Threadlines needs when that differs. */ +function ToolVersionLine({ tool }: { readonly tool: DeviceToolVersion | undefined }) { + const shownVersion = deviceToolShownVersion(tool); + if (shownVersion === null) { + return tool === undefined ? ( + <>Not installed + ) : ( + <> + Not installed · Uses {tool.requiredVersion} + + ); + } + return ( + <> + Installed {shownVersion} + {tool !== undefined && shownVersion !== tool.requiredVersion ? ( + <> + {" "} + · Needs {tool.requiredVersion} + + ) : null} + + ); +} + +function DeviceToolRow({ + entry, + environmentId, + state, +}: { + readonly entry: (typeof DEVICE_TOOLS)[number]; + readonly environmentId: EnvironmentId | null; + readonly state: DeviceServiceState | null; +}) { + const [pending, setPending] = useState(false); + const [error, setError] = useState(null); + const tool = state?.tools?.[entry.tool]; + // The server says only that the host installs; a tool installs then when its version isn't on disk yet. + const installing = + state?.hostStatus === "installing" && + (entry.tool === "hub" || state.agentAccessEnabled) && + (tool === undefined || !tool.installedVersions.includes(tool.requiredVersion)); + const hostFailure = + entry.tool === "hub" && state?.hostStatus === "failed" ? state.hostStatusDetail : undefined; + + const update = () => { + if (environmentId === null) return; + setPending(true); + setError(null); + updateDeviceTool(environmentId, entry.tool) + .catch((cause: unknown) => setError(deviceErrorText(cause))) + .finally(() => setPending(false)); + }; + + return ( + +
+ {installing ? ( + deviceToolInstallMessage(entry.packageName, tool) + ) : ( + + )} +
+ {hostFailure !== undefined ?
{hostFailure}
: null} + {error !== null ? ( +
+ {error} +
+ ) : null} +
+ } + control={ + deviceToolNeedsUpdate(tool) || pending ? ( + + ) : null + } + /> + ); +} diff --git a/apps/web/src/components/settings/SettingsSidebarNav.test.ts b/apps/web/src/components/settings/SettingsSidebarNav.test.ts index 33af536ae..b6af3c367 100644 --- a/apps/web/src/components/settings/SettingsSidebarNav.test.ts +++ b/apps/web/src/components/settings/SettingsSidebarNav.test.ts @@ -32,6 +32,7 @@ describe("SETTINGS_NAV_ITEMS", () => { "Agent Instructions", "| Source Control", "Connections", + "Devices", "Keybindings", ]); }); diff --git a/apps/web/src/components/settings/settingsNavigation.ts b/apps/web/src/components/settings/settingsNavigation.ts index f88a35ed4..0040802b6 100644 --- a/apps/web/src/components/settings/settingsNavigation.ts +++ b/apps/web/src/components/settings/settingsNavigation.ts @@ -16,6 +16,7 @@ import { PlugIcon, Settings2Icon, SmartphoneIcon, + TabletSmartphoneIcon, } from "lucide-react"; import { SourceControlIcon } from "../Icons"; @@ -101,6 +102,11 @@ export const SETTINGS_NAV_ITEMS: ReadonlyArray = [ icon: SmartphoneIcon, group: "computer", }), + settingsNavItem("settings.devices", { + to: "/settings/devices", + icon: TabletSmartphoneIcon, + group: "computer", + }), settingsNavItem("settings.keybindings", { to: "/settings/keybindings", icon: KeyboardIcon, diff --git a/apps/web/src/components/settings/settingsPaths.ts b/apps/web/src/components/settings/settingsPaths.ts index 959dfc2d3..1f4a7c897 100644 --- a/apps/web/src/components/settings/settingsPaths.ts +++ b/apps/web/src/components/settings/settingsPaths.ts @@ -20,6 +20,7 @@ export const VISIBLE_SETTINGS_SECTION_PATHS = [ "/settings/instructions", "/settings/source-control", "/settings/connections", + "/settings/devices", "/settings/keybindings", ] as const; diff --git a/apps/web/src/components/settings/settingsPlaceAvailability.tsx b/apps/web/src/components/settings/settingsPlaceAvailability.tsx index bf6c67a80..4e9c49047 100644 --- a/apps/web/src/components/settings/settingsPlaceAvailability.tsx +++ b/apps/web/src/components/settings/settingsPlaceAvailability.tsx @@ -79,6 +79,8 @@ interface PlaceSignals { readonly remotelyReachable: boolean; /** This computer keeps project briefs and notes. */ readonly projectContext: boolean; + /** This computer's server has devices (capability `devices`). */ + readonly devices: boolean; } /** @@ -123,6 +125,12 @@ const SHOWN_WHEN: Partial !s.phone && s.owner, "connections.oneTimeLink": (s) => !s.phone && s.owner && s.remotelyReachable, "instructions.notesMode": (s) => s.projectContext, + // The phone app has no Devices page; a server without devices draws only a note there. + "settings.devices": (s) => !s.phone && s.devices, + "devices.simulators": (s) => !s.phone && s.devices, + "devices.enable": (s) => !s.phone && s.devices, + "devices.agentAccess": (s) => !s.phone && s.devices, + "devices.tools": (s) => !s.phone && s.devices, }; /** @@ -143,6 +151,7 @@ const STATIC_SIGNALS: PlaceSignals = { outsideAgents: true, remotelyReachable: true, projectContext: true, + devices: true, }; const PlaceSignalsContext = createContext(null); @@ -196,6 +205,7 @@ function usePlaceSignals(): PlaceSignals { const primaryCapabilities = usePrimaryEnvironmentDescriptor()?.capabilities; const outsideAgents = primaryCapabilities?.outsideAgents === true; const projectContext = primaryCapabilities?.projectContext === true; + const devices = primaryCapabilities?.devices === true; const alertsSupport = pushAlertsSupport(); const alertsShown = alertsSupport !== "unsupported" && alertComputers > 0; const canSetBackground = background !== null && canSetDesktopBackgroundOptions(); @@ -217,6 +227,7 @@ function usePlaceSignals(): PlaceSignals { outsideAgents, remotelyReachable: isElectron ? desktopReach : authPolicy === "remote-reachable", projectContext, + devices, }), [ alertsEnabled, @@ -227,6 +238,7 @@ function usePlaceSignals(): PlaceSignals { background, canSetBackground, desktopReach, + devices, dictation, outsideAgents, phone, diff --git a/apps/web/src/components/settings/settingsRowRegistry.tsx b/apps/web/src/components/settings/settingsRowRegistry.tsx index e41cfb517..e374f0ff9 100644 --- a/apps/web/src/components/settings/settingsRowRegistry.tsx +++ b/apps/web/src/components/settings/settingsRowRegistry.tsx @@ -13,6 +13,7 @@ import { } from "../theme/ui/ThemeSettingsRows"; import { ThemeSystemPairRow } from "../theme/ui/ThemeSystemPairRow"; import { KeepAwakeForScheduledTasksRow, RunInBackgroundRow } from "./DesktopBackgroundRows"; +import { AgentDeviceAccessRow, UseDevicesRow } from "./DevicesSettings"; import { DownloadedModelsRow, HoldToRecordRow, SpeechModelRow } from "./DictationSettings"; import type { SettingsRowPlaceId } from "./settingsPlaceAvailability"; import { ProjectNotesModeRow } from "./ProjectNotesSection"; @@ -184,4 +185,7 @@ export const SETTINGS_ROWS = { "connections.sameNetwork": LINK, "connections.tailscale": LINK, "connections.oneTimeLink": LINK, + // Devices + "devices.enable": live(UseDevicesRow), + "devices.agentAccess": live(AgentDeviceAccessRow), } as const satisfies { readonly [Id in SettingsRowPlaceId]: SettingsRowEntry }; diff --git a/apps/web/src/routeTree.gen.ts b/apps/web/src/routeTree.gen.ts index 5cbc6af63..c15cbb23b 100644 --- a/apps/web/src/routeTree.gen.ts +++ b/apps/web/src/routeTree.gen.ts @@ -24,6 +24,7 @@ import { Route as SettingsIndexRouteImport } from './routes/settings.index' import { Route as SettingsAppearanceRouteImport } from './routes/settings.appearance' import { Route as SettingsArchivedRouteImport } from './routes/settings.archived' import { Route as SettingsConnectionsRouteImport } from './routes/settings.connections' +import { Route as SettingsDevicesRouteImport } from './routes/settings.devices' import { Route as SettingsDiagnosticsRouteImport } from './routes/settings.diagnostics' import { Route as SettingsGeneralRouteImport } from './routes/settings.general' import { Route as SettingsInstructionsRouteImport } from './routes/settings.instructions' @@ -112,6 +113,11 @@ const SettingsConnectionsRoute = SettingsConnectionsRouteImport.update({ path: '/connections', getParentRoute: () => SettingsRoute, } as any) +const SettingsDevicesRoute = SettingsDevicesRouteImport.update({ + id: '/devices', + path: '/devices', + getParentRoute: () => SettingsRoute, +} as any) const SettingsDiagnosticsRoute = SettingsDiagnosticsRouteImport.update({ id: '/diagnostics', path: '/diagnostics', @@ -193,6 +199,7 @@ export interface FileRoutesByFullPath { '/settings/appearance': typeof SettingsAppearanceRoute '/settings/archived': typeof SettingsArchivedRoute '/settings/connections': typeof SettingsConnectionsRoute + '/settings/devices': typeof SettingsDevicesRoute '/settings/diagnostics': typeof SettingsDiagnosticsRoute '/settings/general': typeof SettingsGeneralRoute '/settings/instructions': typeof SettingsInstructionsRoute @@ -219,6 +226,7 @@ export interface FileRoutesByTo { '/settings/appearance': typeof SettingsAppearanceRoute '/settings/archived': typeof SettingsArchivedRoute '/settings/connections': typeof SettingsConnectionsRoute + '/settings/devices': typeof SettingsDevicesRoute '/settings/diagnostics': typeof SettingsDiagnosticsRoute '/settings/general': typeof SettingsGeneralRoute '/settings/instructions': typeof SettingsInstructionsRoute @@ -250,6 +258,7 @@ export interface FileRoutesById { '/settings/appearance': typeof SettingsAppearanceRoute '/settings/archived': typeof SettingsArchivedRoute '/settings/connections': typeof SettingsConnectionsRoute + '/settings/devices': typeof SettingsDevicesRoute '/settings/diagnostics': typeof SettingsDiagnosticsRoute '/settings/general': typeof SettingsGeneralRoute '/settings/instructions': typeof SettingsInstructionsRoute @@ -282,6 +291,7 @@ export interface FileRouteTypes { | '/settings/appearance' | '/settings/archived' | '/settings/connections' + | '/settings/devices' | '/settings/diagnostics' | '/settings/general' | '/settings/instructions' @@ -308,6 +318,7 @@ export interface FileRouteTypes { | '/settings/appearance' | '/settings/archived' | '/settings/connections' + | '/settings/devices' | '/settings/diagnostics' | '/settings/general' | '/settings/instructions' @@ -338,6 +349,7 @@ export interface FileRouteTypes { | '/settings/appearance' | '/settings/archived' | '/settings/connections' + | '/settings/devices' | '/settings/diagnostics' | '/settings/general' | '/settings/instructions' @@ -470,6 +482,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof SettingsConnectionsRouteImport parentRoute: typeof SettingsRoute } + '/settings/devices': { + id: '/settings/devices' + path: '/devices' + fullPath: '/settings/devices' + preLoaderRoute: typeof SettingsDevicesRouteImport + parentRoute: typeof SettingsRoute + } '/settings/diagnostics': { id: '/settings/diagnostics' path: '/diagnostics' @@ -608,6 +627,7 @@ interface SettingsRouteChildren { SettingsAppearanceRoute: typeof SettingsAppearanceRoute SettingsArchivedRoute: typeof SettingsArchivedRoute SettingsConnectionsRoute: typeof SettingsConnectionsRoute + SettingsDevicesRoute: typeof SettingsDevicesRoute SettingsDiagnosticsRoute: typeof SettingsDiagnosticsRoute SettingsGeneralRoute: typeof SettingsGeneralRoute SettingsInstructionsRoute: typeof SettingsInstructionsRoute @@ -623,6 +643,7 @@ const SettingsRouteChildren: SettingsRouteChildren = { SettingsAppearanceRoute: SettingsAppearanceRoute, SettingsArchivedRoute: SettingsArchivedRoute, SettingsConnectionsRoute: SettingsConnectionsRoute, + SettingsDevicesRoute: SettingsDevicesRoute, SettingsDiagnosticsRoute: SettingsDiagnosticsRoute, SettingsGeneralRoute: SettingsGeneralRoute, SettingsInstructionsRoute: SettingsInstructionsRoute, diff --git a/apps/web/src/routes/settings.devices.tsx b/apps/web/src/routes/settings.devices.tsx new file mode 100644 index 000000000..44f081f1d --- /dev/null +++ b/apps/web/src/routes/settings.devices.tsx @@ -0,0 +1,7 @@ +import { createFileRoute } from "@tanstack/react-router"; + +import { DevicesSettingsPanel } from "../components/settings/DevicesSettings"; + +export const Route = createFileRoute("/settings/devices")({ + component: DevicesSettingsPanel, +}); diff --git a/packages/shared/src/appPlaces.ts b/packages/shared/src/appPlaces.ts index fb0a19034..3721f870f 100644 --- a/packages/shared/src/appPlaces.ts +++ b/packages/shared/src/appPlaces.ts @@ -476,6 +476,15 @@ const PAGES = [ route: { to: "/settings/connections" }, keywords: ["phone", "pair", "remote", "devices"], }, + { + id: "settings.devices", + kind: "page", + title: "Devices", + about: + "iOS Simulators and Android Emulators in the Device tab, and whether agents can use them.", + route: { to: "/settings/devices" }, + keywords: ["simulator", "emulator", "iOS", "Android", "Xcode", "device tab", "agent-device"], + }, { id: "settings.keybindings", kind: "page", @@ -895,6 +904,24 @@ const SECTIONS = [ keywords: ["tailscale", "lan", "network"], widensAgents: true, }, + // Devices + { + id: "devices.simulators", + kind: "section", + parent: "settings.devices", + title: "Simulators & emulators", + about: + "Whether the Device tab shows this computer's iOS Simulators and Android Emulators, and whether agents can use them.", + }, + { + id: "devices.tools", + kind: "section", + parent: "settings.devices", + title: "Device tools", + about: + "The device hub and agent-device that Threadlines installs for devices: each one's version, and Update when a newer one is needed.", + keywords: ["expo-device-hub", "agent-device", "version", "update", "install"], + }, // Fleet: a computer's page (/fleet/) { id: "fleet.connection", @@ -1906,6 +1933,28 @@ const ROWS = [ keywords: ["link", "pairing link"], widensAgents: true, }, + // Devices › Simulators & emulators + { + id: "devices.enable", + kind: "row", + parent: "devices.simulators", + title: "Use devices", + about: + "The Device tab shows the iOS Simulators and Android Emulators on this computer, and the device hub installs the first time the tab or an agent asks.", + keywords: ["simulator", "emulator", "iOS", "Android", "device tab", "turn on"], + ...editableServerSwitch("enableDevices"), + }, + { + id: "devices.agentAccess", + kind: "row", + parent: "devices.simulators", + title: "Agents can use devices", + about: + "Agents get tools to open, see and tap devices here, in sessions started after you turn it on.", + keywords: ["device_open", "agent-device", "simulator", "emulator", "agent tools"], + value: onOff((settings) => settings.agentDeviceAccess), + widensAgents: true, + }, // Fleet › This app { id: "fleet.removeFromApp", From c26834c853d3c930a1ebec25638e45de39e9c89b Mon Sep 17 00:00:00 2001 From: Badcuban <108198679+badcuban@users.noreply.github.com> Date: Fri, 20 Nov 2026 01:36:51 -0500 Subject: [PATCH 10/49] docs(devices): record where the web half differs from the plan No device.toggle keybinding command (older clients decode the command list strictly), the devices capability, the toolbar layout, the device-tab topic id, how credits are counted, and the parts of T3's viewer not ported. --- docs/design/devices.md | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/docs/design/devices.md b/docs/design/devices.md index fbbe9e7f8..157c9dede 100644 --- a/docs/design/devices.md +++ b/docs/design/devices.md @@ -153,7 +153,12 @@ Flow control, so a suspended phone cannot pile up video in the server: http body it stops reading (the hub's socket backs up). For a ws upstream it pauses the socket; if 4 MiB more queue up anyway, the channel fails with `slow` and the client reopens. -- The client credits a chunk once the decoder has taken it, not on arrival. +- The client credits a chunk once the decoder has taken it, not on arrival: + an http chunk when a body read hands it over, a ws message when its + listeners return. Credits count decoded bytes (before base64; a text + message's UTF-8 bytes) and go out in batches of 64 KiB or after 40 ms. +- A `message` event's `data` is base64 when `binary`, the text itself + otherwise, the same as `deviceChannelSend`. - Channels never resubscribe. On any failure or reconnect the client opens a fresh channel and a fresh decoder that waits for a keyframe, and drops input it had not sent. A pre-disconnect gesture never reaches the new session. @@ -198,14 +203,27 @@ Owner and paired clients may both use devices (same rule as terminals). ### Web -- Right-panel tab `device` (`rightPanelTabs.ts`), available for project - threads on every client. Keybinding `device.toggle`, command palette entry - "Open device". +- Right-panel tab `device` (`rightPanelTabs.ts`, URL param `device=1`), + available for project threads on every client whose server reports the + `devices` capability (`ExecutionEnvironmentCapabilities.devices`; older + servers hide the tab, the palette entry and the Devices settings). Command + palette entry "Open device". No keybinding command: `KeybindingCommand` is + a closed set that older clients and servers decode strictly, and the server + writes every default into the user's keybindings file, so a new command + breaks an older client's whole config (the same reason the lead chat + dropped `navigate.leadChat`, #972). It can come back once keybinding + decoding skips unknown commands. - First open: setup steps (start the hub, platform check, agent access choice). Opening the tab alone downloads nothing. - Device picker, the screen in a flat frame sized to the device's aspect, - toolbar, Tools drawer (`DeviceToolsPanel`), loading and error states that say - what is missing ("Android SDK Command-line Tools are missing from …"). + a toolbar above it (plain until hover: back to the picker, Home, app + switcher, or Back and Recents on Android, rotate, screenshot, Tools, and a + menu with Close and Power off), Tools drawer (`DeviceToolsPanel`, beside + the screen when the panel is wide, over it when not), loading and error + states that say what is missing ("Android SDK Command-line Tools are missing + from …"). T3's accessibility-frame overlay and serve-sim event log are not + ported; the foreground app comes from `deviceDetail`, not serve-sim's + `/appstate` event stream. - Only the visible device streams; switching tabs or threads stops the channel. - Settings → Devices section with the two switches and tool versions. @@ -215,7 +233,8 @@ Owner and paired clients may both use devices (same rule as terminals). `ServerSettings.agentDeviceAccess` (default false), plus patch fields. Turning "Use devices" off stops the hub and daemon; simulators keep running. Turning "Agents can use devices" off removes the tools from sessions started after. -Both get app guide places (`devices.enable`, `devices.agentAccess`); agents may +Both get app guide places (`devices.enable`, `devices.agentAccess`, on the +`settings.devices` page, with the `device-tab` topic); agents may propose turning on "Use devices" through `settings_change` but agent access is user-only (`widensAgents`). From 2b53b81137c4eb2a41dd580eb54a54405394721c Mon Sep 17 00:00:00 2001 From: Badcuban <108198679+badcuban@users.noreply.github.com> Date: Fri, 20 Nov 2026 01:42:05 -0500 Subject: [PATCH 11/49] fix(server): devices report their capability, and channel credit counts exact bytes The web hides the Device tab unless the server reports the devices capability. An http channel counted base64 slices by estimate while the client credits exact decoded bytes, so the window crept shut over hours. --- apps/server/src/device/DeviceChannels.ts | 3 ++- apps/server/src/environment/Layers/ServerEnvironment.ts | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/server/src/device/DeviceChannels.ts b/apps/server/src/device/DeviceChannels.ts index 41851f5b6..380c6dea0 100644 --- a/apps/server/src/device/DeviceChannels.ts +++ b/apps/server/src/device/DeviceChannels.ts @@ -240,7 +240,8 @@ export const make = Effect.gen(function* () { Stream.mapEffect((data) => Effect.gen(function* () { yield* credited.await; - channel.unacknowledged += Math.floor((data.length * 3) / 4); + // Exact decoded size: the client credits decoded bytes, so an estimate would drift. + channel.unacknowledged += Buffer.byteLength(data, "base64"); if (channel.unacknowledged >= DEVICE_CHANNEL_WINDOW_BYTES) Latch.closeUnsafe(credited); return { _tag: "data", data } as const; }), diff --git a/apps/server/src/environment/Layers/ServerEnvironment.ts b/apps/server/src/environment/Layers/ServerEnvironment.ts index 33ca82d6e..576ff9445 100644 --- a/apps/server/src/environment/Layers/ServerEnvironment.ts +++ b/apps/server/src/environment/Layers/ServerEnvironment.ts @@ -120,6 +120,7 @@ export const makeServerEnvironment = Effect.fn("makeServerEnvironment")(function projectLead: true, childStartSelection: true, projectContext: true, + devices: true, deviceNames: true, draftGoalSetup: true, pullRequestSwitchesByNumber: true, From dc6ce0a4d3810b173989a84de9f2e4e7c473cc3b Mon Sep 17 00:00:00 2001 From: Badcuban <108198679+badcuban@users.noreply.github.com> Date: Fri, 20 Nov 2026 02:06:07 -0500 Subject: [PATCH 12/49] fix(server): list iOS simulators that were never booted expo-device-hub's device list leaves out simulators with no usage timestamps. serve-sim's grid lists them; add the ones missing. Ported from T3 Code's fix (pingdotgg/t3code#28915). --- .../src/device/DeviceHubOperations.test.ts | 71 +++++++++++++++++++ apps/server/src/device/DeviceHubOperations.ts | 40 +++++++++++ 2 files changed, 111 insertions(+) create mode 100644 apps/server/src/device/DeviceHubOperations.test.ts diff --git a/apps/server/src/device/DeviceHubOperations.test.ts b/apps/server/src/device/DeviceHubOperations.test.ts new file mode 100644 index 000000000..2d85dbee8 --- /dev/null +++ b/apps/server/src/device/DeviceHubOperations.test.ts @@ -0,0 +1,71 @@ +import * as Http from "node:http"; +import type { AddressInfo } from "node:net"; + +import { describe, expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import { FetchHttpClient, HttpClient } from "effect/http"; + +import { makeDeviceHubOperations } from "./DeviceHubOperations.ts"; + +/** A stand-in hub answering the two listings with fixed JSON. */ +const startFakeHub = (routes: Record) => + Effect.acquireRelease( + Effect.callback<{ origin: string; close: () => void }>((resume) => { + const server = Http.createServer((request, response) => { + const body = routes[request.url ?? ""]; + if (body === undefined) { + response.writeHead(404).end(); + return; + } + response.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify(body)); + }); + server.listen(0, "127.0.0.1", () => { + const { port } = server.address() as AddressInfo; + resume(Effect.succeed({ origin: `http://127.0.0.1:${port}`, close: () => server.close() })); + }); + }), + (hub) => Effect.sync(hub.close), + ); + +describe("DeviceHubOperations list", () => { + it.effect("adds simulators that were never booted, once, from serve-sim's grid", () => + Effect.gen(function* () { + const hub = yield* startFakeHub({ + "/api/devices": { + simulators: [ + { + id: "AAAA-1", + name: "iPhone 17", + version: "iOS 27.2", + platform: "ios", + booted: true, + physical: false, + }, + ], + emulators: [], + }, + "/vendor/serve-sim/grid/api": { + devices: [ + { device: "aaaa-1", name: "iPhone 17", runtime: "iOS-27-2", state: "Booted" }, + { device: "BBBB-2", name: "iPhone 17e", runtime: "iOS-27-2", state: "Shutdown" }, + { device: "CCCC-3", name: "Apple Watch", runtime: "watchOS-12-0", state: "Shutdown" }, + ], + }, + }); + const operations = makeDeviceHubOperations({ + origin: hub.origin, + token: "t", + http: yield* HttpClient.HttpClient, + run: () => Effect.succeed({ code: 0, stdout: "", stderr: "" }), + platformAvailable: (platform) => Effect.succeed(platform === "ios"), + }); + + const listed = yield* operations.list; + + expect(listed.devices.map((device) => [device.id, device.version, device.booted])).toEqual([ + ["AAAA-1", "iOS 27.2", true], + ["BBBB-2", "iOS 27.2", false], + ]); + }).pipe(Effect.scoped, Effect.provide(FetchHttpClient.layer)), + ); +}); diff --git a/apps/server/src/device/DeviceHubOperations.ts b/apps/server/src/device/DeviceHubOperations.ts index a17cccf52..65e9cb8aa 100644 --- a/apps/server/src/device/DeviceHubOperations.ts +++ b/apps/server/src/device/DeviceHubOperations.ts @@ -50,6 +50,17 @@ const HubDeviceList = Schema.Struct({ Schema.Array(Schema.Struct({ message: Schema.String, error: Schema.optional(Schema.String) })), ), }); +/** serve-sim's grid inventory, which, unlike `/api/devices`, includes simulators never booted. */ +const IosGridDeviceList = Schema.Struct({ + devices: Schema.Array( + Schema.Struct({ + device: Schema.String, + name: Schema.String, + runtime: Schema.String, + state: Schema.String, + }), + ), +}); const HubActionResult = Schema.Struct({ ok: Schema.Boolean, id: Schema.optional(Schema.String), @@ -238,6 +249,35 @@ export const makeDeviceHubOperations = (options: { const head = errorHead(error.error ?? ""); return boundDiagnostics(head ? `${error.message} ${head}` : error.message); }) ?? []; + // The hub's device list leaves out simulators that were never booted (they + // carry no usage timestamps); serve-sim's grid lists them. Ported from T3 + // Code's fix (pingdotgg/t3code#28915). + if (yield* options.platformAvailable("ios")) { + const grid = yield* hubJson( + HttpClientRequest.get(`${origin}${vendorPrefix("ios")}/grid/api`), + IosGridDeviceList, + "list", + ).pipe(Effect.result); + if (grid._tag === "Failure") { + details.push("Could not list all iOS simulators. Recently used ones are listed."); + } else { + for (const device of grid.success.devices) { + const known = devices.some( + (listed) => + listed.platform === "ios" && listed.id.toUpperCase() === device.device.toUpperCase(), + ); + if (known || !/^iOS-\d+(?:-\d+)*$/.test(device.runtime)) continue; + devices.push({ + id: device.device, + name: device.name, + version: device.runtime.replace("iOS-", "iOS ").replaceAll("-", "."), + platform: "ios", + booted: device.state === "Booted", + physical: false, + }); + } + } + } if (androidAvailable) { if (hubList.errors?.some((error) => error.message.startsWith(AVDMANAGER_LIST_FAILED))) { const adbDevices = yield* listAdbDevices; From 4b2b90a45f97ea2621bca7572816d89418c97b68 Mon Sep 17 00:00:00 2001 From: Badcuban <108198679+badcuban@users.noreply.github.com> Date: Fri, 20 Nov 2026 02:06:07 -0500 Subject: [PATCH 13/49] fix(web): the Tools drawer shows the iOS app in front simctl cannot report the foreground app, so the drawer asks the device's serve-sim helper every two seconds while it is open. The device header drops its truncated "started by Threadlines" note; the picker shows it. --- .../web/src/components/device/DevicePanel.tsx | 1 - .../src/components/device/DeviceWorkspace.tsx | 2 - .../components/device/useDeviceControls.ts | 31 ++++++++- .../src/device/appState.test.ts | 41 ++++++++++++ .../client-runtime/src/device/appState.ts | 63 +++++++++++++++++++ packages/client-runtime/src/device/index.ts | 1 + 6 files changed, 133 insertions(+), 6 deletions(-) create mode 100644 packages/client-runtime/src/device/appState.test.ts create mode 100644 packages/client-runtime/src/device/appState.ts diff --git a/apps/web/src/components/device/DevicePanel.tsx b/apps/web/src/components/device/DevicePanel.tsx index 5f662a656..3e9ec42ba 100644 --- a/apps/web/src/components/device/DevicePanel.tsx +++ b/apps/web/src/components/device/DevicePanel.tsx @@ -136,7 +136,6 @@ export function DevicePanel(props: { environmentId={environmentId} device={activeDevice} visible={visible} - bootedByThreadlines={activeSession.bootedByThreadlines} onBack={() => selectThreadDevice(threadKey, null)} onClose={() => close(false)} onPowerOff={() => close(true)} diff --git a/apps/web/src/components/device/DeviceWorkspace.tsx b/apps/web/src/components/device/DeviceWorkspace.tsx index 556c24ae2..9d9c6de11 100644 --- a/apps/web/src/components/device/DeviceWorkspace.tsx +++ b/apps/web/src/components/device/DeviceWorkspace.tsx @@ -36,7 +36,6 @@ export function DeviceWorkspace(props: { readonly device: DeviceSummary; /** The tab is on screen; nothing streams while it is not. */ readonly visible: boolean; - readonly bootedByThreadlines: boolean; readonly onBack: () => void; readonly onClose: () => void; readonly onPowerOff: () => void; @@ -98,7 +97,6 @@ export function DeviceWorkspace(props: {

{device.name}

{device.version} - {props.bootedByThreadlines ? " · started by Threadlines" : ""}

press("home")}> diff --git a/apps/web/src/components/device/useDeviceControls.ts b/apps/web/src/components/device/useDeviceControls.ts index c67bceef9..d9ba1eb4d 100644 --- a/apps/web/src/components/device/useDeviceControls.ts +++ b/apps/web/src/components/device/useDeviceControls.ts @@ -1,12 +1,22 @@ import type { DeviceActionInput, DeviceDetail, + DeviceForegroundApp, DeviceSummary, EnvironmentId, } from "@threadlines/contracts"; +import { + createDeviceHubTransport, + subscribeDeviceForeground, +} from "@threadlines/client-runtime/device"; import { useEffect, useRef, useState } from "react"; -import { deviceErrorText, readDeviceDetail, runDeviceAction } from "./deviceState"; +import { + deviceChannelApi, + deviceErrorText, + readDeviceDetail, + runDeviceAction, +} from "./deviceState"; type ActionBody = DeviceActionInput extends infer A ? A extends { readonly type: string } @@ -18,7 +28,8 @@ type ActionBody = DeviceActionInput extends infer A * The Tools drawer's values and actions for one device. Reads the values when * it becomes visible; every action answers with the values read back, so the * drawer only ever shows what the device says. Actions run one at a time. - * Mount it keyed by device. + * On iOS the app in front comes from the device's serve-sim helper, asked + * every two seconds while the drawer is open (`simctl` cannot report it). Mount it keyed by device. */ export function useDeviceControls(input: { readonly environmentId: EnvironmentId; @@ -32,6 +43,20 @@ export function useDeviceControls(input: { // Hiding does not cancel a command on the device, so actions stay one at a // time until each settles rather than letting a reopened drawer race it. const busy = useRef(false); + // undefined: no live reading (Android, or the stream failed); then the detail's answer stands. + const [liveForeground, setLiveForeground] = useState( + undefined, + ); + + useEffect(() => { + if (!visible || device.platform !== "ios") return; + const transport = createDeviceHubTransport(deviceChannelApi(environmentId)); + const stop = subscribeDeviceForeground(transport, device.id, setLiveForeground); + return () => { + stop(); + setLiveForeground(undefined); + }; + }, [device.id, device.platform, environmentId, visible]); useEffect(() => { if (!visible) return; @@ -76,7 +101,7 @@ export function useDeviceControls(input: { error, act, disabled: pending || detail === null || !visible, - foregroundApp: detail?.foregroundApp ?? null, + foregroundApp: liveForeground === undefined ? (detail?.foregroundApp ?? null) : liveForeground, }; } diff --git a/packages/client-runtime/src/device/appState.test.ts b/packages/client-runtime/src/device/appState.test.ts new file mode 100644 index 000000000..6131c2c34 --- /dev/null +++ b/packages/client-runtime/src/device/appState.test.ts @@ -0,0 +1,41 @@ +import { afterEach, describe, expect, it, vi } from "vite-plus/test"; + +import { FOREGROUND_POLL_MS, subscribeDeviceForeground } from "./appState.ts"; +import type { DeviceHubTransport } from "./hubTransport.ts"; + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("subscribeDeviceForeground", () => { + it("reports the app in front as it changes, the home screen as null, and a missing helper as unknown", async () => { + vi.useFakeTimers(); + const answers: Array<() => Response> = [ + () => new Response(JSON.stringify({ pid: 1, bundleId: "com.apple.Preferences" })), + () => new Response(JSON.stringify({ pid: 1, bundleId: "com.apple.Preferences" })), + () => new Response(JSON.stringify({ bundleId: null })), + () => new Response(JSON.stringify({ error: "No serve-sim device" }), { status: 404 }), + ]; + const paths: Array = []; + const transport: DeviceHubTransport = { + fetch: async (path) => { + paths.push(path); + return answers[Math.min(paths.length - 1, answers.length - 1)]!(); + }, + socket: () => { + throw new Error("not used"); + }, + }; + + const seen: Array = []; + const stop = subscribeDeviceForeground(transport, "ABC 1", (app) => seen.push(app)); + for (let poll = 0; poll < 4; poll++) { + await vi.advanceTimersByTimeAsync(FOREGROUND_POLL_MS); + } + stop(); + + expect(paths[0]).toBe("/vendor/serve-sim/helper/ABC%201/foreground"); + // The repeated answer is reported once. + expect(seen).toEqual([{ id: "com.apple.Preferences" }, null, undefined]); + }); +}); diff --git a/packages/client-runtime/src/device/appState.ts b/packages/client-runtime/src/device/appState.ts new file mode 100644 index 000000000..0b8454b2e --- /dev/null +++ b/packages/client-runtime/src/device/appState.ts @@ -0,0 +1,63 @@ +// @effect-diagnostics globalTimers:off - Browser transport code, run without an Effect runtime. +import type { DeviceForegroundApp } from "@threadlines/contracts"; + +import type { DeviceHubTransport } from "./hubTransport.ts"; + +/** How often the open Tools drawer asks which app is in front. */ +export const FOREGROUND_POLL_MS = 2_000; + +/** + * The app in front on an iOS Simulator, as it changes, while the caller + * listens. `simctl` has no way to ask, so this asks the device's serve-sim + * helper (`/vendor/serve-sim/helper//foreground`, JSON with + * `bundleId`), which exists while the device's screen is streaming. Calls + * `onChange` with the app, `null` for the home screen, and `undefined` when + * the helper could not answer (the caller keeps whatever else it knows). + * Repeats every FOREGROUND_POLL_MS until the returned stop. + */ +export function subscribeDeviceForeground( + transport: DeviceHubTransport, + deviceId: string, + onChange: (app: DeviceForegroundApp | null | undefined) => void, +): () => void { + let stopped = false; + let controller: AbortController | null = null; + let timer: ReturnType | null = null; + let last: string | undefined; + + const report = (app: DeviceForegroundApp | null | undefined) => { + const key = app === undefined ? "?" : app === null ? "" : app.id; + if (key === last) return; + last = key; + onChange(app); + }; + + const ask = async () => { + controller = new AbortController(); + try { + const response = await transport.fetch( + `/vendor/serve-sim/helper/${encodeURIComponent(deviceId)}/foreground`, + { signal: controller.signal }, + ); + report(response.ok ? parseForeground(await response.json()) : undefined); + } catch { + if (!stopped) report(undefined); + } + if (!stopped) timer = setTimeout(() => void ask(), FOREGROUND_POLL_MS); + }; + + void ask(); + return () => { + stopped = true; + if (timer !== null) clearTimeout(timer); + controller?.abort(); + }; +} + +/** The helper's answer: the app, `null` for the home screen, `undefined` for anything else. */ +export function parseForeground(body: unknown): DeviceForegroundApp | null | undefined { + if (typeof body !== "object" || body === null || !("bundleId" in body)) return undefined; + const bundleId = (body as { readonly bundleId: unknown }).bundleId; + if (bundleId === null || bundleId === "") return null; + return typeof bundleId === "string" ? { id: bundleId } : undefined; +} diff --git a/packages/client-runtime/src/device/index.ts b/packages/client-runtime/src/device/index.ts index 114851fb5..dd4ed1280 100644 --- a/packages/client-runtime/src/device/index.ts +++ b/packages/client-runtime/src/device/index.ts @@ -1,3 +1,4 @@ +export * from "./appState.ts"; export * from "./frame.ts"; export * from "./hubTransport.ts"; export * from "./screenshot.ts"; From 9bd97b3ceb567b290765c2ca846965c356540b74 Mon Sep 17 00:00:00 2001 From: Badcuban <108198679+badcuban@users.noreply.github.com> Date: Fri, 20 Nov 2026 02:23:17 -0500 Subject: [PATCH 14/49] fix(devices): close the gaps the diff review found The preview tunnel refuses this server's own device tool ports on every platform. Launcher pruning never follows a symlink. device_open never picks a physical phone by default. A refused channel open closes its hub socket. Switching agent access off refuses tools at once. A session the daemon did not confirm closing stays claimed; an open that finishes after its thread was put away records nothing. The state stream keeps only the newest snapshots for a stalled subscriber. The iOS screenshot fallback keeps its temp file in the state dir. Touch screens get a Keyboard button, and iOS types text as US-layout key presses. --- apps/server/src/device/AgentDeviceShim.ts | 13 +++- apps/server/src/device/DeviceChannels.ts | 5 +- apps/server/src/device/DeviceHost.ts | 7 +- .../src/device/DeviceHubOperations.test.ts | 1 + apps/server/src/device/DeviceHubOperations.ts | 19 +++-- apps/server/src/device/DeviceService.test.ts | 57 +++++++++++++- apps/server/src/device/DeviceService.ts | 58 ++++++++++---- apps/server/src/device/LocalDeviceHost.ts | 26 +++++-- .../server/src/mcp/deviceToolHandlers.test.ts | 28 +++++++ apps/server/src/mcp/deviceToolHandlers.ts | 9 ++- apps/server/src/preview/PreviewTunnel.test.ts | 26 +++++++ apps/server/src/preview/deviceToolPorts.ts | 35 ++++++++- .../components/device/DeviceStreamView.tsx | 2 +- .../src/components/device/DeviceWorkspace.tsx | 39 ++++++++++ docs/design/devices.md | 27 +++++++ .../client-runtime/src/device/stream.test.ts | 17 ++++ packages/client-runtime/src/device/stream.ts | 78 ++++++++++++++++++- 17 files changed, 401 insertions(+), 46 deletions(-) create mode 100644 apps/server/src/mcp/deviceToolHandlers.test.ts diff --git a/apps/server/src/device/AgentDeviceShim.ts b/apps/server/src/device/AgentDeviceShim.ts index 3afea3d9a..d6624183c 100644 --- a/apps/server/src/device/AgentDeviceShim.ts +++ b/apps/server/src/device/AgentDeviceShim.ts @@ -297,7 +297,12 @@ export const bindAgentDeviceLauncher = Effect.fn("AgentDeviceShim.bind")(functio }); }); -/** Removes every launcher whose `/` key is not in `live`. Best effort. */ +/** + * Removes every launcher whose `/` key is not in `live`. Best + * effort. A thread entry that is not a real directory (a symlink planted in + * the state dir) is removed as itself and never followed, so pruning cannot + * reach outside `device/bin`. + */ export const pruneAgentDeviceLaunchers = Effect.fn("AgentDeviceShim.prune")(function* (input: { readonly stateDir: string; readonly live: ReadonlySet; @@ -308,6 +313,12 @@ export const pruneAgentDeviceLaunchers = Effect.fn("AgentDeviceShim.prune")(func const threads = yield* fs.readDirectory(root).pipe(Effect.orElseSucceed(() => [])); for (const thread of threads) { const threadDir = path.join(root, thread); + const info = yield* fs.lstat(threadDir).pipe(Effect.option); + if (info._tag === "None") continue; + if (info.value.type !== "Directory") { + yield* fs.remove(threadDir, { force: true }).pipe(Effect.ignore); + continue; + } const devices = yield* fs.readDirectory(threadDir).pipe(Effect.orElseSucceed(() => [])); const stale = devices.filter((device) => !input.live.has(`${thread}/${device}`)); yield* Effect.forEach( diff --git a/apps/server/src/device/DeviceChannels.ts b/apps/server/src/device/DeviceChannels.ts index 380c6dea0..5199ff5fc 100644 --- a/apps/server/src/device/DeviceChannels.ts +++ b/apps/server/src/device/DeviceChannels.ts @@ -333,7 +333,10 @@ export const make = Effect.gen(function* () { socket.terminate(); }; socket.on("error", (error) => fail(failure("broken", error.message))); - yield* register(connectionId, input.channelId, channel, () => socket.terminate()); + // A refused open (a duplicate id, or the connection's limit) must not leave the socket up. + yield* register(connectionId, input.channelId, channel, () => socket.terminate()).pipe( + Effect.onError(() => Effect.sync(() => socket.terminate())), + ); socket.on("message", (data: Buffer | ArrayBuffer | Array, isBinary: boolean) => { const bytes = Array.isArray(data) ? Buffer.concat(data) : Buffer.from(data as Buffer); diff --git a/apps/server/src/device/DeviceHost.ts b/apps/server/src/device/DeviceHost.ts index a036b24e1..1f18f6002 100644 --- a/apps/server/src/device/DeviceHost.ts +++ b/apps/server/src/device/DeviceHost.ts @@ -140,9 +140,12 @@ export class DeviceHost extends Context.Service< /** * Ends an agent-device session on whatever daemon runs for this state * folder, this server's or one an earlier run left. Never starts one. - * Best effort: a daemon that is gone took its sessions with it. + * True when the session is gone (closed now, already released, or no + * daemon runs: a daemon that is gone took its sessions with it); false when + * a live daemon did not confirm the close, so the caller keeps its claim + * and can try again. */ - readonly releaseAgentSession: (session: string) => Effect.Effect; + readonly releaseAgentSession: (session: string) => Effect.Effect; /** Stops the agent-device daemon through its own CLI. */ readonly stopAgent: Effect.Effect; /** Stops the hub and the daemon. Simulators and emulators keep running; the user owns those. */ diff --git a/apps/server/src/device/DeviceHubOperations.test.ts b/apps/server/src/device/DeviceHubOperations.test.ts index 2d85dbee8..49b23ae77 100644 --- a/apps/server/src/device/DeviceHubOperations.test.ts +++ b/apps/server/src/device/DeviceHubOperations.test.ts @@ -58,6 +58,7 @@ describe("DeviceHubOperations list", () => { http: yield* HttpClient.HttpClient, run: () => Effect.succeed({ code: 0, stdout: "", stderr: "" }), platformAvailable: (platform) => Effect.succeed(platform === "ios"), + scratchDir: "/nonexistent", }); const listed = yield* operations.list; diff --git a/apps/server/src/device/DeviceHubOperations.ts b/apps/server/src/device/DeviceHubOperations.ts index 65e9cb8aa..a0646e62b 100644 --- a/apps/server/src/device/DeviceHubOperations.ts +++ b/apps/server/src/device/DeviceHubOperations.ts @@ -31,7 +31,10 @@ const SIMULATOR_LIST_FAILED = const AVDMANAGER_LIST_FAILED = "[android-utils] Failed to run `avdmanager list avd`"; const GETPROP_LINE = /^\[([^\]]+)\]: \[(.*)\]$/; const runsOnXcode26Runtime = (version: string) => Number(/^iOS (\d+)/.exec(version)?.[1]) <= 26; -const DEFAULT_DISPLAY_SCREENSHOT = `f=$(mktemp -t threadlines-screenshot) || exit 1 +// simctl needs a real file (it writes "-" or /dev/stdout as a file of that +// name), so the PNG passes through one in Threadlines' own state folder ($2), +// removed on exit, never the user's temp folder. +const DEFAULT_DISPLAY_SCREENSHOT = `mkdir -p "$2" && f=$(mktemp "$2/screenshot.XXXXXX") || exit 1 trap 'rm -f "$f"' 0 xcrun simctl io "$1" screenshot --type=png "$f" >/dev/null && base64 -i "$f"`; @@ -110,6 +113,8 @@ export const makeDeviceHubOperations = (options: { readonly run: DeviceHostReady["run"]; /** Android gates listing AVDs, which the hub does not report until they boot. */ readonly platformAvailable: (platform: DevicePlatform) => Effect.Effect; + /** A folder in the server's state dir for files a command must write (screenshots). */ + readonly scratchDir: string; }): DeviceOperations => { const { origin, run } = options; const httpClient = options.http.pipe( @@ -394,10 +399,14 @@ export const makeDeviceHubOperations = (options: { const defaultDisplayScreenshot = Effect.fn("DeviceHubOperations.defaultDisplayScreenshot")( function* (udid: string) { - const capture = yield* run("sh", ["-c", DEFAULT_DISPLAY_SCREENSHOT, "sh", udid], { - timeoutMs: Duration.toMillis(SCREENSHOT_TIMEOUT), - maxOutputBytes: 64 * 1024 * 1024, - }); + const capture = yield* run( + "sh", + ["-c", DEFAULT_DISPLAY_SCREENSHOT, "sh", udid, options.scratchDir], + { + timeoutMs: Duration.toMillis(SCREENSHOT_TIMEOUT), + maxOutputBytes: 64 * 1024 * 1024, + }, + ); if (capture.code !== 0) { return yield* new DeviceOperationError({ operation: "screenshot", diff --git a/apps/server/src/device/DeviceService.test.ts b/apps/server/src/device/DeviceService.test.ts index 3335004a6..f6214e903 100644 --- a/apps/server/src/device/DeviceService.test.ts +++ b/apps/server/src/device/DeviceService.test.ts @@ -1,7 +1,9 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { describe, expect, it } from "@effect/vitest"; import { type DeviceSummary, ThreadId } from "@threadlines/contracts"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; @@ -16,8 +18,14 @@ import * as DeviceService from "./DeviceService.ts"; const threadId = ThreadId.make("thread-devices"); const unused = Effect.die("not used in this test"); -/** A host whose one simulator boots when asked, recording the agent sessions it ends. */ -const fakeHost = (released: Array) => { +/** + * A host whose one simulator boots when asked, recording the agent sessions it + * ends. With `bootGate`, booting signals `started` and waits for the gate. + */ +const fakeHost = ( + released: Array, + bootGate?: { readonly started: Deferred.Deferred; readonly gate: Deferred.Deferred }, +) => { const device: { current: DeviceSummary } = { current: { id: "SIM-1", @@ -38,7 +46,11 @@ const fakeHost = (released: Array) => { incompletePlatforms: [], })), boot: (target) => - Effect.sync(() => { + Effect.gen(function* () { + if (bootGate) { + yield* Deferred.succeed(bootGate.started, undefined); + yield* Deferred.await(bootGate.gate); + } device.current = { ...device.current, booted: true }; return target.id; }), @@ -60,7 +72,7 @@ const fakeHost = (released: Array) => { platformAvailability: (platform) => Effect.succeed({ platform, available: true }), ensureReady: () => Effect.succeed(ready), current: Effect.succeed(ready), - releaseAgentSession: (session) => Effect.sync(() => void released.push(session)), + releaseAgentSession: (session) => Effect.sync(() => released.push(session) > 0), stop: Effect.void, stopAgent: Effect.void, }); @@ -114,6 +126,43 @@ describe("DeviceService claims", () => { ); }); +describe("DeviceService release during an open", () => { + it.live("a thread put away while its device boots keeps no claim", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const base = yield* fs.makeTempDirectoryScoped({ prefix: "tl-device-release-" }); + const bootGate = { + started: yield* Deferred.make(), + gate: yield* Deferred.make(), + }; + const layer = Layer.mergeAll( + fakeHost([], bootGate), + settingsLayer, + ServerConfig.layerTest(process.cwd(), base), + ); + const claimsPath = path.join(base, "userdata", "device", "claims.json"); + + yield* Effect.scoped( + Effect.gen(function* () { + const service = yield* DeviceService.make; + const opening = yield* service + .open({ threadId, deviceId: "SIM-1", platform: "ios" }, "agent") + .pipe(Effect.forkScoped); + yield* Deferred.await(bootGate.started); + yield* service.releaseThread(threadId); + yield* Deferred.succeed(bootGate.gate, undefined); + const result = yield* Fiber.join(opening).pipe(Effect.flip); + expect(result._tag).toBe("DeviceHostUnavailableError"); + expect((yield* service.state).sessions).toEqual([]); + }), + ).pipe(Effect.provide(layer)); + const claims = yield* fs.readFileString(claimsPath).pipe(Effect.orElseSucceed(() => "")); + expect(claims).not.toContain("SIM-1"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +}); + describe("extractPng", () => { it("finds one complete PNG and reads its size", () => { const ihdr = Buffer.alloc(25); diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index 8f4597335..57fffdb77 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -213,7 +213,9 @@ export const make = Effect.gen(function* () { session; const initialSummary = yield* host.summary; - const statePubSub = yield* PubSub.unbounded(); + // Every state is a whole snapshot, so a subscriber that stops reading needs + // only the newest: older ones slide out instead of piling up in memory. + const statePubSub = yield* PubSub.sliding(2); const stateRef = yield* SynchronizedRef.make({ enabled: initialSettings.enabled, agentAccessEnabled: initialSettings.agentAccessEnabled, @@ -384,13 +386,25 @@ process.exitCode = result.error ? 1 : (result.status ?? 1);`, }); /** Records the claim, then publishes the session: a claim is on disk before anyone sees it open. */ - const recordSession = (session: DeviceSession) => + /** + * How many times each thread has been released (archived, moved, deleted). + * An open notes the count when it starts and records nothing if it changed, + * so a thread put away while its device was still booting keeps no claim. + */ + const releaseCounts = new Map(); + + const recordSession = (session: DeviceSession, releasesAtStart: number) => Effect.gen(function* () { if (!(yield* readDeviceSettings).enabled) { return yield* new DeviceHostUnavailableError({ reason: "Devices were turned off while the device was opening.", }); } + if ((releaseCounts.get(session.threadId) ?? 0) !== releasesAtStart) { + return yield* new DeviceHostUnavailableError({ + reason: "The thread was put away while the device was opening.", + }); + } const agentSession = agentDeviceSession(session.threadId, session.deviceId); claims.set(agentSession, { ...session, agentSession }); yield* persistClaims; @@ -405,6 +419,7 @@ process.exitCode = result.error ? 1 : (result.status ?? 1);`, const open: DeviceServiceShape["open"] = (input, openedBy = "user") => Effect.gen(function* () { + const releasesAtStart = releaseCounts.get(input.threadId) ?? 0; yield* ensurePlatform(input.platform); const ready = yield* readiness; let state = yield* refresh(ready); @@ -451,7 +466,7 @@ process.exitCode = result.error ? 1 : (result.status ?? 1);`, bootedByThreadlines: bootedNow || earlier !== undefined, openedBy, }; - yield* recordSession(session); + yield* recordSession(session, releasesAtStart); return session; }).pipe(Effect.withSpan("DeviceService.open")); @@ -471,15 +486,20 @@ process.exitCode = result.error ? 1 : (result.status ?? 1);`, * agent opened and frees the device for others), then the claims, then the * state and launchers. */ - const endSessions = (ending: ReadonlyArray) => + const endSessions = (requested: ReadonlyArray) => Effect.gen(function* () { + let ending = requested; if (ending.length === 0) return; - yield* Effect.forEach( + const released = yield* Effect.forEach( ending, (session) => host.releaseAgentSession(agentDeviceSession(session.threadId, session.deviceId)), - { concurrency: 4, discard: true }, + { concurrency: 4 }, ); + // A session whose close the daemon did not confirm stays claimed and open, + // so closing again (or the next startup's sweep) retries it. + ending = ending.filter((_, index) => released[index]); + if (ending.length === 0) return; for (const session of ending) { claims.delete(agentDeviceSession(session.threadId, session.deviceId)); } @@ -658,7 +678,8 @@ process.exitCode = result.error ? 1 : (result.status ?? 1);`, ); const releaseThread: DeviceServiceShape["releaseThread"] = (threadId) => - sessionsForThread(threadId).pipe( + Effect.sync(() => releaseCounts.set(threadId, (releaseCounts.get(threadId) ?? 0) + 1)).pipe( + Effect.andThen(sessionsForThread(threadId)), Effect.flatMap((sessions) => lifecycleLock.withPermit(endSessions(sessions))), Effect.catchCause((cause) => Effect.logWarning("Could not release a thread's devices", { threadId, cause }), @@ -700,16 +721,6 @@ process.exitCode = result.error ? 1 : (result.status ?? 1);`, ) { return; } - if (!next.enabled) { - yield* lifecycleLock.withPermit( - Effect.gen(function* () { - yield* endSessions(previous.sessions); - yield* host.stop; - }), - ); - } else if (!next.agentAccessEnabled && previous.agentAccessEnabled) { - yield* host.stopAgent; - } yield* publish((state) => { const { hostStatusDetail: _detail, ...rest } = state; return { @@ -729,6 +740,19 @@ process.exitCode = result.error ? 1 : (result.status ?? 1);`, bootingDevices: next.enabled ? state.bootingDevices : [], }; }); + // Published first, then stopped: the agent tools check this state on every + // call, so they refuse from the moment the switch turns off, not after the + // daemon or hub has finished stopping. + if (!next.enabled) { + yield* lifecycleLock.withPermit( + Effect.gen(function* () { + yield* endSessions(previous.sessions); + yield* host.stop; + }), + ); + } else if (!next.agentAccessEnabled && previous.agentAccessEnabled) { + yield* host.stopAgent; + } // Agent access turned on: fetch agent-device now, in the background, so the // first device_open is quick. Nothing starts until an agent asks. if (next.enabled && next.agentAccessEnabled && !previous.agentAccessEnabled) { diff --git a/apps/server/src/device/LocalDeviceHost.ts b/apps/server/src/device/LocalDeviceHost.ts index 416646e91..ec126dd13 100644 --- a/apps/server/src/device/LocalDeviceHost.ts +++ b/apps/server/src/device/LocalDeviceHost.ts @@ -39,6 +39,7 @@ import { writeFileStringAtomically } from "../atomicWrite.ts"; import { ServerConfig } from "../config.ts"; import { ProcessRunner } from "../processRunner.ts"; import { releaseAgentDeviceSession } from "./AgentDeviceTarget.ts"; +import { setOwnDeviceToolPort } from "../preview/deviceToolPorts.ts"; import * as DeviceHost from "./DeviceHost.ts"; import { makeDeviceHubOperations } from "./DeviceHubOperations.ts"; import { @@ -470,6 +471,12 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { nodePath: node.node, }; yield* recordHub(hub, hubTool); + // The preview tunnel refuses this port while the hub has it, on every platform. + setOwnDeviceToolPort("hub", announced.value.port); + yield* Scope.addFinalizer( + scope, + Effect.sync(() => setOwnDeviceToolPort("hub", null, announced.value.port)), + ); yield* Effect.logInfo("Device hub started", { pid: Number(child.pid), port: announced.value.port, @@ -581,11 +588,11 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { .pipe(Effect.ignore); const toEndpoint = ( file: typeof AgentDeviceDaemonFile.Type, - ): DeviceHost.AgentDeviceEndpoint => ({ - baseUrl: `http://127.0.0.1:${file.httpPort}`, - token: file.token, - entryPath, - }); + ): DeviceHost.AgentDeviceEndpoint => { + // The preview tunnel refuses the daemon's port too (forgotten when it is stopped). + setOwnDeviceToolPort("agent", file.httpPort); + return { baseUrl: `http://127.0.0.1:${file.httpPort}`, token: file.token, entryPath }; + }; const existing = yield* readDaemonFile; if (Option.isSome(existing) && existing.value.version !== AGENT_DEVICE_VERSION) { // An earlier server's daemon on another pin: stop it with its own CLI and start ours. @@ -650,6 +657,7 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { // The daemon may predate this process (an earlier server was killed, possibly on an // older pin), so it is stopped with the CLI of its recorded version when that is installed. const stopAgentDeviceDaemon = Effect.gen(function* () { + setOwnDeviceToolPort("agent", null); if (!(yield* fs.exists(daemonFilePath).pipe(Effect.orElseSucceed(() => false)))) return; const recorded = yield* readDaemonFile; if ( @@ -780,6 +788,7 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { run, platformAvailable: (platform) => platformAvailability(platform).pipe(Effect.map((availability) => availability.available)), + scratchDir: path.join(deviceStateDir(path, stateDir), "tmp"), }), nodePath: running.hub.nodePath, run, @@ -834,11 +843,14 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { : null, ), )); - if (!endpoint) return; + if (!endpoint) return true; yield* releaseAgentDeviceSession(httpClient, endpoint)(session); + return true; }).pipe( Effect.catch((cause) => - Effect.logWarning("Could not end the agent-device session", { session, cause }), + Effect.logWarning("Could not end the agent-device session", { session, cause }).pipe( + Effect.as(false), + ), ), ); diff --git a/apps/server/src/mcp/deviceToolHandlers.test.ts b/apps/server/src/mcp/deviceToolHandlers.test.ts new file mode 100644 index 000000000..252c99b08 --- /dev/null +++ b/apps/server/src/mcp/deviceToolHandlers.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from "@effect/vitest"; +import type { DeviceId, DeviceSummary } from "@threadlines/contracts"; +import * as Effect from "effect/Effect"; + +import { pickDevice } from "./deviceToolHandlers.ts"; + +const device = (id: string, overrides: Partial): DeviceSummary => ({ + id: id as DeviceId, + platform: "android", + name: id, + version: "Android 16", + booted: false, + physical: false, + ...overrides, +}); + +describe("pickDevice", () => { + it.effect("never picks a plugged-in phone by default, only when named", () => + Effect.gen(function* () { + const phone = device("R5CT1234", { booted: true, physical: true }); + const emulator = device("emulator-5554", {}); + const state = { devices: [phone, emulator], hostStatus: "ready" as const }; + + expect((yield* pickDevice(state, { platform: "android" })).id).toBe("emulator-5554"); + expect((yield* pickDevice(state, { deviceId: phone.id })).id).toBe("R5CT1234"); + }), + ); +}); diff --git a/apps/server/src/mcp/deviceToolHandlers.ts b/apps/server/src/mcp/deviceToolHandlers.ts index eb5eb634c..10d78cd38 100644 --- a/apps/server/src/mcp/deviceToolHandlers.ts +++ b/apps/server/src/mcp/deviceToolHandlers.ts @@ -68,7 +68,11 @@ const requireAccess = (caller: McpInvocationScope) => return state; }); -/** The device an agent means: the one it named, else the booted or first one of its platform. */ +/** + * The device an agent means: the one it named, else the booted or first + * simulator or emulator of its platform. A physical phone is never picked by + * default; an agent gets one only by naming its id. + */ export const pickDevice = ( state: Pick, input: { @@ -92,7 +96,8 @@ export const pickDevice = ( ); } const candidates = state.devices.filter( - (device) => input.platform === undefined || device.platform === input.platform, + (device) => + !device.physical && (input.platform === undefined || device.platform === input.platform), ); if (candidates.length === 0) { return Effect.fail( diff --git a/apps/server/src/preview/PreviewTunnel.test.ts b/apps/server/src/preview/PreviewTunnel.test.ts index 21a53aeb5..dfc27813f 100644 --- a/apps/server/src/preview/PreviewTunnel.test.ts +++ b/apps/server/src/preview/PreviewTunnel.test.ts @@ -9,6 +9,7 @@ import * as Net from "node:net"; import { tmpdir } from "node:os"; import * as Path from "node:path"; +import { setOwnDeviceToolPort } from "./deviceToolPorts.ts"; import { loopbackAddresses, makePreviewTunnel } from "./PreviewTunnel.ts"; const sessionId = "session-a"; @@ -299,6 +300,31 @@ describe("PreviewTunnel", () => { ), ); + it.live("refuses the ports this server's own device tools hold, where no scan finds them", () => + Effect.scoped( + Effect.gen(function* () { + // A plain listener: nothing about its process marks it as a device tool. + const { port } = yield* listen(() => {}); + const tunnels = yield* makePreviewTunnel({ ownPort: -1 }); + setOwnDeviceToolPort("hub", port); + yield* Effect.addFinalizer(() => Effect.sync(() => setOwnDeviceToolPort("hub", null))); + const refused = yield* tunnels + .open(sessionId, { tunnelId: "own-hub", host: "localhost", port }) + .pipe(Effect.flip); + assert.strictEqual( + refused.detail, + "That port belongs to device support and can't be previewed.", + ); + // A replaced hub stopping late does not forget its successor's port. + setOwnDeviceToolPort("hub", null, port + 1); + const still = yield* tunnels + .open(sessionId, { tunnelId: "own-hub-2", host: "localhost", port }) + .pipe(Effect.flip); + assert.strictEqual(still.reason, "notLoopback"); + }), + ), + ); + it.live("says plainly when nothing is listening", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/preview/deviceToolPorts.ts b/apps/server/src/preview/deviceToolPorts.ts index dcd8696fa..9dc727621 100644 --- a/apps/server/src/preview/deviceToolPorts.ts +++ b/apps/server/src/preview/deviceToolPorts.ts @@ -125,8 +125,35 @@ function freshScans
(scan: () => Promise): () => Promise { } /** - * The ports device tools listen on right now, from a scan begun after the - * call. Never cached: a port that was a dev server's a moment ago may be the - * hub's now. Null when that can't be known, which callers treat as a refusal. + * Ports this process's own device tools listen on, by tool, as the device host + * starts and stops them. Refused on every platform, including where the scan + * above cannot run (Windows, a machine without `pgrep`). */ -export const currentDeviceToolPorts = freshScans(scanDeviceToolPorts); +const ownToolPorts = new Map(); + +/** + * The device host records the port one of its tools listens on, or with null + * forgets it. `onlyIf` forgets only while that port is still the recorded one, + * so a replaced hub stopping after its successor started leaves the new port. + */ +export function setOwnDeviceToolPort( + tool: "hub" | "agent", + port: number | null, + onlyIf?: number, +): void { + if (port !== null) ownToolPorts.set(tool, port); + else if (onlyIf === undefined || ownToolPorts.get(tool) === onlyIf) ownToolPorts.delete(tool); +} + +const scanOnce = freshScans(scanDeviceToolPorts); + +/** + * The ports device tools listen on right now: this server's own, plus a scan + * begun after the call. Never cached: a port that was a dev server's a moment + * ago may be the hub's now. Null when the scan can't tell, which callers treat + * as a refusal. + */ +export const currentDeviceToolPorts = async (): Promise | null> => { + const scanned = await scanOnce(); + return scanned === null ? null : new Set([...scanned, ...ownToolPorts.values()]); +}; diff --git a/apps/web/src/components/device/DeviceStreamView.tsx b/apps/web/src/components/device/DeviceStreamView.tsx index f2a5f7f63..49ce7038d 100644 --- a/apps/web/src/components/device/DeviceStreamView.tsx +++ b/apps/web/src/components/device/DeviceStreamView.tsx @@ -15,7 +15,7 @@ const FRAME_INSET = BEZEL + 1; /** * The device's screen in a flat rounded frame sized to its aspect. Pointer * input becomes touches, keys go to the device while the screen has focus, - * and paste types the clipboard (Android). The frame keeps the last picture + * and paste types the clipboard. The frame keeps the last picture * while a dropped stream reconnects. */ export function DeviceStreamView(props: { diff --git a/apps/web/src/components/device/DeviceWorkspace.tsx b/apps/web/src/components/device/DeviceWorkspace.tsx index 9d9c6de11..018c5bf65 100644 --- a/apps/web/src/components/device/DeviceWorkspace.tsx +++ b/apps/web/src/components/device/DeviceWorkspace.tsx @@ -6,6 +6,7 @@ import { ChevronLeftIcon, EllipsisIcon, HomeIcon, + KeyboardIcon, Layers2Icon, PowerIcon, RotateCwIcon, @@ -16,6 +17,7 @@ import { } from "lucide-react"; import { type ReactNode, useRef, useState } from "react"; +import { useMediaQuery } from "~/hooks/useMediaQuery"; import { cn } from "~/lib/utils"; import { Button } from "../ui/button"; import { Menu, MenuItem, MenuPopup, MenuSeparator, MenuTrigger } from "../ui/menu"; @@ -57,6 +59,10 @@ export function DeviceWorkspace(props: { const capturing = useRef(false); const inputReady = state.input.connected; + // A touch screen has no hardware keys to send: the Keyboard button raises the + // on-screen keyboard over a hidden field whose typing goes to the device. + const touchScreen = useMediaQuery({ pointer: "coarse" }); + const keyboardField = useRef(null); const press = (button: "home" | "back" | "recents" | "appSwitcher") => stream.client.pressButton(button); const saveScreenshot = () => { @@ -151,6 +157,15 @@ export function DeviceWorkspace(props: { )} + {touchScreen ? ( + keyboardField.current?.focus()} + > + + + ) : null} ) : null} + {touchScreen ? ( +