diff --git a/apps/server/src/device/AgentDeviceShim.test.ts b/apps/server/src/device/AgentDeviceShim.test.ts new file mode 100644 index 000000000..0e81ef677 --- /dev/null +++ b/apps/server/src/device/AgentDeviceShim.test.ts @@ -0,0 +1,157 @@ +// @effect-diagnostics nodeBuiltinImport:off - runs the generated launcher as a real subprocess. +import * as NodeChildProcess from "node:child_process"; +import * as NodeNet from "node:net"; +import * as NodeUtil from "node:util"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { describe, expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; + +import { + agentDeviceLauncherKey, + agentDeviceLaunchersDir, + bindAgentDeviceLauncher, + pruneAgentDeviceLaunchers, +} from "./AgentDeviceShim.ts"; +import { agentDeviceConfigPath, writeAgentDeviceConfig } from "./AgentDeviceTarget.ts"; + +const exec = NodeUtil.promisify(NodeChildProcess.execFile); + +/** A loopback port that accepts connections until the scope closes, like the agent-device daemon. */ +const listeningPort = Effect.acquireRelease( + Effect.promise( + () => + new Promise((resolve) => { + const server = NodeNet.createServer((socket) => socket.destroy()); + server.listen(0, "127.0.0.1", () => resolve(server)); + }), + ), + (server) => Effect.promise(() => new Promise((resolve) => server.close(() => resolve()))), +).pipe(Effect.map((server) => (server.address() as NodeNet.AddressInfo).port)); + +describe.skipIf(process.platform === "win32")("agent-device launcher", () => { + it.live("binds one device's flags, lets the agent's own flags win, and leaves help alone", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const dir = yield* fs.makeTempDirectoryScoped({ prefix: "tl-device-launcher-" }); + const port = yield* listeningPort; + const entryPath = path.join(dir, "cli.mjs"); + yield* fs.writeFileString(entryPath, "console.log(JSON.stringify(process.argv.slice(2)));"); + const config = agentDeviceConfigPath(path, dir); + yield* writeAgentDeviceConfig(config, { baseUrl: `http://127.0.0.1:${port}`, token: "t" }); + const target = [ + "--platform", + "ios", + "--udid", + "UDID-1", + "--config", + config, + "--session", + "tl-a", + ]; + const command = yield* bindAgentDeviceLauncher({ + stateDir: dir, + threadId: "thread-1", + deviceId: "UDID-1", + node: process.execPath, + entryPath, + targetArgs: target, + }); + expect(command).toBe( + path.join(agentDeviceLaunchersDir(path, dir), "thread-1", "UDID-1", "agent-device"), + ); + const launch = (args: ReadonlyArray) => + exec(command, [...args]).then((result) => JSON.parse(result.stdout)); + + expect(yield* Effect.promise(() => launch(["snapshot", "-i"]))).toEqual([ + "snapshot", + "-i", + ...target, + ]); + expect(yield* Effect.promise(() => launch(["open", "--session", "mine"]))).toEqual([ + "open", + "--session", + "mine", + ...target.slice(0, 6), + ]); + // After `--` everything is text, and bound flags go before it. + expect(yield* Effect.promise(() => launch(["fill", "@e5", "--", "--help"]))).toEqual([ + "fill", + "@e5", + ...target, + "--", + "--help", + ]); + for (const args of [["help", "gesture"], ["open", "--help"], []]) { + expect(yield* Effect.promise(() => launch(args))).toEqual(args); + } + // Commands that would replace Threadlines' connection are refused. + yield* Effect.promise(() => + expect(launch(["daemon", "stop"])).rejects.toThrow("Threadlines manages this command"), + ); + // A runtime that moved since device_open names the fix. + yield* fs.remove(entryPath); + yield* Effect.promise(() => + expect(launch(["snapshot"])).rejects.toThrow("Call device_open again"), + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.live("says the device connection is down before running agent-device", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const dir = yield* fs.makeTempDirectoryScoped({ prefix: "tl-device-launcher-" }); + const entryPath = path.join(dir, "cli.mjs"); + yield* fs.writeFileString(entryPath, "process.exit(3);"); + const port = yield* Effect.scoped(listeningPort); + const config = agentDeviceConfigPath(path, dir); + yield* writeAgentDeviceConfig(config, { baseUrl: `http://127.0.0.1:${port}`, token: "t" }); + const command = yield* bindAgentDeviceLauncher({ + stateDir: dir, + threadId: "thread-1", + deviceId: "UDID-1", + node: process.execPath, + entryPath, + targetArgs: ["--config", config, "--session", "tl-a"], + }); + yield* Effect.promise(() => + expect(exec(command, ["snapshot"])).rejects.toThrow("is not reachable from this shell"), + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("removes the launchers of sessions that ended", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const dir = yield* fs.makeTempDirectoryScoped({ prefix: "tl-device-launcher-" }); + for (const [threadId, deviceId] of [ + ["thread-1", "UDID-1"], + ["thread-1", "Pixel 9"], + ["thread-2", "UDID-1"], + ] as const) { + yield* bindAgentDeviceLauncher({ + stateDir: dir, + threadId, + deviceId, + node: process.execPath, + entryPath: "/missing.mjs", + targetArgs: [], + }); + } + yield* pruneAgentDeviceLaunchers({ + stateDir: dir, + live: new Set([agentDeviceLauncherKey("thread-1", "Pixel 9")]), + }); + const root = agentDeviceLaunchersDir(path, dir); + expect(yield* fs.readDirectory(root)).toEqual(["thread-1"]); + expect(yield* fs.readDirectory(path.join(root, "thread-1"))).toEqual([ + agentDeviceLauncherKey("thread-1", "Pixel 9").split("/")[1], + ]); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +}); diff --git a/apps/server/src/device/AgentDeviceShim.ts b/apps/server/src/device/AgentDeviceShim.ts new file mode 100644 index 000000000..d6624183c --- /dev/null +++ b/apps/server/src/device/AgentDeviceShim.ts @@ -0,0 +1,333 @@ +// @effect-diagnostics globalJsonInEffect:off - JSON string literals embed paths safely into generated JavaScript. +/** + * The `agent-device` launcher `device_open` hands an agent + * (docs/design/devices.md). One per thread and device, at + * `/device/bin///agent-device`, with that thread's + * agent-device session, config and target flags built in, so the agent passes + * none and no provider's PATH changes. Ported from T3 Code's AgentDeviceShim: + * the launcher leaves out flags a command refuses, refuses the commands that + * would replace Threadlines' connection, and checks the connection first so a + * sandboxed shell and a dead daemon get different messages. + */ +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; + +import { deviceStateDir } from "./DeviceToolchain.ts"; + +const STALE_RUNTIME = + "This agent-device command is out of date because Threadlines moved or updated its runtime. Call device_open again for a fresh one."; + +const quoteShell = (value: string) => "'" + value.replaceAll("'", "'\"'\"'") + "'"; + +/** agent-device 0.21.36's flags that take a value, so the launcher can find the command past them. */ +const VALUE_FLAGS: ReadonlyArray = [ + "--activity", + "--all", + "--android-device-allowlist", + "--app", + "--appium-version", + "--artifact", + "--artifacts-dir", + "--aws-app-arn", + "--aws-device-arn", + "--aws-interaction-mode", + "--aws-project-arn", + "--aws-region", + "-b", + "--baseline", + "--bearer-token", + "--bundle-url", + "--button", + "--config", + "--count", + "--crop-on", + "--custom-network", + "-d", + "--daemon-auth-token", + "--daemon-base-url", + "--daemon-server-mode", + "--daemon-transport", + "--delay-ms", + "--depth", + "--device", + "--device-orientation", + "--dsym", + "--duration-ms", + "--fps", + "--from", + "--geo-location", + "--github-actions-artifact", + "--header", + "--hold-ms", + "--host", + "--image-tag", + "--in-app", + "--include", + "--interval-ms", + "--ios-simulator-device-set", + "--ios-xctest-derived-data-path", + "--ios-xctest-env-dir", + "--ios-xctestrun-file", + "--jitter-px", + "--keyframes", + "--kind", + "--language", + "--launch-args", + "--launch-console", + "--launch-url", + "--lease-backend", + "--lease-id", + "--level", + "--listen-host", + "--locale", + "--max-passes", + "--max-steps", + "--metro-host", + "--metro-kind", + "--metro-port", + "--network-profile", + "--on-error", + "--os", + "--os-version", + "--out", + "--pattern", + "--pause-ms", + "--pixel-density", + "--pixels", + "--plan-digest", + "--platform", + "--pointer-count", + "--port", + "--probe-timeout-ms", + "--project-root", + "--provider", + "--provider-app", + "--provider-appium-version", + "--provider-build", + "--provider-custom-network", + "--provider-device-orientation", + "--provider-device-type", + "--provider-geo-location", + "--provider-language", + "--provider-locale", + "--provider-network-hosts", + "--provider-network-profile", + "--provider-os-version", + "--provider-project", + "--provider-session", + "--provider-session-name", + "--provider-timezone", + "--proxy-base-url", + "--public-base-url", + "--quality", + "--record-as", + "--remote-config", + "--reporter", + "--report-junit", + "--retention-ms", + "--retries", + "--run-id", + "--runtime-file", + "-s", + "--scale", + "--scope", + "--search-path", + "--serial", + "--session", + "--session-isolation", + "--session-lock", + "--settle-quiet", + "--shard-all", + "--shard-split", + "--startup-timeout-ms", + "--state-dir", + "--status-host", + "--steps", + "--steps-file", + "--surface", + "--system", + "--target", + "--target-app", + "--template", + "--tenant", + "--threshold", + "--timeout", + "--timezone", + "--udid", + "--until", + "--wait", +]; + +// Help gets no bound flags (agent-device reads --config even for help); only args before `--` are flags. +// agent-device 0.21.31 refuses some bound flags per command; connect/disconnect/auth/daemon would +// replace Threadlines' connection; device status reads this machine's claims. It reports a sandboxed shell and a +// dropped tunnel alike, so check first, at --daemon-base-url if given, as agent-device prefers it. +const launcherSource = (node: string, entryPath: string, boundArgs: ReadonlyArray) => + `import { spawn } from "node:child_process"; +import { existsSync, readFileSync } from "node:fs"; +import { connect } from "node:net"; +const entry = ${JSON.stringify(entryPath)}; +const bound = ${JSON.stringify(boundArgs)}; +const refused = new Map([["cdp", ["--platform", "--udid", "--config", "--session"]], ["connection", ["--platform", "--udid"]]]); +const managed = ["connect", "disconnect", "auth", "daemon"]; +const args = process.argv.slice(2); +const end = args.indexOf("--"); +const options = end < 0 ? args : args.slice(0, end); +const valued = new Set(${JSON.stringify(VALUE_FLAGS)}); +let name = ""; +for (let index = 0; index < args.length; index++) { + const arg = args[index]; + if (arg === "--") { name = args[index + 1] ?? ""; break; } + if (!arg.startsWith("-")) { name = arg; break; } + if (!arg.includes("=") && valued.has(arg)) index++; +} +const informational = args.length === 0 || ["help", "--help", "-h", "--version", "version"].includes(args[0]) || options.includes("--help") || options.includes("-h"); +if (!informational && managed.includes(name)) { + console.error("Threadlines manages this command's connection to the device, so \`" + name + "\` is not available through it."); + process.exit(1); +} +if (!informational && name === "device") { + console.error("\`device status\` reads this machine's device claims, not those on the device's host, so it is not available through this command. Quote the DEVICE_IN_USE error instead."); + process.exit(1); +} +const skip = refused.get(name) ?? []; +if (!informational) for (let index = 0; index + 1 < bound.length; index += 2) if (!skip.includes(bound[index]) && !options.some(arg => arg === bound[index] || arg.startsWith(bound[index] + "="))) options.push(bound[index], bound[index + 1]); +const command = end < 0 ? options : [...options, ...args.slice(end)]; +const hasValue = flag => { const index = options.indexOf(flag); return options.some(arg => arg.startsWith(flag + "=") && arg.length > flag.length + 1) || (index >= 0 && !!options[index + 1] && !options[index + 1].startsWith("--")); }; +if (!informational && !skip.includes("--session") && !(hasValue("--config") && hasValue("--session"))) { + console.error("Call device_open first and run the agent-device command it returns."); + process.exit(1); +} +if (!existsSync(entry)) { + console.error(${JSON.stringify(STALE_RUNTIME)}); + process.exit(1); +} +const valueOf = flag => { const inline = options.find(arg => arg.startsWith(flag + "=")); if (inline) return inline.slice(flag.length + 1); const index = options.indexOf(flag); return index >= 0 ? options[index + 1] : undefined; }; +const endpoint = () => { try { const url = new URL(valueOf("--daemon-base-url") ?? JSON.parse(readFileSync(valueOf("--config"), "utf8")).daemonBaseUrl); return ["127.0.0.1", "localhost", "[::1]"].includes(url.hostname) && url.port ? url : undefined; } catch { return undefined; } }; +const local = informational || refused.has(name) ? undefined : endpoint(); +if (local) { + const failure = await new Promise(resolve => { + const socket = connect({ host: local.hostname.replace(/^\\[|\\]$/g, ""), port: Number(local.port) }); + socket.setTimeout(3000, () => { socket.destroy(); resolve("ETIMEDOUT"); }); + socket.once("connect", () => { socket.destroy(); resolve(undefined); }); + socket.once("error", error => resolve(error.code ?? "ERROR")); + }); + if (failure === "EPERM" || failure === "EACCES") { + console.error("This shell cannot reach Threadlines' device connection at " + local.host + " (connect " + failure + "): a network sandbox is blocking localhost. Run this command again with network access, outside the sandbox."); + process.exit(1); + } + if (failure) { + console.error("Threadlines' device connection at " + local.host + " is not reachable from this shell (connect " + failure + "). Call device_open again to reconnect; if that succeeds and this repeats, this shell runs in a network sandbox, so run the command with network access, outside the sandbox."); + process.exit(1); + } +} +const env = { ...process.env, ELECTRON_RUN_AS_NODE: "1" }; +delete env.AGENT_DEVICE_DAEMON_BASE_URL; +delete env.AGENT_DEVICE_DAEMON_AUTH_TOKEN; +delete env.AGENT_DEVICE_CONFIG; +const child = spawn(${JSON.stringify(node)}, [entry, ...command], { stdio: "inherit", env }); +child.on("error", error => { console.error(error.message); process.exitCode = 1; }); +child.on("exit", code => { process.exitCode = code ?? 1; }); +`; + +/** Writes a launcher into `dir` and returns the executable agents run. */ +const writeLauncher = Effect.fn("AgentDeviceShim.writeLauncher")(function* (input: { + readonly dir: string; + readonly node: string; + readonly entryPath: string; + readonly boundArgs: ReadonlyArray; +}) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const node = input.node; + yield* fs.makeDirectory(input.dir, { recursive: true }); + const launcherPath = path.join(input.dir, "agent-device-launcher.mjs"); + yield* fs.writeFileString(launcherPath, launcherSource(node, input.entryPath, input.boundArgs)); + if (process.platform === "win32") { + const script = `@echo off\r\nsetlocal EnableExtensions DisableDelayedExpansion\r\nif not exist "${node}" (echo ${STALE_RUNTIME} 1>&2 & exit /b 1)\r\n"${node}" "${launcherPath}" %*\r\n`; + const scriptPath = path.join(input.dir, "agent-device.cmd"); + yield* fs.writeFileString(scriptPath, script); + return scriptPath; + } + const script = `#!/bin/sh\n[ -x ${quoteShell(node)} ] || { echo ${quoteShell(STALE_RUNTIME)} >&2; exit 1; }\nexec ${quoteShell(node)} ${quoteShell(launcherPath)} "$@"\n`; + const scriptPath = path.join(input.dir, "agent-device"); + yield* fs.writeFileString(scriptPath, script); + yield* fs.chmod(scriptPath, 0o755); + return scriptPath; +}); + +const SAFE_SEGMENT = /^[A-Za-z0-9._-]{1,128}$/; + +/** A path segment for an id: the id itself when it is plain, else a digest of it. */ +const segment = (id: string) => + SAFE_SEGMENT.test(id) && id !== "." && id !== ".." + ? id + : `x-${Buffer.from(id).toString("base64url").slice(0, 96)}`; + +/** `/device/bin`. */ +export const agentDeviceLaunchersDir = (path: Path.Path, stateDir: string) => + path.join(deviceStateDir(path, stateDir), "bin"); + +/** Where one thread's launcher for one device lives, relative to the launchers folder. */ +export const agentDeviceLauncherKey = (threadId: string, deviceId: string) => + `${segment(threadId)}/${segment(deviceId)}`; + +/** + * A launcher for one agent-device session with its target flags built in, so + * agents pass none, and a shell that does not word-split a variable cannot mangle them. + */ +export const bindAgentDeviceLauncher = Effect.fn("AgentDeviceShim.bind")(function* (input: { + readonly stateDir: string; + readonly threadId: string; + readonly deviceId: string; + readonly node: string; + readonly entryPath: string; + readonly targetArgs: ReadonlyArray; +}) { + const path = yield* Path.Path; + return yield* writeLauncher({ + dir: path.join( + agentDeviceLaunchersDir(path, input.stateDir), + ...agentDeviceLauncherKey(input.threadId, input.deviceId).split("/"), + ), + node: input.node, + entryPath: input.entryPath, + boundArgs: input.targetArgs, + }); +}); + +/** + * Removes every launcher whose `/` key is not in `live`. Best + * effort. A thread entry that is not a real directory (a symlink planted in + * the state dir) is removed as itself and never followed, so pruning cannot + * reach outside `device/bin`. + */ +export const pruneAgentDeviceLaunchers = Effect.fn("AgentDeviceShim.prune")(function* (input: { + readonly stateDir: string; + readonly live: ReadonlySet; +}) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = agentDeviceLaunchersDir(path, input.stateDir); + const threads = yield* fs.readDirectory(root).pipe(Effect.orElseSucceed(() => [])); + for (const thread of threads) { + const threadDir = path.join(root, thread); + const info = yield* fs.lstat(threadDir).pipe(Effect.option); + if (info._tag === "None") continue; + if (info.value.type !== "Directory") { + yield* fs.remove(threadDir, { force: true }).pipe(Effect.ignore); + continue; + } + const devices = yield* fs.readDirectory(threadDir).pipe(Effect.orElseSucceed(() => [])); + const stale = devices.filter((device) => !input.live.has(`${thread}/${device}`)); + yield* Effect.forEach( + stale, + (device) => fs.remove(path.join(threadDir, device), { recursive: true, force: true }), + { discard: true }, + ).pipe(Effect.ignore); + if (stale.length === devices.length) { + yield* fs.remove(threadDir, { recursive: true, force: true }).pipe(Effect.ignore); + } + } +}); diff --git a/apps/server/src/device/AgentDeviceTarget.ts b/apps/server/src/device/AgentDeviceTarget.ts new file mode 100644 index 000000000..54cfbc099 --- /dev/null +++ b/apps/server/src/device/AgentDeviceTarget.ts @@ -0,0 +1,95 @@ +// @effect-diagnostics nodeBuiltinImport:off - a stable session name from a sha256 +/** + * Where agent-device finds this server's daemon, and the session each thread + * drives a device through (docs/design/devices.md). Ported from T3 Code's + * AgentDeviceTarget, with one host per server. + */ +import { createHash } from "node:crypto"; + +import { DeviceOperationError } from "@threadlines/contracts"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http"; + +import type { AgentDeviceEndpoint } from "./DeviceHost.ts"; +import { deviceStateDir } from "./DeviceToolchain.ts"; + +const encodeEndpoint = Schema.encodeEffect( + Schema.fromJsonString( + Schema.Struct({ + daemonBaseUrl: Schema.String, + daemonAuthToken: Schema.String, + }), + ), +); + +const key = (value: string) => createHash("sha256").update(value).digest("hex").slice(0, 24); + +/** The config file every launcher points agent-device at: the daemon's address and token. */ +export const agentDeviceConfigPath = (path: Path.Path, stateDir: string) => + path.join(deviceStateDir(path, stateDir), "agent-device.json"); + +/** One agent-device session per thread and device, named so it never collides across them. */ +export const agentDeviceSession = (threadId: string, deviceId: string): string => + `tl-${key(JSON.stringify([threadId, deviceId]))}`; + +/** Writes the daemon's address and token where launchers read it, readable by this user only. */ +export const writeAgentDeviceConfig = Effect.fn("AgentDeviceTarget.writeConfig")(function* ( + file: string, + endpoint: Pick, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* fs.makeDirectory(path.dirname(file), { recursive: true }); + const content = yield* encodeEndpoint({ + daemonBaseUrl: endpoint.baseUrl, + daemonAuthToken: endpoint.token, + }); + if ((yield* fs.readFileString(file).pipe(Effect.orElseSucceed(() => ""))) === content) return; + const temporary = yield* fs.makeTempFile({ directory: path.dirname(file), prefix: ".endpoint-" }); + yield* Effect.gen(function* () { + yield* fs.chmod(temporary, 0o600); + yield* fs.writeFileString(temporary, content); + yield* fs.rename(temporary, file); + }).pipe(Effect.ensuring(fs.remove(temporary, { force: true }).pipe(Effect.ignore))); +}); + +const AgentCloseResult = Schema.Union([ + Schema.Struct({ result: Schema.Struct({ ok: Schema.Literal(true) }) }), + Schema.Struct({ + error: Schema.Struct({ + data: Schema.Struct({ code: Schema.Literal("SESSION_NOT_FOUND") }), + }), + }), +]); + +/** + * Ends an agent-device session on a running daemon, which quits the app that + * session opened and releases its claim on the device. An already-released + * session succeeds. + */ +export const releaseAgentDeviceSession = + (http: HttpClient.HttpClient, endpoint: Pick) => + (session: string) => + HttpClientRequest.post(`${endpoint.baseUrl}/rpc`).pipe( + HttpClientRequest.bearerToken(endpoint.token), + HttpClientRequest.bodyJson({ + jsonrpc: "2.0", + id: session, + method: "agent_device.command", + params: { command: "close", session, positionals: [], flags: {} }, + }), + Effect.flatMap((request) => http.pipe(HttpClient.withScope).execute(request)), + // agent-device reports an already-released session as HTTP 404. + Effect.filterOrElse((response) => response.status === 404, HttpClientResponse.filterStatusOk), + Effect.flatMap(HttpClientResponse.schemaBodyJson(AgentCloseResult)), + Effect.scoped, + Effect.timeout(Duration.seconds(15)), + Effect.asVoid, + Effect.mapError( + () => new DeviceOperationError({ operation: "close", reason: "request_failed" }), + ), + ); diff --git a/apps/server/src/device/DeviceActions.test.ts b/apps/server/src/device/DeviceActions.test.ts new file mode 100644 index 000000000..ae7088a94 --- /dev/null +++ b/apps/server/src/device/DeviceActions.test.ts @@ -0,0 +1,234 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { describe, expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as ChildProcess from "effect/process/ChildProcess"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; + +import { readDeviceDetail, runDeviceAction } from "./DeviceActions.ts"; +import type { DeviceHostReady } from "./DeviceHost.ts"; + +type Call = { command: string; args: ReadonlyArray; stdin?: string }; + +const makeReady = ( + respond: (call: Call) => { stdout?: string; stderr?: string; code?: number } = () => ({}), + helpers: DeviceHostReady["helpers"] = { + serveSimAxSettings: "/hub/simax/serve-sim-ax-settings", + serveSimCli: "/hub/serve-sim.js", + }, +) => { + const calls: Call[] = []; + const unused = Effect.die("Device actions run host commands, never device operations"); + const ready: DeviceHostReady = { + nodePath: process.execPath, + hub: { origin: "http://127.0.0.1:1", token: "test-token" }, + devices: { + list: unused, + boot: () => unused, + attach: () => unused, + shutdown: () => unused, + screenshot: () => unused, + }, + helpers, + run: (command, args, options) => { + const call = { command, args, ...(options?.stdin ? { stdin: options.stdin } : {}) }; + calls.push(call); + const result = respond(call); + return Effect.succeed({ + stdout: result.stdout ?? "", + stderr: result.stderr ?? "", + code: result.code ?? 0, + }); + }, + }; + return { ready, calls }; +}; + +const udid = "SIM-1"; + +describe("runDeviceAction", () => { + it.live.each([ + "https://example.com/callback?code=1&state=2", + "https://example.com/it's?a=1&b=2", + "https://example.com/a b?key=value", + "https://example.com/plain", + ])("opens the complete Android URL %s", (url) => + Effect.gen(function* () { + if (process.platform === "win32") return; + const { ready, calls } = makeReady(); + yield* runDeviceAction(ready, "android", { + type: "openUrl", + deviceId: "emulator-5554", + url, + }); + const args = calls[0]!.args; + expect(args.slice(0, 3)).toEqual(["-s", "emulator-5554", "shell"]); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + // adb joins arguments verbatim before the Android shell parses them. + const stdout = yield* spawner.string( + ChildProcess.make("/bin/sh", [ + "-c", + `am() { printf '%s\\0' "$@"; }\n${args.slice(3).join(" ")}`, + ]), + ); + expect(stdout.split("\0").slice(0, -1)).toEqual([ + "start", + "-a", + "android.intent.action.VIEW", + "-d", + url, + ]); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect.each([ + ["emulator-5554", "Physical size: 2560x1600", "portrait", "-9.81:0:0"], + ["emulator-5554", "Physical size: 2560x1600", "landscape_left", "0:9.81:0"], + ["R5CT1234", "Physical size: 2560x1600", "portrait", "3"], + ["R5CT1234", "Physical size: 2560x1600", "landscape_left", "0"], + ["R5CT1234", "Physical size: 1080x1920\nOverride size: 1920x1080", "portrait", "3"], + ] as const)( + "rotates from the natural orientation reported by wm size: %s %j %s", + ([deviceId, wmSize, value, expected]) => + Effect.gen(function* () { + const { ready, calls } = makeReady((call) => + call.args.join(" ").endsWith("shell wm size") ? { stdout: `${wmSize}\n` } : {}, + ); + yield* runDeviceAction(ready, "android", { type: "setOrientation", deviceId, value }); + expect(calls.at(-1)?.args.at(-1)).toBe(expected); + }), + ); + + it.effect.each([ + ["camera", "grant", "android.permission.CAMERA"], + ["photos", "revoke", "android.permission.READ_MEDIA_IMAGES"], + ] as const)( + "reports an Android %s %s that no pm command applied", + ([permission, decision, firstPermission]) => + Effect.gen(function* () { + const { ready } = makeReady((call) => ({ + code: 1, + stderr: `Unknown package: ${call.args.at(-1)}`, + })); + const error = yield* Effect.flip( + runDeviceAction(ready, "android", { + type: "setPermission", + deviceId: "emulator-5554", + appId: "com.example.missing", + permission, + decision, + }), + ); + expect(error._tag).toBe("DeviceOperationError"); + expect(error.operation).toBe("permission"); + expect(error._tag === "DeviceOperationError" && error.detail).toBe( + `Unknown package: ${firstPermission}`, + ); + }), + ); + + it.effect.each([ + [ + "photos", + ["android.permission.READ_MEDIA_IMAGES", "android.permission.READ_EXTERNAL_STORAGE"], + 1, + ], + ["contacts", ["android.permission.READ_CONTACTS", "android.permission.WRITE_CONTACTS"], 0], + ] as const)( + "keeps a partially applied %s permission change", + ([permission, permissions, success]) => + Effect.gen(function* () { + const { ready, calls } = makeReady((call) => ({ + code: call.args.at(-1) === permissions[success] ? 0 : 1, + })); + const result = yield* runDeviceAction(ready, "android", { + type: "setPermission", + deviceId: "emulator-5554", + appId: "com.example.app", + permission, + decision: "grant", + }); + expect(result).toBeUndefined(); + expect(calls.map((call) => call.args.at(-1))).toEqual(permissions); + }), + ); +}); + +describe("readDeviceDetail", () => { + it.effect("reads the smallest iOS text size as small", () => + Effect.gen(function* () { + const { ready } = makeReady((call) => + call.args.join(" ").endsWith("ui SIM-1 content_size") + ? { stdout: "extra-small\n" } + : { code: 1 }, + ); + const detail = yield* readDeviceDetail(ready, "ios", udid); + expect(detail.settings.textSize).toBe("small"); + }), + ); + + it.effect("reads iOS settings from simctl and the accessibility helper", () => + Effect.gen(function* () { + const { ready } = makeReady((call) => { + const key = call.args.join(" "); + if (key.endsWith("ui SIM-1 appearance")) return { stdout: "dark\n" }; + if (key.endsWith("ui SIM-1 content_size")) return { stdout: "extra-extra-large\n" }; + if (key.endsWith("ui SIM-1 increase_contrast")) return { stdout: "enabled\n" }; + if (key.includes("serve-sim-ax-settings status")) { + return { + stdout: + '{"reduce-motion":"on","reduce-transparency":"off","show-borders":"off","voiceover":"off","liquid-glass":"tinted","color-filter":"grayscale"}', + }; + } + return { code: 1 }; + }); + const detail = yield* readDeviceDetail(ready, "ios", udid); + expect(detail.settings).toEqual({ + appearance: "dark", + textSize: "large", + increaseContrast: true, + reduceMotion: true, + reduceTransparency: false, + showBorders: false, + voiceOver: false, + liquidGlass: "tinted", + colorFilter: "grayscale", + }); + }), + ); + + it.effect("degrades unreadable values to unknown instead of failing", () => + Effect.gen(function* () { + const { ready } = makeReady(() => ({ code: 1, stderr: "boom" })); + const detail = yield* readDeviceDetail(ready, "ios", udid); + expect(detail.settings).toEqual({}); + expect(detail.foregroundApp).toBeNull(); + }), + ); + + it.effect("reads Android settings and the focused package", () => + Effect.gen(function* () { + const { ready } = makeReady((call) => { + const key = call.args.join(" "); + if (key.endsWith("cmd uimode night")) return { stdout: "Night mode: yes\n" }; + if (key.endsWith("font_scale")) return { stdout: "0.85\n" }; + if (key.endsWith("animator_duration_scale")) return { stdout: "0\n" }; + if (key.endsWith("wifi_on")) return { stdout: "1\n" }; + if (key.endsWith("dumpsys window")) { + return { + stdout: + " mFocusedApp=ActivityRecord{155579877 u0 com.example.app/.MainActivity t15}\n mCurrentFocus=Window{1a2b u0 com.example.app/com.example.app.MainActivity}\n", + }; + } + return { code: 1 }; + }); + const detail = yield* readDeviceDetail(ready, "android", "emulator-5554"); + expect(detail.settings).toEqual({ + appearance: "dark", + textSize: "small", + reduceMotion: true, + networkEnabled: true, + }); + expect(detail.foregroundApp).toEqual({ id: "com.example.app" }); + }), + ); +}); diff --git a/apps/server/src/device/DeviceActions.ts b/apps/server/src/device/DeviceActions.ts new file mode 100644 index 000000000..2333671cf --- /dev/null +++ b/apps/server/src/device/DeviceActions.ts @@ -0,0 +1,585 @@ +/** + * The Tools drawer (docs/design/devices.md): one typed action per control. + * serve-sim's preview drives its own Tools panel with shell commands over its + * exec channel; proxying that would hand any client command execution on this + * machine, so these run `simctl`, `adb` and serve-sim's helper binaries + * directly and never go through the hub. Ported from T3 Code's DeviceActions. + */ +import { + type DeviceActionInput, + type DeviceActionType, + type DeviceForegroundApp, + DeviceOperationError, + DeviceActionUnavailableError, + type DeviceOrientation, + type DevicePlatform, + type DeviceSettings, + type DeviceTextSize, +} from "@threadlines/contracts"; +import * as Effect from "effect/Effect"; +import * as Result from "effect/Result"; +import * as Schema from "effect/Schema"; + +import type { DeviceHostReady } from "./DeviceHost.ts"; + +type Runner = DeviceHostReady["run"]; + +const decodeAxStatus = Schema.decodeUnknownEffect( + Schema.fromJsonString(Schema.Record(Schema.String, Schema.String)), +); +const encodePushPayload = Schema.encodeUnknownEffect( + Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)), +); + +const IOS_ACTIONS: ReadonlySet = new Set([ + "setAppearance", + "setTextSize", + "setToggle", + "setLiquidGlass", + "setColorFilter", + "setLocation", + "clearLocation", + "setPermission", + "openUrl", + "launchApp", + "terminateApp", + "sendPush", +]); + +const ANDROID_ACTIONS: ReadonlySet = new Set([ + "setAppearance", + "setTextSize", + "setToggle", + "setOrientation", + "setLocation", + "clearLocation", + "setPermission", + "openUrl", + "launchApp", + "terminateApp", +]); + +const IOS_TOGGLES = new Set([ + "reduceMotion", + "increaseContrast", + "reduceTransparency", + "showBorders", + "voiceOver", +]); +const ANDROID_TOGGLES = new Set(["reduceMotion", "networkEnabled"]); + +const supportsAction = (platform: DevicePlatform, input: DeviceActionInput): boolean => { + const actions = platform === "ios" ? IOS_ACTIONS : ANDROID_ACTIONS; + if (!actions.has(input.type)) return false; + if (input.type === "setToggle") { + return (platform === "ios" ? IOS_TOGGLES : ANDROID_TOGGLES).has(input.setting); + } + return true; +}; + +/** What a failed command said, trimmed for the browser. */ +const failureDetail = (result: { stderr: string; stdout: string }) => { + const text = (result.stderr.trim() || result.stdout.trim()).slice(0, 1_999); + return text ? { detail: text } : {}; +}; + +const ok = (operation: string) => (result: { code: number; stderr: string; stdout: string }) => + result.code === 0 + ? Effect.succeed(result.stdout) + : Effect.fail( + new DeviceOperationError({ + operation, + reason: "command_failed", + exitCode: result.code, + ...failureDetail(result), + }), + ); + +const IOS_TEXT_SIZES: Record = { + small: "small", + default: "large", + large: "extra-extra-large", + "extra-large": "accessibility-large", +}; +const ANDROID_TEXT_SIZES: Record = { + small: "0.85", + default: "1.0", + large: "1.15", + "extra-large": "1.3", +}; + +const textSizeFromIos = (category: string): DeviceTextSize | undefined => { + const entry = (Object.entries(IOS_TEXT_SIZES) as Array<[DeviceTextSize, string]>).find( + ([, value]) => value === category, + ); + if (entry) return entry[0]; + if (category.startsWith("accessibility")) return "extra-large"; + if (category === "extra-small" || category === "small" || category === "medium") return "small"; + return category.includes("extra") ? "large" : "default"; +}; + +const textSizeFromAndroid = (scale: number): DeviceTextSize => { + if (scale <= 0.9) return "small"; + if (scale >= 1.25) return "extra-large"; + if (scale >= 1.1) return "large"; + return "default"; +}; + +const IOS_TOGGLE_OPTIONS: Record = { + reduceMotion: "reduce-motion", + increaseContrast: "increase-contrast", + reduceTransparency: "reduce-transparency", + showBorders: "show-borders", + voiceOver: "voiceover", +}; + +const IOS_TCC_SERVICES: Record = { + camera: "camera", + microphone: "microphone", + photos: "photos", + contacts: "contacts", + calendar: "calendar", + reminders: "reminders", + motion: "motion", + "media-library": "media-library", + faceid: "faceid", +}; + +const ANDROID_PERMISSIONS: Record> = { + camera: ["android.permission.CAMERA"], + microphone: ["android.permission.RECORD_AUDIO"], + photos: ["android.permission.READ_MEDIA_IMAGES", "android.permission.READ_EXTERNAL_STORAGE"], + contacts: ["android.permission.READ_CONTACTS", "android.permission.WRITE_CONTACTS"], + calendar: ["android.permission.READ_CALENDAR", "android.permission.WRITE_CALENDAR"], + location: [ + "android.permission.ACCESS_FINE_LOCATION", + "android.permission.ACCESS_COARSE_LOCATION", + ], + notifications: ["android.permission.POST_NOTIFICATIONS"], + motion: ["android.permission.ACTIVITY_RECOGNITION"], +}; + +// Android numbers rotations from the display's natural orientation and the emulator's accelerometer +// reports gravity in that same frame, so both tables are keyed by the orientation a portrait-native +// device shows at each rotation. +const ANDROID_GRAVITY: Record = { + portrait: "0:9.81:0", + landscape_left: "9.81:0:0", + portrait_upside_down: "0:-9.81:0", + landscape_right: "-9.81:0:0", +}; +const ANDROID_ROTATION: Record = { + portrait: "0", + landscape_left: "1", + portrait_upside_down: "2", + landscape_right: "3", +}; +// A landscape-native tablet sits a quarter turn further along that sequence: its portrait is +// rotation 3 and its landscape is rotation 0. +const ANDROID_LANDSCAPE_NATIVE_ROTATION: Record = { + portrait: "landscape_right", + landscape_left: "portrait", + portrait_upside_down: "landscape_left", + landscape_right: "portrait_upside_down", +}; + +// `wm size` reports the natural-orientation size; an `Override size:` line is what the window +// manager uses and is printed after `Physical size:`. +const isLandscapeNative = (wmSize: string | undefined) => { + const size = wmSize ? [...wmSize.matchAll(/size: (\d+)x(\d+)/g)].at(-1) : undefined; + return size !== undefined && Number(size[1]) > Number(size[2]); +}; + +export const runDeviceAction = Effect.fn("DeviceActions.run")(function* ( + ready: DeviceHostReady, + platform: DevicePlatform, + input: DeviceActionInput, +) { + if (!supportsAction(platform, input)) { + return yield* new DeviceActionUnavailableError({ + operation: input.type, + platform, + reason: "unsupported", + }); + } + if (platform === "ios") return yield* runIos(ready, input); + return yield* runAndroid(ready.run, input); +}); + +const simctl = (run: Runner, udid: string, args: ReadonlyArray, operation: string) => + run("xcrun", ["simctl", ...args.slice(0, 1), udid, ...args.slice(1)]).pipe( + Effect.flatMap(ok(operation)), + ); + +const axSettings = Effect.fn("DeviceActions.axSettings")(function* ( + ready: DeviceHostReady, + udid: string, + args: ReadonlyArray, +) { + const helper = ready.helpers.serveSimAxSettings; + if (!helper) { + return yield* new DeviceActionUnavailableError({ + operation: "accessibility", + platform: "ios", + reason: "helper_missing", + }); + } + return yield* ready + .run("xcrun", ["simctl", "spawn", udid, helper, ...args]) + .pipe(Effect.flatMap(ok("accessibility"))); +}); + +const runIos = Effect.fn("DeviceActions.runIos")(function* ( + ready: DeviceHostReady, + input: DeviceActionInput, +) { + const udid = input.deviceId; + const { run } = ready; + switch (input.type) { + case "setAppearance": + yield* simctl(run, udid, ["ui", "appearance", input.value], "appearance"); + return; + case "setTextSize": + yield* simctl(run, udid, ["ui", "content_size", IOS_TEXT_SIZES[input.value]], "text size"); + return; + case "setToggle": + if (input.setting === "increaseContrast") { + yield* simctl( + run, + udid, + ["ui", "increase_contrast", input.value ? "enabled" : "disabled"], + "increase contrast", + ); + return; + } + yield* axSettings(ready, udid, [ + "set", + IOS_TOGGLE_OPTIONS[input.setting]!, + input.value ? "on" : "off", + ]); + return; + case "setLiquidGlass": + yield* axSettings(ready, udid, ["set", "liquid-glass", input.value]); + return; + case "setColorFilter": + yield* axSettings(ready, udid, ["set", "color-filter", input.value]); + return; + case "setLocation": + yield* simctl( + run, + udid, + ["location", "set", `${input.latitude},${input.longitude}`], + "location", + ); + return; + case "clearLocation": + yield* simctl(run, udid, ["location", "clear"], "location"); + return; + case "setPermission": { + if (input.permission === "notifications") { + // simctl lacks notification permissions; serve-sim edits the BulletinBoard plist. + yield* serveSimPermissions(ready, udid, input); + return; + } + const service = + input.permission === "location" ? "location" : IOS_TCC_SERVICES[input.permission]; + if (!service) + return yield* new DeviceActionUnavailableError({ + operation: "permission", + platform: "ios", + reason: "unsupported", + }); + yield* simctl(run, udid, ["privacy", input.decision, service, input.appId], "permission"); + return; + } + case "openUrl": + yield* simctl(run, udid, ["openurl", input.url], "open url"); + return; + case "launchApp": + yield* simctl(run, udid, ["launch", input.appId], "launch"); + return; + case "terminateApp": + yield* simctl(run, udid, ["terminate", input.appId], "terminate"); + return; + case "sendPush": { + const payload = + typeof input.payload === "string" ? { aps: { alert: input.payload } } : input.payload; + const encoded = yield* encodePushPayload(payload).pipe( + Effect.mapError( + () => new DeviceOperationError({ operation: "push", reason: "invalid_payload" }), + ), + ); + yield* run("xcrun", ["simctl", "push", udid, input.appId, "-"], { stdin: encoded }).pipe( + Effect.flatMap(ok("push")), + ); + return; + } + case "shake": + case "setOrientation": + return yield* new DeviceActionUnavailableError({ + operation: input.type, + platform: "ios", + reason: "unsupported", + }); + } +}); + +const serveSimPermissions = Effect.fn("DeviceActions.serveSimPermissions")(function* ( + ready: DeviceHostReady, + udid: string, + input: Extract, +) { + const cli = ready.helpers.serveSimCli; + if (!cli) + return yield* new DeviceActionUnavailableError({ + operation: "permission", + platform: "ios", + reason: "helper_missing", + }); + yield* ready + .run(ready.nodePath, [ + cli, + "permissions", + input.decision, + input.permission, + input.appId, + "-d", + udid, + ]) + .pipe(Effect.flatMap(ok("permission"))); +}); + +const adb = (run: Runner, serial: string, args: ReadonlyArray, operation: string) => + run("adb", ["-s", serial, ...args]).pipe(Effect.flatMap(ok(operation))); + +// adb shell joins arguments without escaping before the device's shell parses them. +const androidShellWord = (value: string) => + /^[\w@%+=:,./-]+$/.test(value) ? value : `'${value.replaceAll("'", "'\\''")}'`; + +const runAndroid = Effect.fn("DeviceActions.runAndroid")(function* ( + run: Runner, + input: DeviceActionInput, +) { + const serial = input.deviceId; + const shell = (args: ReadonlyArray, operation: string) => + adb(run, serial, ["shell", ...args.map(androidShellWord)], operation); + switch (input.type) { + case "setAppearance": + yield* shell(["cmd", "uimode", "night", input.value === "dark" ? "yes" : "no"], "appearance"); + return; + case "setTextSize": + yield* shell( + ["settings", "put", "system", "font_scale", ANDROID_TEXT_SIZES[input.value]], + "text size", + ); + return; + case "setToggle": + if (input.setting === "networkEnabled") { + const state = input.value ? "enable" : "disable"; + yield* shell(["svc", "wifi", state], "network"); + yield* shell(["svc", "data", state], "network"); + return; + } + if (input.setting === "reduceMotion") { + const scale = input.value ? "0" : "1"; + for (const key of [ + "animator_duration_scale", + "transition_animation_scale", + "window_animation_scale", + ]) { + yield* shell(["settings", "put", "global", key, scale], "reduce motion"); + } + return; + } + return yield* new DeviceActionUnavailableError({ + operation: input.type, + platform: "android", + reason: "unsupported", + }); + case "setOrientation": { + const wmSize = yield* quiet(shell(["wm", "size"], "orientation")); + const rotation = isLandscapeNative(wmSize) + ? ANDROID_LANDSCAPE_NATIVE_ROTATION[input.value] + : input.value; + // `user-rotation lock` only rotates window content on recent images; the display the + // encoder captures stays put. Tilting the emulator's accelerometer rotates it for real, so + // that is used whenever the target is an emulator. + if (serial.startsWith("emulator-")) { + yield* shell(["settings", "put", "system", "accelerometer_rotation", "1"], "orientation"); + yield* shell(["cmd", "window", "user-rotation", "free"], "orientation"); + yield* adb( + run, + serial, + ["emu", "sensor", "set", "acceleration", ANDROID_GRAVITY[rotation]], + "orientation", + ); + return; + } + yield* shell( + ["cmd", "window", "user-rotation", "lock", ANDROID_ROTATION[rotation]], + "orientation", + ); + return; + } + case "setLocation": + yield* adb( + run, + serial, + ["emu", "geo", "fix", String(input.longitude), String(input.latitude)], + "location", + ); + return; + case "clearLocation": + // The emulator cannot clear location; keep the fix and refresh the reading. + return; + case "setPermission": { + const permissions = ANDROID_PERMISSIONS[input.permission]; + if (!permissions) { + return yield* new DeviceActionUnavailableError({ + operation: "permission", + platform: "android", + reason: "unsupported", + }); + } + const verb = input.decision === "grant" ? "grant" : "revoke"; + // `pm` rejects a permission the app or system image does not declare, and Photos lists both + // the Android 13 and the older storage permission, so fail only when none of them applied. + const results = yield* Effect.forEach(permissions, (permission) => + shell(["pm", verb, input.appId, permission], "permission").pipe(Effect.result), + ); + const failure = results.find(Result.isFailure); + if (failure && !results.some(Result.isSuccess)) return yield* failure.failure; + return; + } + case "openUrl": + yield* shell( + ["am", "start", "-a", "android.intent.action.VIEW", "-d", input.url], + "open url", + ); + return; + case "launchApp": + yield* shell( + ["monkey", "-p", input.appId, "-c", "android.intent.category.LAUNCHER", "1"], + "launch", + ); + return; + case "terminateApp": + yield* shell(["am", "force-stop", input.appId], "terminate"); + return; + case "setLiquidGlass": + case "setColorFilter": + case "shake": + case "sendPush": + return yield* new DeviceActionUnavailableError({ + operation: input.type, + platform: "android", + reason: "unsupported", + }); + } +}); + +export const readDeviceDetail = Effect.fn("DeviceActions.readDetail")(function* ( + ready: DeviceHostReady, + platform: DevicePlatform, + deviceId: string, +): Effect.fn.Return<{ settings: DeviceSettings; foregroundApp: DeviceForegroundApp | null }> { + return platform === "ios" + ? yield* readIos(ready, deviceId) + : yield* readAndroid(ready.run, deviceId); +}); + +const quiet = (effect: Effect.Effect) => + effect.pipe(Effect.orElseSucceed((): A | undefined => undefined)); + +const readIos = Effect.fn("DeviceActions.readIos")(function* ( + ready: DeviceHostReady, + udid: string, +) { + const { run } = ready; + const uiValue = (option: string) => + quiet( + simctl(run, udid, ["ui", option], option).pipe(Effect.map((out) => out.trim().toLowerCase())), + ); + const [appearance, contentSize, contrast, axStatus] = yield* Effect.all( + [ + uiValue("appearance"), + uiValue("content_size"), + uiValue("increase_contrast"), + quiet(axSettings(ready, udid, ["status"]).pipe(Effect.flatMap(decodeAxStatus))), + ], + { concurrency: 4 }, + ); + const onOff = (value: string | undefined) => + value === "on" ? true : value === "off" ? false : undefined; + const settings: DeviceSettings = { + ...(appearance === "light" || appearance === "dark" ? { appearance } : {}), + ...(contentSize ? { textSize: textSizeFromIos(contentSize) } : {}), + ...(contrast ? { increaseContrast: contrast === "enabled" } : {}), + ...(axStatus + ? { + ...(onOff(axStatus["reduce-motion"]) === undefined + ? {} + : { reduceMotion: onOff(axStatus["reduce-motion"]) }), + ...(onOff(axStatus["reduce-transparency"]) === undefined + ? {} + : { reduceTransparency: onOff(axStatus["reduce-transparency"]) }), + ...(onOff(axStatus["show-borders"]) === undefined + ? {} + : { showBorders: onOff(axStatus["show-borders"]) }), + ...(onOff(axStatus.voiceover) === undefined + ? {} + : { voiceOver: onOff(axStatus.voiceover) }), + ...(axStatus["liquid-glass"] === "clear" || axStatus["liquid-glass"] === "tinted" + ? { liquidGlass: axStatus["liquid-glass"] } + : {}), + ...(isColorFilter(axStatus["color-filter"]) + ? { colorFilter: axStatus["color-filter"] } + : {}), + } + : {}), + }; + return { settings, foregroundApp: null }; +}); + +const isColorFilter = (value: unknown): value is DeviceSettings["colorFilter"] & string => + value === "none" || + value === "grayscale" || + value === "red-green" || + value === "green-red" || + value === "blue-yellow"; + +const readAndroid = Effect.fn("DeviceActions.readAndroid")(function* (run: Runner, serial: string) { + const shell = (args: ReadonlyArray) => + quiet( + adb(run, serial, ["shell", ...args], args[0] ?? "shell").pipe(Effect.map((s) => s.trim())), + ); + const [night, fontScale, animator, wifi, focus] = yield* Effect.all( + [ + shell(["cmd", "uimode", "night"]), + shell(["settings", "get", "system", "font_scale"]), + shell(["settings", "get", "global", "animator_duration_scale"]), + shell(["settings", "get", "global", "wifi_on"]), + // API 36 dropped focus from `dumpsys window windows`; use the unfiltered dump. + shell(["dumpsys", "window"]), + ], + { concurrency: 5 }, + ); + const scale = fontScale && fontScale !== "null" ? Number(fontScale) : Number.NaN; + const focused = focus?.match(/m(?:CurrentFocus|FocusedApp)=\w+\{[^ ]+ u\d+ ([^/ ]+)\//); + const settings: DeviceSettings = { + ...(night?.includes("yes") + ? { appearance: "dark" } + : night?.includes("no") + ? { appearance: "light" } + : {}), + ...(Number.isFinite(scale) ? { textSize: textSizeFromAndroid(scale) } : {}), + ...(animator !== undefined && animator !== "null" + ? { reduceMotion: Number(animator) === 0 } + : {}), + ...(wifi === "1" || wifi === "0" ? { networkEnabled: wifi === "1" } : {}), + }; + return { + settings, + foregroundApp: focused ? { id: focused[1]! } : null, + }; +}); diff --git a/apps/server/src/device/DeviceChannels.test.ts b/apps/server/src/device/DeviceChannels.test.ts new file mode 100644 index 000000000..f9040ed29 --- /dev/null +++ b/apps/server/src/device/DeviceChannels.test.ts @@ -0,0 +1,239 @@ +import * as Http from "node:http"; +import type { AddressInfo } from "node:net"; + +import { WebSocketServer } from "@effect/platform-node/NodeWS"; +import { describe, expect, it } from "@effect/vitest"; +import { + DEVICE_CHANNEL_CHUNK_BYTES, + DEVICE_CHANNEL_WINDOW_BYTES, + type DeviceChannelEvent, +} from "@threadlines/contracts"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Queue from "effect/Queue"; +import * as Stream from "effect/Stream"; +import { FetchHttpClient } from "effect/http"; + +import * as DeviceChannels from "./DeviceChannels.ts"; +import { DeviceService } from "./DeviceService.ts"; + +const TOKEN = "hub-session-token-for-tests"; +const BODY_BYTES = 5 * 1024 * 1024; + +/** A stand-in hub: refuses requests without the token, streams a large body, echoes ws messages. */ +const startFakeHub = Effect.acquireRelease( + Effect.callback<{ origin: string; seen: Array; close: () => void }>((resume) => { + const seen: Array = []; + const server = Http.createServer((request, response) => { + seen.push(`${request.method} ${request.url} ${request.headers.authorization ?? "-"}`); + if (request.headers.authorization !== `Bearer ${TOKEN}`) { + response.writeHead(401).end(); + return; + } + response.writeHead(200, { "content-type": "application/octet-stream" }); + let sent = 0; + const pump = () => { + while (sent < BODY_BYTES) { + const size = Math.min(64 * 1024, BODY_BYTES - sent); + sent += size; + if (!response.write(Buffer.alloc(size, 7))) { + response.once("drain", pump); + return; + } + } + response.end(); + }; + pump(); + }); + const sockets = new WebSocketServer({ noServer: true }); + server.on("upgrade", (request, socket, head) => { + seen.push(`WS ${request.url} ${request.headers.authorization ?? "-"}`); + if (request.headers.authorization !== `Bearer ${TOKEN}`) { + socket.destroy(); + return; + } + sockets.handleUpgrade(request, socket, head, (ws) => { + ws.on("message", (data, isBinary) => ws.send(data, { binary: isBinary })); + }); + }); + server.listen(0, "127.0.0.1", () => { + const { port } = server.address() as AddressInfo; + resume( + Effect.succeed({ + origin: `http://127.0.0.1:${port}`, + seen, + close: () => { + sockets.close(); + server.closeAllConnections(); + server.close(); + }, + }), + ); + }); + }), + (hub) => Effect.sync(() => hub.close()), +); + +const channelsFor = (origin: string) => + DeviceChannels.make.pipe( + Effect.provide( + Layer.mergeAll( + Layer.mock(DeviceService)({ + currentHub: Effect.succeed({ origin, token: TOKEN }), + }), + FetchHttpClient.layer, + ), + ), + ); + +describe("isAllowedDevicePath", () => { + it("keeps serve-sim's and serve-emu's /api bodies, which carry the exec token, out of reach", () => { + expect(DeviceChannels.isAllowedDevicePath("GET", "/vendor/serve-sim/api")).toBe(false); + expect(DeviceChannels.isAllowedDevicePath("GET", "/vendor/serve-emu/api")).toBe(false); + expect(DeviceChannels.isAllowedDevicePath("ws", "/vendor/serve-sim/exec-ws")).toBe(false); + expect(DeviceChannels.isAllowedDevicePath("POST", "/api/devices/boot")).toBe(false); + }); + + it("refuses segments a URL parser would rewrite", () => { + expect( + DeviceChannels.isAllowedDevicePath("GET", "/vendor/serve-sim/helper/../stream.avcc"), + ).toBe(false); + expect( + DeviceChannels.isAllowedDevicePath("GET", "/vendor/serve-sim/helper/%2e%2e/stream.avcc"), + ).toBe(false); + }); + + it("allows the viewer's stream and input routes", () => { + expect( + DeviceChannels.isAllowedDevicePath("GET", "/vendor/serve-sim/helper/ABC-1/stream.avcc"), + ).toBe(true); + expect(DeviceChannels.isAllowedDevicePath("ws", "/vendor/serve-sim/helper/ws")).toBe(true); + expect(DeviceChannels.isAllowedDevicePath("ws", "/vendor/serve-emu/ws")).toBe(true); + expect(DeviceChannels.isAllowedDevicePath("POST", "/vendor/serve-emu/api/stream-mode")).toBe( + true, + ); + }); +}); + +describe("DeviceChannels", () => { + it.live("refuses a forbidden path without reaching the hub", () => + Effect.scoped( + Effect.gen(function* () { + const hub = yield* startFakeHub; + const channels = yield* channelsFor(hub.origin); + const error = yield* channels + .open("connection", { + _tag: "http", + channelId: "c1", + method: "GET", + path: "/vendor/serve-sim/api", + }) + .pipe(Effect.flip); + expect(error.reason).toBe("forbidden"); + expect(hub.seen).toEqual([]); + }), + ), + ); + + it.live("adds the hub token, slices the body, and holds it at the credit window", () => + Effect.scoped( + Effect.gen(function* () { + const hub = yield* startFakeHub; + const channels = yield* channelsFor(hub.origin); + const events = yield* channels.open("connection", { + _tag: "http", + channelId: "c1", + method: "GET", + path: "/vendor/serve-sim/helper/SIM-1/stream.avcc", + query: { token: "client-supplied", fps: "30" }, + }); + const received = yield* Queue.unbounded(); + const reader = yield* events.pipe( + Stream.runForEach((event) => Queue.offer(received, event)), + Effect.forkScoped, + ); + let delivered = 0; + const take = Effect.gen(function* () { + const event = yield* Queue.take(received); + if (event._tag === "data") { + const size = Buffer.from(event.data, "base64").length; + expect(size).toBeLessThanOrEqual(DEVICE_CHANNEL_CHUNK_BYTES); + delivered += size; + } + return event; + }); + expect((yield* take)._tag).toBe("opened"); + // Uncredited, the channel stops at the window instead of reading the whole body. + yield* Effect.sleep("300 millis"); + while ((yield* Queue.size(received)) > 0) yield* take; + expect(delivered).toBeGreaterThanOrEqual(DEVICE_CHANNEL_WINDOW_BYTES); + expect(delivered).toBeLessThan(DEVICE_CHANNEL_WINDOW_BYTES + DEVICE_CHANNEL_CHUNK_BYTES); + // Crediting lets the rest through. + let ended = false; + while (!ended) { + yield* channels.credit("connection", { channelId: "c1", bytes: delivered || 1 }); + const before = delivered; + const event = yield* take; + ended = event._tag === "end"; + if (delivered > before) { + yield* channels.credit("connection", { channelId: "c1", bytes: delivered - before }); + } + } + expect(delivered).toBe(BODY_BYTES); + yield* Fiber.join(reader); + // The client's token is dropped and the server's bearer added. + expect(hub.seen).toEqual([ + `GET /vendor/serve-sim/helper/SIM-1/stream.avcc?fps=30 Bearer ${TOKEN}`, + ]); + }), + ), + ); + + it.live("relays ws messages both ways with the hub token", () => + Effect.scoped( + Effect.gen(function* () { + const hub = yield* startFakeHub; + const channels = yield* channelsFor(hub.origin); + const events = yield* channels.open("connection", { + _tag: "ws", + channelId: "w1", + path: "/vendor/serve-sim/helper/ws", + }); + const received = yield* Queue.unbounded(); + yield* events.pipe( + Stream.runForEach((event) => Queue.offer(received, event)), + Effect.forkScoped, + ); + expect((yield* Queue.take(received))._tag).toBe("opened"); + // Sent out of order, written in order. + yield* Effect.all( + [ + channels.send("connection", { channelId: "w1", seq: 1, data: "second", binary: false }), + channels.send("connection", { + channelId: "w1", + seq: 0, + data: Buffer.from("first").toString("base64"), + binary: true, + }), + ], + { concurrency: "unbounded" }, + ); + const first = yield* Queue.take(received); + const second = yield* Queue.take(received); + expect(first).toEqual({ + _tag: "message", + data: Buffer.from("first").toString("base64"), + binary: true, + final: true, + }); + expect(second).toEqual({ _tag: "message", data: "second", binary: false, final: true }); + expect(hub.seen).toEqual([`WS /vendor/serve-sim/helper/ws Bearer ${TOKEN}`]); + // The connection going away closes its channels. + yield* channels.releaseConnection("connection"); + const end = yield* Queue.take(received); + expect(end._tag).toBe("end"); + }), + ), + ); +}); diff --git a/apps/server/src/device/DeviceChannels.ts b/apps/server/src/device/DeviceChannels.ts new file mode 100644 index 000000000..5199ff5fc --- /dev/null +++ b/apps/server/src/device/DeviceChannels.ts @@ -0,0 +1,471 @@ +/** + * The device channel (docs/design/devices.md, "Transport"): every byte + * between a Device tab and the hub rides the client's WS RPC connection, so + * the desktop app, a browser, a phone and a saved computer over the relay all + * reach the hub the same way. Modeled on PreviewTunnel. + * + * The hub is never reachable from a client directly. A channel opens only on + * a path the allowlist below names, with the hub's token added here (clients + * never see it) and nothing of the client's but the path, query and body. + * serve-sim's `/api` body carries the token of its shell-exec route, so + * neither `/vendor/serve-sim/api` nor `/vendor/serve-emu/api` is allowed. + * + * Flow control, so a suspended phone cannot pile video up in the server: + * events carry at most DEVICE_CHANNEL_CHUNK_BYTES; at most + * DEVICE_CHANNEL_WINDOW_BYTES may be sent and not yet credited. An http body + * then simply stops being read (the hub's socket backs up). A ws upstream is + * paused; when more than DEVICE_CHANNEL_SLOW_BYTES queue up past the window + * anyway, the channel fails with `slow` and the client opens a fresh one. + */ +import { + DEVICE_CHANNEL_CHUNK_BYTES, + DEVICE_CHANNEL_WINDOW_BYTES, + type DeviceChannelCredit, + DeviceChannelError, + type DeviceChannelEvent, + type DeviceChannelOpen, + type DeviceChannelSend, +} from "@threadlines/contracts"; +import { WebSocket } from "@effect/platform-node/NodeWS"; +import * as Cause from "effect/Cause"; +import * as Context from "effect/Context"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Latch from "effect/Latch"; +import * as Layer from "effect/Layer"; +import * as Queue from "effect/Queue"; +import type * as Scope from "effect/Scope"; +import * as Stream from "effect/Stream"; +import { HttpClient, HttpClientRequest } from "effect/http"; + +import { DeviceService } from "./DeviceService.ts"; +import { repeatStillFrame } from "./mjpegStillFrame.ts"; + +/** Channels one connection may hold open: a Device tab uses two or three. */ +export const DEVICE_CHANNEL_MAX_PER_CONNECTION = 16; +/** What a paused ws upstream may still queue past the window before the channel fails `slow`. */ +export const DEVICE_CHANNEL_SLOW_BYTES = 4 * 1024 * 1024; +const OPEN_TIMEOUT_MS = 10_000; +const MAX_SENDS_OUT_OF_ORDER = 64; + +/** Read routes and stream routes, for GET. */ +const GET_PATHS: ReadonlyArray = [ + /^\/api\/devices$/, + /^\/vendor\/serve-sim\/api\/screenshot$/, + /^\/vendor\/serve-sim\/api\/event-log(\/events)?$/, + /^\/vendor\/serve-sim\/helper\/[^/]+\/(stream\.mjpeg|stream\.avcc|config|health|ax|foreground)$/, + /^\/vendor\/serve-sim\/helper\/[^/]+\/panel\/(1|3)\/stream\.avcc$/, + /^\/vendor\/serve-sim\/appstate$/, + /^\/vendor\/serve-emu\/api\/(devices|screenshot|stream-mode|stream-settings|accessibility|fold)$/, + /^\/vendor\/serve-emu\/health$/, +]; + +/** The few routes that take a POST: screenshot capture and stream tuning. */ +const POST_PATHS: ReadonlyArray = [ + /^\/vendor\/serve-sim\/api\/screenshot$/, + /^\/vendor\/serve-emu\/api\/(screenshot|stream-mode|stream-settings|fold)$/, +]; + +/** Sockets: the device list, iOS input, and Android's video and input. */ +const WS_PATHS: ReadonlyArray = [ + /^\/api\/devices\/ws$/, + /^\/vendor\/serve-sim\/helper\/ws$/, + /^\/vendor\/serve-emu\/ws$/, +]; + +const SCREENSHOT_PATHS = new Set([ + "/vendor/serve-sim/api/screenshot", + "/vendor/serve-emu/api/screenshot", +]); + +/** + * Whether a client may open `path` this way. Segments that a URL parser + * would rewrite (`.`, `..`, escapes, backslashes, empty) are refused outright, + * so what is checked is what reaches the hub. + */ +export const isAllowedDevicePath = (kind: "GET" | "POST" | "ws", path: string): boolean => { + const segments = path.split("/").slice(1); + if ( + segments.some( + (segment) => + segment === "" || + segment === "." || + segment === ".." || + segment.includes("%") || + segment.includes("\\"), + ) + ) { + return false; + } + const patterns = kind === "ws" ? WS_PATHS : kind === "POST" ? POST_PATHS : GET_PATHS; + return patterns.some((pattern) => pattern.test(path)); +}; + +/** The query a client sent, without anything that could carry a token of its own. */ +const upstreamSearch = (query: Readonly> | undefined) => { + const params = new URLSearchParams(); + for (const [name, value] of Object.entries(query ?? {})) { + if (name.toLowerCase() === "token") continue; + params.append(name, value); + } + const text = params.toString(); + return text === "" ? "" : `?${text}`; +}; + +const oneEvent = (event: DeviceChannelEvent): Stream.Stream => + Stream.succeed(event); + +const failure = (reason: DeviceChannelError["reason"], detail: string) => + new DeviceChannelError({ reason, detail }); + +/** Slices of at most DEVICE_CHANNEL_CHUNK_BYTES, base64. */ +const toChunks = (bytes: Uint8Array): ReadonlyArray => { + const buffer = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const chunks: Array = []; + for (let offset = 0; offset < buffer.length; offset += DEVICE_CHANNEL_CHUNK_BYTES) { + chunks.push(buffer.subarray(offset, offset + DEVICE_CHANNEL_CHUNK_BYTES).toString("base64")); + } + return chunks; +}; + +interface PendingSend { + readonly data: Buffer | string; + readonly done: Deferred.Deferred; +} + +interface Channel { + /** Bytes sent down the channel that the client has not yet credited. */ + unacknowledged: number; + /** Credit arrived: let a waiting http body continue, or resume a paused socket. */ + readonly onCredit: () => void; + /** Set for a ws channel. */ + readonly socket: WebSocket | null; + nextSeq: number; + readonly waiting: Map; +} + +export interface DeviceChannelsShape { + /** Opens one channel. Scoped: the subscription ending closes it. */ + readonly open: ( + connectionId: string, + input: DeviceChannelOpen, + ) => Effect.Effect< + Stream.Stream, + DeviceChannelError, + Scope.Scope + >; + /** One ws message for the hub, in `seq` order; done once written to the hub's socket. */ + readonly send: ( + connectionId: string, + input: DeviceChannelSend, + ) => Effect.Effect; + /** The client's decoder took more of a channel's data. */ + readonly credit: (connectionId: string, input: DeviceChannelCredit) => Effect.Effect; + /** The connection went away: every channel it held closes. */ + readonly releaseConnection: (connectionId: string) => Effect.Effect; +} + +export class DeviceChannels extends Context.Service()( + "@threadlines/server/device/DeviceChannels", +) {} + +export const make = Effect.gen(function* () { + const devices = yield* DeviceService; + const httpClient = (yield* HttpClient.HttpClient).pipe(HttpClient.withScope); + const channels = new Map(); + /** Every channel's closer, by connection, for `releaseConnection`. */ + const closers = new Map void>>(); + const keyOf = (connectionId: string, channelId: string) => `${connectionId}\u0000${channelId}`; + + const settleWaiting = (channel: Channel, error: DeviceChannelError) => { + for (const pending of channel.waiting.values()) + Deferred.doneUnsafe(pending.done, Exit.fail(error)); + channel.waiting.clear(); + }; + + /** Holds the connection's slot and the channel's entry until the scope closes. */ + const register = (connectionId: string, channelId: string, channel: Channel, close: () => void) => + Effect.gen(function* () { + const key = keyOf(connectionId, channelId); + const held = closers.get(connectionId) ?? new Map void>(); + if (channels.has(key)) return yield* failure("limit", "That channel is already open."); + if (held.size >= DEVICE_CHANNEL_MAX_PER_CONNECTION) { + return yield* failure("limit", `Too many device channels are open (${held.size}).`); + } + held.set(channelId, close); + closers.set(connectionId, held); + channels.set(key, channel); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + if (channels.get(key) === channel) channels.delete(key); + held.delete(channelId); + if (held.size === 0 && closers.get(connectionId) === held) closers.delete(connectionId); + settleWaiting(channel, failure("broken", "The channel closed.")); + close(); + }), + ); + }); + + const hubOrUnavailable = devices.currentHub.pipe( + Effect.flatMap((hub) => + hub + ? Effect.succeed(hub) + : Effect.fail(failure("unavailable", "The device hub is not running.")), + ), + ); + + const openHttp = Effect.fn("DeviceChannels.openHttp")(function* ( + connectionId: string, + input: Extract, + ) { + if (!isAllowedDevicePath(input.method, input.path)) { + return yield* failure("forbidden", "That device path is not available."); + } + const credited = yield* Latch.make(true); + const channel: Channel = { + unacknowledged: 0, + onCredit: () => { + if (channel.unacknowledged < DEVICE_CHANNEL_WINDOW_BYTES) Latch.openUnsafe(credited); + }, + socket: null, + nextSeq: 0, + waiting: new Map(), + }; + yield* register(connectionId, input.channelId, channel, () => Latch.openUnsafe(credited)); + + // Waits for credit before each slice goes out: the body is pulled only as fast as the client takes it. + const paced = (slices: Stream.Stream) => + slices.pipe( + Stream.mapEffect((data) => + Effect.gen(function* () { + yield* credited.await; + // Exact decoded size: the client credits decoded bytes, so an estimate would drift. + channel.unacknowledged += Buffer.byteLength(data, "base64"); + if (channel.unacknowledged >= DEVICE_CHANNEL_WINDOW_BYTES) Latch.closeUnsafe(credited); + return { _tag: "data", data } as const; + }), + ), + ); + + // A screenshot asked for by device is the server's own, checked to be one PNG. + const screenshotDevice = input.query?.device; + if (SCREENSHOT_PATHS.has(input.path) && screenshotDevice) { + const shot = yield* devices + .screenshot(screenshotDevice) + .pipe(Effect.mapError((error) => failure("unreachable", error.message))); + return Stream.concat( + oneEvent({ _tag: "opened", status: 200, contentType: "image/png" }), + Stream.concat(paced(Stream.fromIterable(toChunks(shot.png))), oneEvent({ _tag: "end" })), + ); + } + + const hub = yield* hubOrUnavailable; + const url = `${hub.origin}${input.path}${upstreamSearch(input.query)}`; + const request = ( + input.method === "POST" ? HttpClientRequest.post(url) : HttpClientRequest.get(url) + ).pipe( + HttpClientRequest.bearerToken(hub.token), + HttpClientRequest.setHeader("accept-encoding", "identity"), + input.body === undefined + ? (self) => self + : HttpClientRequest.bodyUint8Array( + Buffer.from(input.body, "base64"), + input.contentType ?? "application/octet-stream", + ), + ); + const response = yield* httpClient.execute(request).pipe( + Effect.timeout(OPEN_TIMEOUT_MS), + Effect.mapError(() => failure("unreachable", "The device hub did not answer.")), + ); + const contentType = response.headers["content-type"]; + // A multipart MJPEG body re-sends its last frame once the screen goes still. + const body = contentType?.startsWith("multipart/x-mixed-replace") + ? repeatStillFrame(response.stream) + : response.stream; + const slices = body.pipe( + Stream.mapError(() => failure("broken", "The device hub stopped sending.")), + Stream.flatMap((bytes) => Stream.fromIterable(toChunks(bytes))), + ); + return Stream.concat( + oneEvent({ + _tag: "opened", + status: response.status, + ...(contentType ? { contentType } : {}), + }), + Stream.concat(paced(slices), oneEvent({ _tag: "end" })), + ); + }); + + const openWs = Effect.fn("DeviceChannels.openWs")(function* ( + connectionId: string, + input: Extract, + ) { + if (!isAllowedDevicePath("ws", input.path)) { + return yield* failure("forbidden", "That device path is not available."); + } + const hub = yield* hubOrUnavailable; + const url = `${hub.origin.replace(/^http/, "ws")}${input.path}${upstreamSearch(input.query)}`; + const queue = yield* Queue.unbounded(); + let paused = false; + // An `error` with no listener would take the server down: one is on from the start. + const socket = new WebSocket(url, input.protocols ? [...input.protocols] : [], { + headers: { authorization: `Bearer ${hub.token}` }, + handshakeTimeout: OPEN_TIMEOUT_MS, + perMessageDeflate: false, + }); + socket.binaryType = "nodebuffer"; + const channel: Channel = { + unacknowledged: 0, + onCredit: () => { + if (paused && channel.unacknowledged < DEVICE_CHANNEL_WINDOW_BYTES / 2) { + paused = false; + socket.resume(); + } + }, + socket, + nextSeq: 0, + waiting: new Map(), + }; + const fail = (error: DeviceChannelError) => { + settleWaiting(channel, error); + Queue.failCauseUnsafe(queue, Cause.fail(error)); + socket.terminate(); + }; + socket.on("error", (error) => fail(failure("broken", error.message))); + // A refused open (a duplicate id, or the connection's limit) must not leave the socket up. + yield* register(connectionId, input.channelId, channel, () => socket.terminate()).pipe( + Effect.onError(() => Effect.sync(() => socket.terminate())), + ); + + socket.on("message", (data: Buffer | ArrayBuffer | Array, isBinary: boolean) => { + const bytes = Array.isArray(data) ? Buffer.concat(data) : Buffer.from(data as Buffer); + if (isBinary) { + const parts = toChunks(bytes); + parts.forEach((part, index) => + Queue.offerUnsafe(queue, { + _tag: "message", + data: part, + binary: true, + final: index === parts.length - 1, + }), + ); + if (parts.length === 0) + Queue.offerUnsafe(queue, { _tag: "message", data: "", binary: true, final: true }); + } else { + const text = bytes.toString("utf8"); + const step = DEVICE_CHANNEL_CHUNK_BYTES / 4; + for (let offset = 0; offset < Math.max(text.length, 1); offset += step) { + Queue.offerUnsafe(queue, { + _tag: "message", + data: text.slice(offset, offset + step), + binary: false, + final: offset + step >= text.length, + }); + } + } + channel.unacknowledged += bytes.length; + if (channel.unacknowledged > DEVICE_CHANNEL_WINDOW_BYTES + DEVICE_CHANNEL_SLOW_BYTES) { + fail(failure("slow", "The device stream got too far ahead of this client.")); + return; + } + if (!paused && channel.unacknowledged >= DEVICE_CHANNEL_WINDOW_BYTES) { + paused = true; + socket.pause(); + } + }); + socket.on("close", (code: number, reason: Buffer) => { + settleWaiting(channel, failure("broken", "The device socket closed.")); + const text = reason.toString("utf8"); + Queue.offerUnsafe(queue, { _tag: "end", code, ...(text ? { reason: text } : {}) }); + Queue.endUnsafe(queue); + }); + socket.once("open", () => { + Queue.offerUnsafe(queue, { + _tag: "opened", + ...(socket.protocol ? { protocol: socket.protocol } : {}), + }); + }); + yield* Effect.callback((resume) => { + if (socket.readyState === WebSocket.OPEN) return resume(Effect.void); + socket.once("open", () => resume(Effect.void)); + socket.once("unexpected-response", (_request, response) => { + resume( + Effect.fail( + failure("unreachable", `The device hub refused the socket (${response.statusCode}).`), + ), + ); + socket.terminate(); + }); + socket.once("close", () => + resume(Effect.fail(failure("unreachable", "The device hub closed the socket."))), + ); + socket.once("error", (error) => resume(Effect.fail(failure("unreachable", error.message)))); + }); + return Stream.fromQueue(queue); + }); + + const open: DeviceChannelsShape["open"] = (connectionId, input) => + input._tag === "http" ? openHttp(connectionId, input) : openWs(connectionId, input); + + const send: DeviceChannelsShape["send"] = (connectionId, input) => + Effect.gen(function* () { + const channel = channels.get(keyOf(connectionId, input.channelId)); + if (channel === undefined || channel.socket === null) { + return yield* failure("unknownChannel", "That device channel is not open."); + } + const socket = channel.socket; + if ( + input.seq < channel.nextSeq || + channel.waiting.has(input.seq) || + channel.waiting.size >= MAX_SENDS_OUT_OF_ORDER + ) { + const error = failure("limit", "The channel's messages arrived out of all order."); + settleWaiting(channel, error); + socket.terminate(); + return yield* error; + } + const done = yield* Deferred.make(); + channel.waiting.set(input.seq, { + data: input.binary ? Buffer.from(input.data, "base64") : input.data, + done, + }); + // Writes every message whose turn has come, in order. + for (;;) { + const next = channel.waiting.get(channel.nextSeq); + if (next === undefined) break; + channel.waiting.delete(channel.nextSeq); + channel.nextSeq += 1; + if (socket.readyState !== WebSocket.OPEN) { + Deferred.doneUnsafe(next.done, Exit.fail(failure("broken", "The device socket closed."))); + continue; + } + socket.send(next.data, { binary: typeof next.data !== "string" }, (error) => + Deferred.doneUnsafe( + next.done, + error ? Exit.fail(failure("broken", error.message)) : Exit.void, + ), + ); + } + yield* Deferred.await(done); + }); + + const credit: DeviceChannelsShape["credit"] = (connectionId, input) => + Effect.sync(() => { + const channel = channels.get(keyOf(connectionId, input.channelId)); + if (channel === undefined) return; + channel.unacknowledged = Math.max(0, channel.unacknowledged - input.bytes); + channel.onCredit(); + }); + + const releaseConnection: DeviceChannelsShape["releaseConnection"] = (connectionId) => + Effect.sync(() => { + const held = closers.get(connectionId); + if (!held) return; + for (const close of held.values()) close(); + }); + + return DeviceChannels.of({ open, send, credit, releaseConnection }); +}); + +export const layer = Layer.effect(DeviceChannels, make); diff --git a/apps/server/src/device/DeviceHost.ts b/apps/server/src/device/DeviceHost.ts new file mode 100644 index 000000000..1f18f6002 --- /dev/null +++ b/apps/server/src/device/DeviceHost.ts @@ -0,0 +1,154 @@ +/** + * The seam between the device service and the machine (docs/design/devices.md). + * One implementation, LocalDeviceHost: a Threadlines server has one device + * host, its own machine. A Mac linked in Fleet shows its devices through its + * own server. + */ +import type { + DeviceBootError, + DeviceHostUnavailableError, + DeviceOperationError, + DevicePlatform, + DevicePlatformAvailability, + DeviceSummary, + DeviceToolVersions, +} from "@threadlines/contracts"; +import * as Context from "effect/Context"; +import type * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +/** Something went wrong starting or running the device tools; `step` says where. */ +export class DeviceHostError extends Schema.TaggedError()("DeviceHostError", { + step: Schema.String, + /** Words the user can act on, when there are some. */ + detail: Schema.optional(Schema.String), + cause: Schema.optional(Schema.Defect()), +}) { + override get message(): string { + return this.detail ?? `Device support failed while ${this.step}.`; + } +} + +/** The running hub: where it listens and the session token it printed. Never leaves the server. */ +export interface DeviceHubEndpoint { + readonly origin: string; + readonly token: string; +} + +/** The running agent-device daemon, read from its own `daemon.json`. */ +export interface AgentDeviceEndpoint { + readonly baseUrl: string; + readonly token: string; + /** The agent-device CLI that talks to it. */ + readonly entryPath: string; +} + +/** Discovery and lifecycle for the devices a ready host can run. */ +export interface DeviceOperations { + readonly list: Effect.Effect< + { + readonly devices: ReadonlyArray; + readonly detail: string | undefined; + /** Platforms that could not be listed; their last known devices stay listed. */ + readonly incompletePlatforms: ReadonlyArray; + }, + DeviceOperationError + >; + /** + * Boots the device and prepares its stream. Android AVDs change id when they + * boot (AVD name to emulator serial), so the returned id is authoritative. + */ + readonly boot: ( + device: DeviceSummary, + ) => Effect.Effect; + /** Prepares the stream of a device that is already booted. */ + readonly attach: (device: DeviceSummary) => Effect.Effect; + readonly shutdown: (device: { + readonly id: string; + readonly platform: DevicePlatform; + }) => Effect.Effect; + readonly screenshot: (device: DeviceSummary) => Effect.Effect; +} + +export interface DeviceRunResult { + readonly stdout: string; + readonly stderr: string; + /** 124 on timeout, 127 when it could not start. */ + readonly code: number; +} + +export interface DeviceHostReady { + /** The Node.js the tools run on (Threadlines' managed one). */ + readonly nodePath: string; + readonly hub: DeviceHubEndpoint; + readonly devices: DeviceOperations; + /** Runs a command with the Android SDK on PATH. Never fails: see `code`. */ + readonly run: ( + command: string, + args: ReadonlyArray, + options?: { + readonly timeoutMs?: number; + readonly stdin?: string; + readonly maxOutputBytes?: number; + }, + ) => Effect.Effect; + /** serve-sim's helper binaries, for the Tools drawer. Null where the install lacks one. */ + readonly helpers: { + readonly serveSimAxSettings: string | null; + readonly serveSimCli: string | null; + }; + /** + * Ends an agent-device session on the running daemon. Present only while + * the daemon runs; cleanup never starts a daemon or a device. + */ + readonly releaseAgentSession?: (session: string) => Effect.Effect; +} + +export interface DeviceHostAgentReady extends DeviceHostReady { + readonly agentDevice: AgentDeviceEndpoint; +} + +export interface DeviceHostSummary { + readonly platforms: ReadonlyArray; + readonly hubInstalled: boolean; + readonly agentDeviceInstalled: boolean; + readonly tools: DeviceToolVersions | undefined; + readonly toolInspectionError: string | undefined; +} + +export type DeviceHostPhase = "installing" | "starting"; + +export class DeviceHost extends Context.Service< + DeviceHost, + { + readonly summary: Effect.Effect; + readonly platformAvailability: ( + platform: DevicePlatform, + ) => Effect.Effect; + /** Installs (when missing) and starts the hub. */ + readonly ensureReady: ( + onPhase: (phase: DeviceHostPhase, detail?: string) => Effect.Effect, + ) => Effect.Effect; + /** As `ensureReady`, and installs and starts the agent-device daemon too. */ + readonly ensureAgentReady: ( + onPhase: (phase: DeviceHostPhase, detail?: string) => Effect.Effect, + ) => Effect.Effect; + /** Installs the pinned version of a tool; restarts it if it was running. */ + readonly updateTool: (tool: "hub" | "agent") => Effect.Effect; + /** The running hub, without starting anything. */ + readonly current: Effect.Effect; + /** + * Ends an agent-device session on whatever daemon runs for this state + * folder, this server's or one an earlier run left. Never starts one. + * True when the session is gone (closed now, already released, or no + * daemon runs: a daemon that is gone took its sessions with it); false when + * a live daemon did not confirm the close, so the caller keeps its claim + * and can try again. + */ + readonly releaseAgentSession: (session: string) => Effect.Effect; + /** Stops the agent-device daemon through its own CLI. */ + readonly stopAgent: Effect.Effect; + /** Stops the hub and the daemon. Simulators and emulators keep running; the user owns those. */ + readonly stop: Effect.Effect; + } +>()("@threadlines/server/device/DeviceHost") {} diff --git a/apps/server/src/device/DeviceHubOperations.test.ts b/apps/server/src/device/DeviceHubOperations.test.ts new file mode 100644 index 000000000..f3c1fd489 --- /dev/null +++ b/apps/server/src/device/DeviceHubOperations.test.ts @@ -0,0 +1,166 @@ +import * as Http from "node:http"; +import type { AddressInfo } from "node:net"; + +import { describe, expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import { FetchHttpClient, HttpClient } from "effect/http"; + +import { + makeDeviceHubOperations, + makeDeviceModelCache, + makeHostDeviceOperations, +} from "./DeviceHubOperations.ts"; + +/** A stand-in hub answering the two listings with fixed JSON. */ +const startFakeHub = (routes: Record) => + Effect.acquireRelease( + Effect.callback<{ origin: string; close: () => void }>((resume) => { + const server = Http.createServer((request, response) => { + const body = routes[request.url ?? ""]; + if (body === undefined) { + response.writeHead(404).end(); + return; + } + response.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify(body)); + }); + server.listen(0, "127.0.0.1", () => { + const { port } = server.address() as AddressInfo; + resume(Effect.succeed({ origin: `http://127.0.0.1:${port}`, close: () => server.close() })); + }); + }), + (hub) => Effect.sync(hub.close), + ); + +describe("DeviceHubOperations list", () => { + it.effect("adds simulators that were never booted, once, from serve-sim's grid", () => + Effect.gen(function* () { + const hub = yield* startFakeHub({ + "/api/devices": { + simulators: [ + { + id: "AAAA-1", + name: "iPhone 17", + version: "iOS 27.2", + platform: "ios", + booted: true, + physical: false, + }, + ], + emulators: [], + }, + "/vendor/serve-sim/grid/api": { + devices: [ + { device: "aaaa-1", name: "iPhone 17", runtime: "iOS-27-2", state: "Booted" }, + { device: "BBBB-2", name: "iPhone 17e", runtime: "iOS-27-2", state: "Shutdown" }, + { device: "CCCC-3", name: "Apple Watch", runtime: "watchOS-12-0", state: "Shutdown" }, + ], + }, + }); + const operations = makeDeviceHubOperations({ + origin: hub.origin, + token: "t", + modelCache: makeDeviceModelCache(), + http: yield* HttpClient.HttpClient, + run: () => Effect.succeed({ code: 0, stdout: "", stderr: "" }), + platformAvailable: (platform) => Effect.succeed(platform === "ios"), + scratchDir: "/nonexistent", + }); + + const listed = yield* operations.list; + + expect(listed.devices.map((device) => [device.id, device.version, device.booted])).toEqual([ + ["AAAA-1", "iOS 27.2", true], + ["BBBB-2", "iOS 27.2", false], + ]); + }).pipe(Effect.scoped, Effect.provide(FetchHttpClient.layer)), + ); + + it.effect( + "names each device's hardware: simctl once per new simulator, AVD profile, phone model", + () => + Effect.gen(function* () { + const hub = yield* startFakeHub({ + "/api/devices": { + simulators: [ + { + id: "aaaa-1", + name: "My renamed phone", + version: "iOS 27.2", + platform: "ios", + booted: true, + physical: false, + }, + ], + emulators: [ + { + id: "emulator-5554", + name: "fold_test", + version: "Android 16.0", + platform: "android", + booted: true, + physical: false, + }, + { + id: "R5CT1234", + name: "Pixel 8", + version: "Android 16.0", + platform: "android", + booted: true, + physical: true, + }, + ], + }, + "/vendor/serve-sim/grid/api": { devices: [] }, + }); + const commands: Array = []; + const files: Record = { + "/avd/fold_test.ini": "avd.ini.encoding=UTF-8\npath=/elsewhere/fold_test.avd\n", + "/elsewhere/fold_test.avd/config.ini": + "hw.lcd.width = 2208\nhw.device.name = pixel_fold\n", + }; + // As LocalDeviceHost does: one factory per host, fresh operations on every ready hub. + const hostOperations = makeHostDeviceOperations({ + http: yield* HttpClient.HttpClient, + run: (command, args) => { + commands.push([command, ...args].join(" ")); + if (command === "xcrun") { + return Effect.succeed({ + code: 0, + stdout: JSON.stringify({ + devices: { + "com.apple.CoreSimulator.SimRuntime.iOS-27-2": [ + { + udid: "AAAA-1", + deviceTypeIdentifier: "com.apple.CoreSimulator.SimDeviceType.iPhone-Duo", + }, + ], + }, + }), + stderr: "", + }); + } + if (command === "adb") + return Effect.succeed({ code: 0, stdout: "Pixel 8\n", stderr: "" }); + return Effect.succeed({ code: 0, stdout: "", stderr: "" }); + }, + platformAvailable: () => Effect.succeed(true), + scratchDir: "/nonexistent", + androidAvds: { home: "/avd", readTextFile: (path) => Effect.succeed(files[path]) }, + }); + + const first = yield* hostOperations({ origin: hub.origin, token: "t" }).list; + const second = yield* hostOperations({ origin: hub.origin, token: "t" }).list; + + for (const listed of [first, second]) { + expect(listed.devices.map((device) => [device.id, device.model])).toEqual([ + ["aaaa-1", "com.apple.CoreSimulator.SimDeviceType.iPhone-Duo"], + ["emulator-5554", "pixel_fold"], + ["R5CT1234", "Pixel 8"], + ]); + } + // Device types and phone models are asked for once, not on every listing. + expect(commands.filter((command) => command.startsWith("xcrun"))).toHaveLength(1); + expect(commands.filter((command) => command.includes("getprop"))).toHaveLength(1); + }).pipe(Effect.scoped, Effect.provide(FetchHttpClient.layer)), + ); +}); diff --git a/apps/server/src/device/DeviceHubOperations.ts b/apps/server/src/device/DeviceHubOperations.ts new file mode 100644 index 000000000..457317acc --- /dev/null +++ b/apps/server/src/device/DeviceHubOperations.ts @@ -0,0 +1,609 @@ +/** + * List, boot, shut down and screenshot through the device hub, with the hub's + * session token on every request (docs/design/devices.md). Ported from T3 + * Code's DeviceHubOperations. + */ +import { + DeviceBootError, + DeviceOperationError, + type DevicePlatform, + type DeviceSummary, +} from "@threadlines/contracts"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import { HttpClient, HttpClientError, HttpClientRequest, HttpClientResponse } from "effect/http"; + +import type { DeviceHostReady, DeviceOperations } from "./DeviceHost.ts"; + +type HostDevice = DeviceSummary; + +const BOOT_TIMEOUT = Duration.minutes(3); +const SCREENSHOT_TIMEOUT = Duration.seconds(20); +// Let the hub's 30 s simctl timeout report its cause first. +const LIST_TIMEOUT_SECONDS = 40; +// expo-device-hub 0.19.1 reports a failed simulator listing only as text; its error ids are hashes. +const SIMULATOR_LIST_FAILED = + /^\[apple-utils\] (?:Failed to (?:run `xcrun simctl list devices|parse JSON output:|list devices:)|`xcrun simctl list devices --json` did not respond within )/; +// The hub passes numeric display IDs that Xcode 26 rejects; iOS 27 needs them for multiple displays. +// expo-device-hub 0.22.1 returns no Android devices when avdmanager fails, before it runs adb. +const AVDMANAGER_LIST_FAILED = "[android-utils] Failed to run `avdmanager list avd`"; +const GETPROP_LINE = /^\[([^\]]+)\]: \[(.*)\]$/; +// An AVD name as `emulator -list-avds` prints it; anything else is never joined into a path. +const AVD_NAME = /^[A-Za-z0-9._-]+$/; +const runsOnXcode26Runtime = (version: string) => Number(/^iOS (\d+)/.exec(version)?.[1]) <= 26; +// simctl needs a real file (it writes "-" or /dev/stdout as a file of that +// name), so the PNG passes through one in Threadlines' own state folder ($2), +// removed on exit, never the user's temp folder. +const DEFAULT_DISPLAY_SCREENSHOT = `mkdir -p "$2" && f=$(mktemp "$2/screenshot.XXXXXX") || exit 1 +trap 'rm -f "$f"' 0 +xcrun simctl io "$1" screenshot --type=png "$f" >/dev/null && base64 -i "$f"`; + +const HubDevice = Schema.Struct({ + id: Schema.String, + name: Schema.String, + version: Schema.String, + platform: Schema.Literals(["ios", "android"]), + booted: Schema.Boolean, + physical: Schema.Boolean, +}); +const HubDeviceList = Schema.Struct({ + simulators: Schema.Array(HubDevice), + emulators: Schema.Array(HubDevice), + errors: Schema.optional( + Schema.Array(Schema.Struct({ message: Schema.String, error: Schema.optional(Schema.String) })), + ), +}); +/** serve-sim's grid inventory, which, unlike `/api/devices`, includes simulators never booted. */ +const IosGridDeviceList = Schema.Struct({ + devices: Schema.Array( + Schema.Struct({ + device: Schema.String, + name: Schema.String, + runtime: Schema.String, + state: Schema.String, + }), + ), +}); +const HubActionResult = Schema.Struct({ + ok: Schema.Boolean, + id: Schema.optional(Schema.String), + serial: Schema.optional(Schema.String), + error: Schema.optional(Schema.String), + errors: Schema.optional( + Schema.Array(Schema.Struct({ message: Schema.String, error: Schema.optional(Schema.String) })), + ), +}); + +const SimctlDevices = Schema.Struct({ + devices: Schema.Record( + Schema.String, + Schema.Array( + Schema.Struct({ udid: Schema.String, deviceTypeIdentifier: Schema.optional(Schema.String) }), + ), + ), +}); +const decodeSimctlDevices = Schema.decodeUnknownOption(Schema.fromJsonString(SimctlDevices)); + +/** + * Each simulator's device type (`com.apple.CoreSimulator.SimDeviceType.iPhone-Duo`) + * by upper-case UDID, from `xcrun simctl list devices -j`. Empty when the + * output is not that listing. + */ +export const parseSimctlDeviceTypes = (stdout: string): ReadonlyMap => { + const parsed = decodeSimctlDevices(stdout); + const types = new Map(); + if (Option.isNone(parsed)) return types; + for (const devices of Object.values(parsed.value.devices)) { + for (const device of devices) { + const type = device.deviceTypeIdentifier?.trim(); + if (type) types.set(device.udid.toUpperCase(), type); + } + } + return types; +}; + +/** One `key = value` (or `key=value`) from an AVD's `.ini` or `config.ini`. */ +export const readIniValue = (text: string, key: string): string | undefined => { + for (const line of text.split(/\r?\n/)) { + const separator = line.indexOf("="); + if (separator === -1 || line.slice(0, separator).trim() !== key) continue; + const value = line.slice(separator + 1).trim(); + return value === "" ? undefined : value; + } + return undefined; +}; + +/** A failure's class and first line, never a stack: this text reaches the browser. */ +const briefCause = (cause: unknown): string | undefined => { + if (typeof cause !== "object" || cause === null) return undefined; + const tag = "_tag" in cause && typeof cause._tag === "string" ? cause._tag : undefined; + const message = + "message" in cause && typeof cause.message === "string" + ? cause.message.split(/\r?\n/)[0] + : undefined; + const text = [tag, message].filter(Boolean).join(": "); + return text ? boundDiagnostics(text) : undefined; +}; + +// Hub diagnostics can carry a whole stderr dump. +const boundDiagnostics = (text: string) => + text.length > 2_000 ? `${text.slice(0, 1_999)}…` : text; + +// The hub's `error` is a util.inspect of Node's exec error; the stack and the +// inspected fields follow the first frame, so keep the lines before it. +const errorHead = (error: string) => { + const lines = error.split(/\r?\n/); + const firstFrame = lines.findIndex((line) => /^\s+at /.test(line)); + return (firstFrame === -1 ? lines : lines.slice(0, firstFrame)) + .map((line) => line.trim()) + .filter(Boolean) + .join(" "); +}; + +const vendorPrefix = (platform: DevicePlatform) => + platform === "ios" ? "/vendor/serve-sim" : "/vendor/serve-emu"; + +/** + * What a device is never changes: a simulator's device type by UDID and a + * phone's model by serial. An emulator's profile is read again each listing + * (one small file), since a serial is reused across AVDs. + */ +export interface DeviceModelCache { + readonly simulatorTypes: Map; + readonly phoneModels: Map; +} + +export const makeDeviceModelCache = (): DeviceModelCache => ({ + simulatorTypes: new Map(), + phoneModels: new Map(), +}); + +export interface DeviceHubOperationsOptions { + readonly origin: string; + /** The hub's session token (`--require-token`), sent as a bearer on every request. */ + readonly token: string; + readonly http: HttpClient.HttpClient; + readonly run: DeviceHostReady["run"]; + /** Android gates listing AVDs, which the hub does not report until they boot. */ + readonly platformAvailable: (platform: DevicePlatform) => Effect.Effect; + /** A folder in the server's state dir for files a command must write (screenshots). */ + readonly scratchDir: string; + /** Kept by the host across hub starts, so a refresh does not ask simctl or adb again. */ + readonly modelCache: DeviceModelCache; + /** + * Where the AVDs live (`ANDROID_AVD_HOME`, else `~/.android/avd`), and a + * reader for their `.ini` files: an emulator's hardware profile becomes its + * `model`. Without them Android devices list without one. + */ + readonly androidAvds?: { + readonly home: string; + readonly readTextFile: (path: string) => Effect.Effect; + }; +} + +/** + * How a host makes device operations for each hub start: made once per host, + * it keeps the model cache for the host's whole life, so asking again for an + * already running hub (every picker refresh) does not run simctl or adb again. + */ +export const makeHostDeviceOperations = ( + options: Omit, +) => { + const modelCache = makeDeviceModelCache(); + return (hub: { readonly origin: string; readonly token: string }) => + makeDeviceHubOperations({ ...options, ...hub, modelCache }); +}; + +/** Device operations for one running hub. Hosts make them through `makeHostDeviceOperations`. */ +export const makeDeviceHubOperations = (options: DeviceHubOperationsOptions): DeviceOperations => { + const { origin, run } = options; + const { simulatorTypes, phoneModels } = options.modelCache; + const httpClient = options.http.pipe( + HttpClient.mapRequest(HttpClientRequest.bearerToken(options.token)), + HttpClient.withScope, + ); + + const fetchJson = Effect.fn("DeviceHubOperations.fetchJson")(function* ( + request: HttpClientRequest.HttpClientRequest, + schema: Schema.Codec, + ) { + return yield* httpClient + .execute(request) + .pipe( + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap(HttpClientResponse.schemaBodyJson(schema)), + Effect.scoped, + ); + }); + + const hubJson = ( + request: HttpClientRequest.HttpClientRequest, + schema: Schema.Codec, + operation: string, + timeout: Duration.Input = Duration.seconds(15), + ) => + fetchJson(request, schema).pipe( + Effect.timeout(timeout), + Effect.mapError( + (cause) => + new DeviceOperationError({ + operation, + reason: "request_failed", + detail: briefCause(cause), + }), + ), + ); + + const postJson = (path: string, body: Record, operation: string) => + HttpClientRequest.post(`${origin}${path}`).pipe( + HttpClientRequest.bodyJson(body), + Effect.mapError(() => new DeviceOperationError({ operation, reason: "invalid_payload" })), + ); + + // Booting alone does not attach a serve-sim helper; the grid start does both + // and is idempotent for a booted simulator. + const attachIosStream = (udid: string, operation: string) => + postJson(`${vendorPrefix("ios")}/grid/api/start`, { udid }, operation).pipe( + Effect.flatMap((request) => hubJson(request, HubActionResult, "attach stream", BOOT_TIMEOUT)), + ); + + // Mirrors the hub's own adb discovery for when it skips it. + const listAdbDevices = Effect.gen(function* () { + const listed = yield* run("adb", ["devices", "-l"]); + if (listed.code !== 0) return undefined; + const serials = listed.stdout + .split(/\r?\n/) + .map((line) => line.trim().split(/\s+/)) + .flatMap(([serial, state]) => (serial && state === "device" ? [serial] : [])); + return yield* Effect.forEach(serials, (serial) => + Effect.gen(function* () { + const getprop = yield* run("adb", ["-s", serial, "shell", "getprop"]); + const props = new Map( + getprop.stdout.split(/\r?\n/).flatMap((line) => { + const match = GETPROP_LINE.exec(line.trim()); + return match ? [[match[1]!, match[2]!] as const] : []; + }), + ); + const emulator = props.get("ro.kernel.qemu") === "1" || props.get("ro.boot.qemu") === "1"; + const avdName = emulator + ? (yield* run("adb", ["-s", serial, "emu", "avd", "name"])).stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .find((line) => line && line !== "OK" && !line.startsWith("KO")) + : undefined; + const model = props.get("ro.product.model"); + const manufacturer = props.get("ro.product.manufacturer"); + const release = props.get("ro.build.version.release"); + return { + id: serial, + name: emulator + ? (avdName ?? serial) + : model + ? [manufacturer, model].filter(Boolean).join(" ") + : serial, + platform: "android", + version: release + ? `Android ${/^\d+$/.test(release) ? `${release}.0` : release}` + : "Android", + booted: true, + physical: !emulator, + } satisfies HostDevice; + }), + ); + }); + + /** Asks simctl once for every simulator whose type is not known yet. */ + const simulatorModels = Effect.fn("DeviceHubOperations.simulatorModels")(function* ( + udids: ReadonlyArray, + ) { + if (udids.some((udid) => !simulatorTypes.has(udid.toUpperCase()))) { + const listed = yield* run("xcrun", ["simctl", "list", "devices", "-j"], { + timeoutMs: 30_000, + maxOutputBytes: 16 * 1024 * 1024, + }); + if (listed.code === 0) { + const types = parseSimctlDeviceTypes(listed.stdout); + for (const [udid, type] of types) simulatorTypes.set(udid, type); + // simctl answered without these: they are not simulators it knows, so don't ask again. + for (const udid of udids) { + if (!simulatorTypes.has(udid.toUpperCase())) simulatorTypes.set(udid.toUpperCase(), null); + } + } + } + }); + + /** An emulator's hardware profile (`hw.device.name`), through its AVD's `.ini`. */ + const emulatorModel = (avdName: string) => { + const avds = options.androidAvds; + if (!avds || !AVD_NAME.test(avdName)) return Effect.succeed(undefined); + return Effect.gen(function* () { + const pointer = yield* avds.readTextFile(`${avds.home}/${avdName}.ini`); + const folder = (pointer && readIniValue(pointer, "path")) ?? `${avds.home}/${avdName}.avd`; + const config = yield* avds.readTextFile(`${folder}/config.ini`); + return config ? readIniValue(config, "hw.device.name") : undefined; + }); + }; + + const phoneModel = Effect.fn("DeviceHubOperations.phoneModel")(function* (serial: string) { + const known = phoneModels.get(serial); + if (known !== undefined) return known ?? undefined; + const read = yield* run("adb", ["-s", serial, "shell", "getprop", "ro.product.model"]); + if (read.code !== 0) return undefined; + const model = read.stdout.trim() || null; + phoneModels.set(serial, model); + return model ?? undefined; + }); + + /** Fills in `model` where it can be told; a lookup that fails leaves it out. */ + const withModels = Effect.fn("DeviceHubOperations.withModels")(function* ( + devices: ReadonlyArray, + ) { + const simulators = devices.filter((device) => device.platform === "ios" && !device.physical); + if (simulators.length > 0) yield* simulatorModels(simulators.map((device) => device.id)); + return yield* Effect.forEach( + devices, + (device) => + Effect.gen(function* () { + if (device.model) return device; + const model = + device.platform === "ios" + ? device.physical + ? undefined + : (simulatorTypes.get(device.id.toUpperCase()) ?? undefined) + : device.physical + ? device.booted + ? yield* phoneModel(device.id) + : undefined + : yield* emulatorModel(device.name); + return model ? { ...device, model } : device; + }), + { concurrency: 4 }, + ); + }); + + const list: DeviceOperations["list"] = Effect.gen(function* () { + const listed = yield* fetchJson( + HttpClientRequest.get(`${origin}/api/devices`), + HubDeviceList, + ).pipe( + Effect.mapError( + (cause) => + new DeviceOperationError({ + operation: "list", + reason: "request_failed", + detail: briefCause(cause), + }), + ), + Effect.timeoutOption(Duration.seconds(LIST_TIMEOUT_SECONDS)), + ); + if (Option.isNone(listed)) { + return { + devices: [], + detail: `The device hub did not answer within ${LIST_TIMEOUT_SECONDS} seconds.`, + incompletePlatforms: yield* Effect.filter(["ios", "android"] as const, (platform) => + options.platformAvailable(platform), + ), + }; + } + const hubList = listed.value; + const devices: Array = [...hubList.simulators, ...hubList.emulators]; + const androidAvailable = yield* options.platformAvailable("android"); + const warnings = hubList.errors?.filter( + (error) => + // The host summary already explains a missing Android SDK; the hub's failed probes are noise. + !(!androidAvailable && error.message.startsWith("[android-utils] ")) && + // CoreSimulator can retain deleted devices; their optional usage timestamps are absent. + !( + error.message.startsWith("[apple-utils] Failed to read CoreSimulator plist at ") && + /\/CoreSimulator\/Devices\/[^/]+\/device\.plist:$/.test(error.message) && + /(?:^Error:\s*ENOENT:|\bcode:\s*['"]ENOENT['"])/.test(error.error ?? "") + ), + ); + const details = + warnings?.map((error) => { + const head = errorHead(error.error ?? ""); + return boundDiagnostics(head ? `${error.message} ${head}` : error.message); + }) ?? []; + // The hub's device list leaves out simulators that were never booted (they + // carry no usage timestamps); serve-sim's grid lists them. Ported from T3 + // Code's fix (pingdotgg/t3code#28915). + if (yield* options.platformAvailable("ios")) { + const grid = yield* hubJson( + HttpClientRequest.get(`${origin}${vendorPrefix("ios")}/grid/api`), + IosGridDeviceList, + "list", + ).pipe(Effect.result); + if (grid._tag === "Failure") { + details.push("Could not list all iOS simulators. Recently used ones are listed."); + } else { + for (const device of grid.success.devices) { + const known = devices.some( + (listed) => + listed.platform === "ios" && listed.id.toUpperCase() === device.device.toUpperCase(), + ); + if (known || !/^iOS-\d+(?:-\d+)*$/.test(device.runtime)) continue; + devices.push({ + id: device.device, + name: device.name, + version: device.runtime.replace("iOS-", "iOS ").replaceAll("-", "."), + platform: "ios", + booted: device.state === "Booted", + physical: false, + }); + } + } + } + if (androidAvailable) { + if (hubList.errors?.some((error) => error.message.startsWith(AVDMANAGER_LIST_FAILED))) { + const adbDevices = yield* listAdbDevices; + if (adbDevices === undefined) { + details.push("Could not list Android devices (adb devices -l)."); + } else { + devices.push(...adbDevices); + } + } + const avds = yield* run("emulator", ["-list-avds"]); + // Discovered devices stay listed; the AVDs that could not be listed are a warning. + if (avds.code !== 0) { + details.push( + `Could not list Android virtual devices (emulator -list-avds, exit code ${avds.code}). Discovered devices remain available.`, + ); + } + for (const name of (avds.code === 0 ? avds.stdout : "") + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean)) { + if (!devices.some((device) => device.platform === "android" && device.name === name)) { + devices.push({ + id: name, + name, + platform: "android", + version: "Android", + booted: false, + physical: false, + }); + } + } + } + const modelled = yield* withModels(devices); + const simulatorsUnlisted = + hubList.simulators.length === 0 && + (hubList.errors ?? []).some((error) => SIMULATOR_LIST_FAILED.test(error.message)); + return { + devices: modelled, + detail: details.join("\n") || undefined, + incompletePlatforms: simulatorsUnlisted ? (["ios"] as const) : [], + }; + }).pipe(Effect.withSpan("DeviceHubOperations.list")); + + const boot: DeviceOperations["boot"] = Effect.fn("DeviceHubOperations.boot")(function* (device) { + const result = yield* postJson( + "/api/devices/boot", + { platform: device.platform, id: device.id, name: device.name }, + "boot", + ).pipe(Effect.flatMap((request) => hubJson(request, HubActionResult, "boot", BOOT_TIMEOUT))); + if (!result.ok) { + // Keep the hub's launch command and nested failure available to agents for recovery. + const diagnostics = [ + result.error, + ...(result.errors ?? []).flatMap((error) => [error.message, error.error]), + ] + .filter((text) => text?.trim()) + .join("\n\n"); + return yield* new DeviceBootError({ + deviceId: device.id, + reason: /insufficient.*(?:disk|space)|not enough.*(?:disk|space)|no space left/i.test( + diagnostics, + ) + ? "disk_space" + : /timed? out|timeout/i.test(diagnostics) + ? "timeout" + : "launch_failed", + ...(diagnostics ? { diagnostics: boundDiagnostics(diagnostics) } : {}), + }); + } + if (device.platform === "ios") yield* attachIosStream(device.id, "boot"); + return result.serial ?? result.id ?? device.id; + }); + + const attach: DeviceOperations["attach"] = (device) => + device.platform === "ios" + ? attachIosStream(device.id, "open").pipe(Effect.asVoid) + : Effect.void; + + const shutdown: DeviceOperations["shutdown"] = Effect.fn("DeviceHubOperations.shutdown")( + function* (device) { + const postShutdown = (path: string, body: Record) => + postJson(path, body, "shutdown").pipe( + Effect.flatMap((request) => hubJson(request, HubActionResult, "shutdown")), + Effect.flatMap((result) => + result.ok + ? Effect.void + : Effect.fail( + new DeviceOperationError({ + operation: "shutdown", + reason: "hub_rejected", + ...(result.error ? { detail: boundDiagnostics(errorHead(result.error)) } : {}), + }), + ), + ), + ); + // serve-sim's shutdown closes its in-process capture session before it + // runs `simctl shutdown`; the hub's generic shutdown can leave that + // session cached across a reboot. serve-sim runs simctl bare, though, so + // a simulator that is already off fails there. + yield* device.platform === "ios" + ? postShutdown(`${vendorPrefix("ios")}/grid/api/shutdown`, { udid: device.id }).pipe( + Effect.catch((cause) => + list.pipe( + Effect.flatMap(({ devices }) => + devices.find((candidate) => candidate.id === device.id)?.booted === false + ? Effect.logInfo("iOS simulator was already shut down", { deviceId: device.id }) + : Effect.fail(cause), + ), + ), + ), + ) + : postShutdown("/api/devices/shutdown", { platform: device.platform, id: device.id }); + }, + ); + + const defaultDisplayScreenshot = Effect.fn("DeviceHubOperations.defaultDisplayScreenshot")( + function* (udid: string) { + const capture = yield* run( + "sh", + ["-c", DEFAULT_DISPLAY_SCREENSHOT, "sh", udid, options.scratchDir], + { + timeoutMs: Duration.toMillis(SCREENSHOT_TIMEOUT), + maxOutputBytes: 64 * 1024 * 1024, + }, + ); + if (capture.code !== 0) { + return yield* new DeviceOperationError({ + operation: "screenshot", + reason: "command_failed", + exitCode: capture.code, + detail: boundDiagnostics(capture.stderr.trim()), + }); + } + return new Uint8Array(Buffer.from(capture.stdout, "base64")); + }, + ); + + const screenshot: DeviceOperations["screenshot"] = (device) => + httpClient + .execute( + HttpClientRequest.post( + `${origin}${vendorPrefix(device.platform)}/api/screenshot?device=${encodeURIComponent(device.id)}`, + ), + ) + .pipe( + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap((response) => response.arrayBuffer), + Effect.map((buffer) => new Uint8Array(buffer)), + Effect.scoped, + Effect.timeout(SCREENSHOT_TIMEOUT), + Effect.mapError( + (cause) => + new DeviceOperationError({ + operation: "screenshot", + reason: + HttpClientError.isHttpClientError(cause) && cause.reason._tag === "StatusCodeError" + ? "hub_rejected" + : "request_failed", + }), + ), + Effect.catchIf( + (error) => + device.platform === "ios" && + runsOnXcode26Runtime(device.version) && + error.reason === "hub_rejected", + () => defaultDisplayScreenshot(device.id), + ), + Effect.withSpan("DeviceHubOperations.screenshot"), + ); + + return { list, boot, attach, shutdown, screenshot }; +}; diff --git a/apps/server/src/device/DeviceService.test.ts b/apps/server/src/device/DeviceService.test.ts new file mode 100644 index 000000000..329e1116a --- /dev/null +++ b/apps/server/src/device/DeviceService.test.ts @@ -0,0 +1,259 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { describe, expect, it } from "@effect/vitest"; +import { type DeviceSummary, ThreadId } from "@threadlines/contracts"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as Stream from "effect/Stream"; + +import { ServerConfig } from "../config.ts"; +import { ServerSettingsService } from "../serverSettings.ts"; +import { agentDeviceSession } from "./AgentDeviceTarget.ts"; +import * as DeviceHost from "./DeviceHost.ts"; +import * as DeviceService from "./DeviceService.ts"; + +const threadId = ThreadId.make("thread-devices"); +const unused = Effect.die("not used in this test"); + +/** + * A host whose one simulator boots when asked, recording the agent sessions it + * ends. With `bootGate`, booting signals `started` and waits for the gate. + */ +const fakeHost = ( + released: Array, + bootGate?: { readonly started: Deferred.Deferred; readonly gate: Deferred.Deferred }, + readyGate?: { readonly started: Deferred.Deferred; readonly gate: Deferred.Deferred }, +) => { + const device: { current: DeviceSummary } = { + current: { + id: "SIM-1", + platform: "ios", + name: "iPhone", + version: "iOS 27.2", + booted: false, + physical: false, + }, + }; + const ready: DeviceHost.DeviceHostReady = { + nodePath: process.execPath, + hub: { origin: "http://127.0.0.1:1", token: "token" }, + devices: { + list: Effect.sync(() => ({ + devices: [device.current], + detail: undefined, + incompletePlatforms: [], + })), + boot: (target) => + Effect.gen(function* () { + if (bootGate) { + yield* Deferred.succeed(bootGate.started, undefined); + yield* Deferred.await(bootGate.gate); + } + device.current = { ...device.current, booted: true }; + return target.id; + }), + attach: () => Effect.void, + shutdown: () => unused, + screenshot: () => unused, + }, + run: () => Effect.succeed({ stdout: "", stderr: "", code: 0 }), + helpers: { serveSimAxSettings: null, serveSimCli: null }, + }; + return Layer.mock(DeviceHost.DeviceHost)({ + summary: Effect.succeed({ + platforms: [{ platform: "ios", available: true }], + hubInstalled: true, + agentDeviceInstalled: false, + tools: undefined, + toolInspectionError: undefined, + }), + platformAvailability: (platform) => Effect.succeed({ platform, available: true }), + ensureReady: (onPhase) => + readyGate + ? Effect.gen(function* () { + yield* onPhase("installing", "Installing the device hub…"); + yield* Deferred.succeed(readyGate.started, undefined); + yield* Deferred.await(readyGate.gate); + return ready; + }) + : Effect.succeed(ready), + current: Effect.succeed(ready), + releaseAgentSession: (session) => Effect.sync(() => released.push(session) > 0), + stop: Effect.void, + stopAgent: Effect.void, + }); +}; + +const settingsLayer = Layer.mock(ServerSettingsService)({ + getSettings: Effect.succeed({ enableDevices: true, agentDeviceAccess: false } as never), + streamChanges: Stream.never, +}); + +describe("DeviceService claims", () => { + it.live("records a session before it is open, reloads it after a restart, and ends it", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const base = yield* fs.makeTempDirectoryScoped({ prefix: "tl-device-claims-" }); + const released: Array = []; + const layer = Layer.mergeAll( + fakeHost(released), + settingsLayer, + ServerConfig.layerTest(process.cwd(), base), + ); + const claimsPath = path.join(base, "userdata", "device", "claims.json"); + + // First server: the user opens a simulator that was off. + const session = yield* Effect.scoped( + Effect.gen(function* () { + const service = yield* DeviceService.make; + return yield* service.open({ threadId, deviceId: "SIM-1", platform: "ios" }, "user"); + }), + ).pipe(Effect.provide(layer)); + expect(session).toMatchObject({ + deviceId: "SIM-1", + bootedByThreadlines: true, + openedBy: "user", + }); + expect(yield* fs.readFileString(claimsPath)).toContain("SIM-1"); + + // Next server: the claim is a session again, and releasing the thread ends it. + yield* Effect.scoped( + Effect.gen(function* () { + const service = yield* DeviceService.make; + expect((yield* service.state).sessions).toEqual([session]); + yield* service.releaseThread(threadId); + expect((yield* service.state).sessions).toEqual([]); + }), + ).pipe(Effect.provide(layer)); + expect(released).toEqual([agentDeviceSession(threadId, "SIM-1")]); + expect(yield* fs.readFileString(claimsPath)).not.toContain("SIM-1"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +}); + +describe("DeviceService release during an open", () => { + it.live("a thread put away while its device boots keeps no claim", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const base = yield* fs.makeTempDirectoryScoped({ prefix: "tl-device-release-" }); + const bootGate = { + started: yield* Deferred.make(), + gate: yield* Deferred.make(), + }; + const layer = Layer.mergeAll( + fakeHost([], bootGate), + settingsLayer, + ServerConfig.layerTest(process.cwd(), base), + ); + const claimsPath = path.join(base, "userdata", "device", "claims.json"); + + yield* Effect.scoped( + Effect.gen(function* () { + const service = yield* DeviceService.make; + const opening = yield* service + .open({ threadId, deviceId: "SIM-1", platform: "ios" }, "agent") + .pipe(Effect.forkScoped); + yield* Deferred.await(bootGate.started); + yield* service.releaseThread(threadId); + yield* Deferred.succeed(bootGate.gate, undefined); + const result = yield* Fiber.join(opening).pipe(Effect.flip); + expect(result._tag).toBe("DeviceHostUnavailableError"); + expect((yield* service.state).sessions).toEqual([]); + }), + ).pipe(Effect.provide(layer)); + const claims = yield* fs.readFileString(claimsPath).pipe(Effect.orElseSucceed(() => "")); + expect(claims).not.toContain("SIM-1"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +}); + +describe("DeviceService start", () => { + it.live("keeps installing when the client that asked goes away, and ends ready", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const base = yield* fs.makeTempDirectoryScoped({ prefix: "tl-device-start-" }); + const readyGate = { + started: yield* Deferred.make(), + gate: yield* Deferred.make(), + }; + const layer = Layer.mergeAll( + fakeHost([], undefined, readyGate), + settingsLayer, + ServerConfig.layerTest(process.cwd(), base), + ); + yield* Effect.scoped( + Effect.gen(function* () { + const service = yield* DeviceService.make; + // The Device tab asks, then closes while the hub is still installing. + const asking = yield* service.list({ refresh: true }).pipe(Effect.forkScoped); + yield* Deferred.await(readyGate.started); + yield* Fiber.interrupt(asking); + expect((yield* service.state).hostStatus).toBe("installing"); + yield* Deferred.succeed(readyGate.gate, undefined); + yield* Effect.promise(() => + expect + .poll(() => Effect.runPromise(service.state.pipe(Effect.map((s) => s.hostStatus)))) + .toBe("ready"), + ); + }), + ).pipe(Effect.provide(layer)); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +}); + +describe("DeviceService first listing", () => { + it.live("never shows a ready hub with no devices while the first listing is on its way", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const base = yield* fs.makeTempDirectoryScoped({ prefix: "tl-device-first-" }); + const layer = Layer.mergeAll( + fakeHost([]), + settingsLayer, + ServerConfig.layerTest(process.cwd(), base), + ); + yield* Effect.scoped( + Effect.gen(function* () { + const service = yield* DeviceService.make; + const seen: Array<{ status: string; devices: number }> = []; + yield* service.stream.pipe( + Stream.runForEach((state) => + Effect.sync(() => + seen.push({ status: state.hostStatus, devices: state.devices.length }), + ), + ), + Effect.forkScoped, + ); + yield* service.list({ refresh: true }); + yield* Effect.sleep("50 millis"); + const firstReady = seen.find((state) => state.status === "ready"); + expect(firstReady).toEqual({ status: "ready", devices: 1 }); + }), + ).pipe(Effect.provide(layer)); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +}); + +describe("extractPng", () => { + it("finds one complete PNG and reads its size", () => { + const ihdr = Buffer.alloc(25); + ihdr.writeUInt32BE(13, 0); + ihdr.write("IHDR", 4, "latin1"); + ihdr.writeUInt32BE(390, 8); + ihdr.writeUInt32BE(844, 12); + const png = Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + ihdr, + Buffer.from([0, 0, 0, 0, 0x49, 0x45, 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82]), + ]); + const extracted = DeviceService.extractPng(Buffer.concat([Buffer.from("noise"), png])); + expect(extracted?.width).toBe(390); + expect(extracted?.height).toBe(844); + expect(extracted?.png.length).toBe(png.length); + expect(DeviceService.extractPng(png.subarray(0, 30))).toBeUndefined(); + }); +}); diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts new file mode 100644 index 000000000..ddb7202ac --- /dev/null +++ b/apps/server/src/device/DeviceService.ts @@ -0,0 +1,850 @@ +/** + * Simulators and emulators on this server's machine (docs/design/devices.md). + * Ported from T3 Code's DeviceService with one host per server. + * + * Owns the state clients subscribe to (`DeviceServiceState`), the sessions + * threads hold on devices, and the agent's way in (`agentOpen`). Nothing + * installs or starts until a client or an agent asks, and only after the + * matching setting: the hub after "Use devices", agent-device after "Agents + * can use devices". + * + * Sessions survive a crash. `/device/claims.json` records each one + * before it is reported open and drops it on close; the next server loads + * them back, and DeviceThreadReactor releases those whose thread was put away + * while it was down. Closing never shuts a device down unless asked: a + * simulator the user booted is theirs. + */ +import { + type DeviceActionInput, + type DeviceCloseInput, + type DeviceDetail, + type DeviceDetailInput, + type DeviceError, + DeviceHostUnavailableError, + type DeviceId, + DeviceNotFoundError, + DeviceOperationError, + type DeviceOpenInput, + type DevicePlatform, + DevicePlatformUnavailableError, + type DeviceServiceState, + DeviceSession, + type DeviceShutdownInput, + type DeviceSummary, + type ThreadId, +} from "@threadlines/contracts"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Fiber from "effect/Fiber"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as PubSub from "effect/PubSub"; +import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; +import * as SynchronizedRef from "effect/SynchronizedRef"; + +import { writeFileStringAtomically } from "../atomicWrite.ts"; +import { ServerConfig } from "../config.ts"; +import { ServerSettingsService } from "../serverSettings.ts"; +import { + agentDeviceLauncherKey, + bindAgentDeviceLauncher, + pruneAgentDeviceLaunchers, +} from "./AgentDeviceShim.ts"; +import { + agentDeviceConfigPath, + agentDeviceSession, + writeAgentDeviceConfig, +} from "./AgentDeviceTarget.ts"; +import { readDeviceDetail, runDeviceAction } from "./DeviceActions.ts"; +import * as DeviceHost from "./DeviceHost.ts"; +import { deviceStateDir } from "./DeviceToolchain.ts"; +import * as LocalDeviceHost from "./LocalDeviceHost.ts"; + +export interface DeviceScreenshot { + readonly device: DeviceSummary; + readonly png: Uint8Array; + readonly width: number; + readonly height: number; +} + +/** What `device_open` hands an agent: the device and the launcher bound to it. */ +export interface DeviceAgentOpened { + readonly device: DeviceSummary; + readonly session: DeviceSession; + /** Absolute path of the `agent-device` launcher for this thread and device. */ + readonly command: string; + readonly agentSession: string; +} + +/** A session as claims.json keeps it: with the agent-device session it may hold. */ +const DeviceClaim = Schema.Struct({ + ...DeviceSession.fields, + agentSession: Schema.String, +}); +type DeviceClaim = typeof DeviceClaim.Type; +const ClaimsFile = Schema.Struct({ version: Schema.Literal(1), claims: Schema.Array(DeviceClaim) }); +const decodeClaimsFile = Schema.decodeUnknownEffect(Schema.fromJsonString(ClaimsFile)); +const encodeClaimsFile = Schema.encodeUnknownEffect(Schema.fromJsonString(ClaimsFile)); + +export interface DeviceServiceShape { + readonly state: Effect.Effect; + /** The state now and after every change. */ + readonly stream: Stream.Stream; + /** The devices; with `refresh`, asks the hub again (starting it). */ + readonly list: (input: { + readonly refresh?: boolean | undefined; + }) => Effect.Effect; + readonly updateTool: (tool: "hub" | "agent") => Effect.Effect; + readonly open: ( + input: DeviceOpenInput, + openedBy?: "user" | "agent", + ) => Effect.Effect; + readonly close: (input: DeviceCloseInput) => Effect.Effect; + readonly shutdown: (input: DeviceShutdownInput) => Effect.Effect; + readonly detail: (input: DeviceDetailInput) => Effect.Effect; + readonly action: (input: DeviceActionInput) => Effect.Effect; + readonly screenshot: (deviceId: DeviceId) => Effect.Effect; + /** + * An agent opens a device: agent tools readied, the device opened for the + * thread (`openedBy: "agent"`), and a launcher bound to them. + */ + readonly agentOpen: (input: { + readonly threadId: ThreadId; + readonly deviceId: DeviceId; + readonly platform: DevicePlatform; + }) => Effect.Effect; + /** Every session a thread holds, oldest first. */ + readonly sessionsForThread: (threadId: ThreadId) => Effect.Effect>; + /** The running hub, for the device channel. Never starts it. */ + readonly currentHub: Effect.Effect; + /** A thread was archived, deleted or moved: its sessions end, devices keep running. */ + readonly releaseThread: (threadId: ThreadId) => Effect.Effect; +} + +export class DeviceService extends Context.Service()( + "@threadlines/server/device/DeviceService", +) {} + +const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]; +const PNG_END = Buffer.from([0x49, 0x45, 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82]); + +/** The one complete PNG in `bytes` (from its signature to its IEND chunk), with its size. */ +export const extractPng = ( + bytes: Uint8Array, +): { readonly png: Uint8Array; readonly width: number; readonly height: number } | undefined => { + const buffer = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const start = buffer.indexOf(Buffer.from(PNG_SIGNATURE)); + if (start < 0) return undefined; + const end = buffer.indexOf(PNG_END, start); + if (end < 0 || end - start < 24) return undefined; + const png = new Uint8Array(buffer.subarray(start, end + PNG_END.length)); + const view = new DataView(png.buffer, png.byteOffset, png.byteLength); + if (view.getUint32(12) !== 0x49484452) return undefined; + return { png, width: view.getUint32(16), height: view.getUint32(20) }; +}; + +const sameSession = ( + a: { readonly threadId: ThreadId; readonly deviceId: DeviceId }, + b: { readonly threadId: ThreadId; readonly deviceId: DeviceId }, +) => a.threadId === b.threadId && a.deviceId === b.deviceId; + +const agentTargetArgs = (device: DeviceSummary): ReadonlyArray => + device.platform === "ios" + ? ["--platform", "ios", "--udid", device.id] + : ["--platform", "android", "--serial", device.id]; + +const hostUnavailable = (error: DeviceHost.DeviceHostError) => + new DeviceHostUnavailableError({ reason: error.message }); + +export const make = Effect.gen(function* () { + const host = yield* DeviceHost.DeviceHost; + const settings = yield* ServerSettingsService; + const config = yield* ServerConfig; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const scope = yield* Scope.Scope; + const stateDir = config.stateDir; + const claimsPath = path.join(deviceStateDir(path, stateDir), "claims.json"); + const services = (effect: Effect.Effect) => + effect.pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + ); + + const readDeviceSettings = settings.getSettings.pipe( + Effect.map((value) => ({ + enabled: value.enableDevices, + agentAccessEnabled: value.agentDeviceAccess, + })), + Effect.mapError( + () => new DeviceOperationError({ operation: "settings", reason: "settings_failed" }), + ), + ); + const initialSettings = yield* readDeviceSettings.pipe( + Effect.orElseSucceed(() => ({ enabled: false, agentAccessEnabled: false })), + ); + + // Claims a previous run left. A file that can't be read is kept as it is and treated as none. + const loadedClaims = yield* fs.readFileString(claimsPath).pipe( + Effect.flatMap(decodeClaimsFile), + Effect.map((file) => file.claims), + Effect.catch(() => Effect.succeed([] as ReadonlyArray)), + ); + const claims = new Map( + loadedClaims.map((claim) => [claim.agentSession, claim]), + ); + const claimsLock = yield* Semaphore.make(1); + const persistClaims = claimsLock.withPermit( + Effect.suspend(() => encodeClaimsFile({ version: 1, claims: [...claims.values()] })).pipe( + Effect.flatMap((contents) => writeFileStringAtomically({ filePath: claimsPath, contents })), + services, + Effect.mapError( + () => new DeviceOperationError({ operation: "save claim", reason: "settings_failed" }), + ), + ), + ); + + const toSession = ({ agentSession: _agentSession, ...session }: DeviceClaim): DeviceSession => + session; + + const initialSummary = yield* host.summary; + // Every state is a whole snapshot, so a subscriber that stops reading needs + // only the newest: older ones slide out instead of piling up in memory. + const statePubSub = yield* PubSub.sliding(2); + const stateRef = yield* SynchronizedRef.make({ + enabled: initialSettings.enabled, + agentAccessEnabled: initialSettings.agentAccessEnabled, + hostStatus: initialSettings.enabled ? "idle" : "disabled", + platforms: initialSummary.platforms, + incompletePlatforms: [], + ...(initialSummary.tools ? { tools: initialSummary.tools } : {}), + ...(initialSummary.toolInspectionError + ? { toolInspectionError: initialSummary.toolInspectionError } + : {}), + hubInstalled: initialSummary.hubInstalled, + agentDeviceInstalled: initialSummary.agentDeviceInstalled, + devices: [], + sessions: [...claims.values()].map(toSession), + bootingDevices: [], + revision: 0, + }); + const lifecycleLock = yield* Semaphore.make(1); + const launcherLock = yield* Semaphore.make(1); + + const publish = (update: (state: DeviceServiceState) => DeviceServiceState) => + SynchronizedRef.modifyEffect(stateRef, (state) => { + const next = { ...update(state), revision: state.revision + 1 }; + return PubSub.publish(statePubSub, next).pipe(Effect.as([next, next] as const)); + }).pipe(Effect.uninterruptible); + + const currentState = SynchronizedRef.get(stateRef); + + const publishSummary = host.summary.pipe( + Effect.flatMap((summary) => + publish((state) => { + const { tools: _tools, toolInspectionError: _error, ...rest } = state; + return { + ...rest, + platforms: summary.platforms, + hubInstalled: summary.hubInstalled, + agentDeviceInstalled: summary.agentDeviceInstalled, + ...(summary.tools ? { tools: summary.tools } : {}), + ...(summary.toolInspectionError + ? { toolInspectionError: summary.toolInspectionError } + : {}), + }; + }), + ), + ); + + const setHostStatus = (status: DeviceServiceState["hostStatus"], detail?: string) => + publish((state) => { + const { hostStatusDetail: _detail, ...rest } = state; + return { ...rest, hostStatus: status, ...(detail ? { hostStatusDetail: detail } : {}) }; + }); + + const requireEnabled = Effect.gen(function* () { + if (!(yield* readDeviceSettings).enabled) { + return yield* new DeviceHostUnavailableError({ + reason: "Devices are off. Turn on Use devices in Settings → Devices first.", + }); + } + }); + + /** + * Runs `work` once in the service's own scope and lets any number of + * callers wait on it. A caller that goes away (a Device tab closed during + * the first install) stops waiting; the install and start carry on, so the + * hub never stays half-started with its status stuck on "installing". + */ + const shared = (work: Effect.Effect) => + Effect.gen(function* () { + const slot = yield* SynchronizedRef.make | null>(null); + return SynchronizedRef.modifyEffect(slot, (running) => + running !== null + ? Effect.succeed([running, running] as const) + : work.pipe( + Effect.ensuring(SynchronizedRef.set(slot, null)), + Effect.forkIn(scope), + Effect.map((fiber) => [fiber, fiber] as const), + ), + ).pipe(Effect.flatMap(Fiber.join)); + }); + + const readiness = yield* shared( + Effect.gen(function* () { + yield* requireEnabled; + const ready = yield* host + .ensureReady((phase, detail) => setHostStatus(phase, detail).pipe(Effect.asVoid)) + .pipe( + Effect.tapError((error) => setHostStatus("failed", error.message)), + Effect.mapError(hostUnavailable), + ); + const { hostStatus } = yield* currentState; + if (hostStatus !== "ready") { + // The first listing and "ready" go out together; a failed listing still leaves the hub ready. + yield* refresh(ready, { markReady: true }).pipe(Effect.catch(() => setHostStatus("ready"))); + yield* publishSummary; + } + return ready; + }).pipe(lifecycleLock.withPermit, Effect.withSpan("DeviceService.readiness")), + ); + + const agentReadiness = yield* shared( + Effect.gen(function* () { + const deviceSettings = yield* readDeviceSettings; + if (!deviceSettings.enabled || !deviceSettings.agentAccessEnabled) { + return yield* new DeviceHostUnavailableError({ + reason: + "Agents can't use devices on this computer. Ask the user to turn on Use devices and Agents can use devices in Settings → Devices.", + }); + } + const ready = yield* host + .ensureAgentReady((phase, detail) => setHostStatus(phase, detail).pipe(Effect.asVoid)) + .pipe( + Effect.tapError((error) => setHostStatus("failed", error.message)), + Effect.mapError(hostUnavailable), + ); + yield* setHostStatus("ready"); + yield* publishSummary; + return ready; + }).pipe(lifecycleLock.withPermit, Effect.withSpan("DeviceService.agentReadiness")), + ); + + /** + * Lists the devices and publishes them. With `markReady`, the hub's "ready" + * goes out in the same state as the listing, so a client never sees a ready + * hub with no devices while the first listing is still on its way. + */ + const refresh = Effect.fn("DeviceService.refresh")(function* ( + ready: DeviceHost.DeviceHostReady, + options?: { readonly markReady?: boolean }, + ) { + const { devices, detail, incompletePlatforms } = yield* ready.devices.list; + if (!(yield* readDeviceSettings).enabled) return yield* currentState; + return yield* publish((state) => { + const { hostStatusDetail: _detail, ...rest } = state; + return { + ...rest, + ...(options?.markReady ? { hostStatus: "ready" as const } : {}), + ...(detail ? { hostStatusDetail: detail } : {}), + incompletePlatforms: [...incompletePlatforms], + // A platform that could not be listed keeps its last known devices. + devices: [ + ...state.devices.filter((device) => incompletePlatforms.includes(device.platform)), + ...devices.filter((device) => !incompletePlatforms.includes(device.platform)), + ], + }; + }); + }); + + const list: DeviceServiceShape["list"] = (input) => + Effect.gen(function* () { + if (!(yield* readDeviceSettings).enabled) return yield* currentState; + if (input.refresh !== true) return yield* currentState; + const platforms = (yield* host.summary).platforms; + if (!platforms.some((platform) => platform.available)) { + yield* publishSummary; + return yield* currentState; + } + const ready = yield* readiness; + return yield* refresh(ready); + }).pipe(Effect.withSpan("DeviceService.list")); + + const ensurePlatform = Effect.fn("DeviceService.ensurePlatform")(function* ( + platform: DevicePlatform, + ) { + const availability = yield* host.platformAvailability(platform); + if (!availability.available) { + return yield* new DevicePlatformUnavailableError({ + platform, + reason: availability.reason ?? "This platform's tools are missing.", + }); + } + }); + + const findDevice = (state: DeviceServiceState, deviceId: string) => + state.devices.find((device) => device.id === deviceId); + + /** serve-emu encodes emulator video with ffmpeg's libx264; without it there is nothing to show. */ + const requireAndroidVideo = Effect.fn("DeviceService.requireAndroidVideo")(function* ( + ready: DeviceHost.DeviceHostReady, + ) { + const encoders = yield* ready.run(ready.nodePath, [ + "-e", + `const { spawnSync } = require("node:child_process"); +const result = spawnSync(process.env.SERVE_EMU_FFMPEG?.trim() || "ffmpeg", ["-hide_banner", "-encoders"], { + stdio: "inherit", timeout: 10000, killSignal: "SIGKILL", +}); +process.exitCode = result.error ? 1 : (result.status ?? 1);`, + ]); + if (encoders.code !== 0) { + return yield* new DevicePlatformUnavailableError({ + platform: "android", + reason: + "Android emulator video needs ffmpeg with the libx264 encoder. Install it on this computer's PATH (or set SERVE_EMU_FFMPEG), restart Threadlines, then open the emulator again.", + }); + } + if (!/^\s*V\S*[ \t]+libx264(?:[ \t]|$)/m.test(`${encoders.stdout}\n${encoders.stderr}`)) { + return yield* new DevicePlatformUnavailableError({ + platform: "android", + reason: + "Android emulator video needs the libx264 encoder. Install an ffmpeg build that includes libx264, restart Threadlines, then open the emulator again.", + }); + } + }); + + /** Records the claim, then publishes the session: a claim is on disk before anyone sees it open. */ + /** + * How many times each thread has been released (archived, moved, deleted). + * An open notes the count when it starts and records nothing if it changed, + * so a thread put away while its device was still booting keeps no claim. + */ + const releaseCounts = new Map(); + + const recordSession = (session: DeviceSession, releasesAtStart: number) => + Effect.gen(function* () { + if (!(yield* readDeviceSettings).enabled) { + return yield* new DeviceHostUnavailableError({ + reason: "Devices were turned off while the device was opening.", + }); + } + if ((releaseCounts.get(session.threadId) ?? 0) !== releasesAtStart) { + return yield* new DeviceHostUnavailableError({ + reason: "The thread was put away while the device was opening.", + }); + } + const agentSession = agentDeviceSession(session.threadId, session.deviceId); + claims.set(agentSession, { ...session, agentSession }); + yield* persistClaims; + yield* publish((state) => ({ + ...state, + sessions: [ + ...state.sessions.filter((existing) => !sameSession(existing, session)), + session, + ], + })); + }).pipe(lifecycleLock.withPermit); + + const open: DeviceServiceShape["open"] = (input, openedBy = "user") => + Effect.gen(function* () { + const releasesAtStart = releaseCounts.get(input.threadId) ?? 0; + yield* ensurePlatform(input.platform); + const ready = yield* readiness; + let state = yield* refresh(ready); + let device = findDevice(state, input.deviceId); + if (!device) return yield* new DeviceNotFoundError({ deviceId: input.deviceId }); + if (device.platform === "android" && !device.physical) yield* requireAndroidVideo(ready); + let bootedNow = false; + if (!device.booted && input.boot !== false) { + const booting = { ...device, threadId: input.threadId }; + yield* publish((current) => ({ + ...current, + bootingDevices: [ + ...current.bootingDevices.filter((entry) => entry.id !== booting.id), + booting, + ], + })); + const bootedId = yield* ready.devices.boot(device).pipe( + Effect.ensuring( + publish((current) => ({ + ...current, + bootingDevices: current.bootingDevices.filter((entry) => entry.id !== booting.id), + })), + ), + ); + bootedNow = true; + state = yield* refresh(ready); + // Retained entries predate the boot, so only a complete listing confirms the booted device. + device = state.incompletePlatforms.includes(device.platform) + ? undefined + : (findDevice(state, bootedId) ?? findDevice(state, device.id)); + if (!device) return yield* new DeviceNotFoundError({ deviceId: input.deviceId }); + } else if (device.booted) { + yield* ready.devices.attach(device); + } + const opened = device; + const earlier = (yield* currentState).sessions.find( + (session) => session.deviceId === opened.id && session.bootedByThreadlines, + ); + const session: DeviceSession = { + threadId: input.threadId, + deviceId: opened.id, + platform: opened.platform, + openedAt: DateTime.formatIso(yield* DateTime.now), + bootedByThreadlines: bootedNow || earlier !== undefined, + openedBy, + }; + yield* recordSession(session, releasesAtStart); + return session; + }).pipe(Effect.withSpan("DeviceService.open")); + + /** Removes launchers whose session is gone. Under the launcher lock, so binding never races it. */ + const pruneLaunchers = launcherLock.withPermit( + Effect.gen(function* () { + const { sessions } = yield* currentState; + const live = new Set( + sessions.map((session) => agentDeviceLauncherKey(session.threadId, session.deviceId)), + ); + yield* pruneAgentDeviceLaunchers({ stateDir, live }).pipe(services); + }), + ); + + /** + * Ends sessions: their agent-device sessions first (which quits the app the + * agent opened and frees the device for others), then the claims, then the + * state and launchers. + */ + const endSessions = (requested: ReadonlyArray) => + Effect.gen(function* () { + let ending = requested; + if (ending.length === 0) return; + const released = yield* Effect.forEach( + ending, + (session) => + host.releaseAgentSession(agentDeviceSession(session.threadId, session.deviceId)), + { concurrency: 4 }, + ); + // A session whose close the daemon did not confirm stays claimed and open, + // so closing again (or the next startup's sweep) retries it. + ending = ending.filter((_, index) => released[index]); + if (ending.length === 0) return; + for (const session of ending) { + claims.delete(agentDeviceSession(session.threadId, session.deviceId)); + } + yield* persistClaims.pipe( + Effect.catch((error) => Effect.logWarning("Could not save device claims", { error })), + ); + yield* publish((state) => ({ + ...state, + sessions: state.sessions.filter( + (session) => !ending.some((ended) => sameSession(ended, session)), + ), + })); + yield* pruneLaunchers; + }); + + const shutdownDevice = Effect.fn("DeviceService.shutdownDevice")(function* ( + ready: DeviceHost.DeviceHostReady, + deviceId: DeviceId, + platform: DevicePlatform, + ) { + yield* ready.devices.shutdown({ id: deviceId, platform }); + yield* publish((state) => ({ + ...state, + devices: state.devices.map((device) => + device.id === deviceId ? { ...device, booted: false } : device, + ), + })); + }); + + const refreshQuietly = (ready: DeviceHost.DeviceHostReady) => + refresh(ready).pipe( + Effect.catch((cause) => + Effect.logWarning("Device listing unavailable after shutdown", { cause }), + ), + ); + + const close: DeviceServiceShape["close"] = (input) => + Effect.gen(function* () { + const { sessions } = yield* currentState; + const closing = sessions.filter( + (session) => + session.threadId === input.threadId && + (input.deviceId === undefined || session.deviceId === input.deviceId), + ); + if (!input.shutdown) { + yield* lifecycleLock.withPermit(endSessions(closing)); + return; + } + // Powering off ends every thread's session on those devices, before the power-off, + // while their agent-device sessions can still quit the app. + const ending = sessions.filter((session) => + closing.some((closed) => closed.deviceId === session.deviceId), + ); + yield* lifecycleLock.withPermit(endSessions(ending)); + const ready = yield* readiness; + for (const session of closing) { + yield* shutdownDevice(ready, session.deviceId, session.platform); + } + if (closing.length > 0) yield* refreshQuietly(ready); + }).pipe(Effect.withSpan("DeviceService.close")); + + const shutdown: DeviceServiceShape["shutdown"] = (input) => + Effect.gen(function* () { + const { sessions } = yield* currentState; + yield* lifecycleLock.withPermit( + endSessions(sessions.filter((session) => session.deviceId === input.deviceId)), + ); + const ready = yield* readiness; + yield* shutdownDevice(ready, input.deviceId, input.platform); + yield* refreshQuietly(ready); + }).pipe(Effect.withSpan("DeviceService.shutdown")); + + const resolveDevice = Effect.fn("DeviceService.resolveDevice")(function* (deviceId: DeviceId) { + const ready = yield* readiness; + const device = + findDevice(yield* currentState, deviceId) ?? findDevice(yield* refresh(ready), deviceId); + if (!device) return yield* new DeviceNotFoundError({ deviceId }); + return { ready, device }; + }); + + const detail: DeviceServiceShape["detail"] = (input) => + Effect.gen(function* () { + const { ready, device } = yield* resolveDevice(input.deviceId); + const read = yield* readDeviceDetail(ready, device.platform, device.id); + return { + deviceId: device.id, + settings: read.settings, + foregroundApp: read.foregroundApp, + readAt: DateTime.formatIso(yield* DateTime.now), + }; + }).pipe(Effect.withSpan("DeviceService.detail")); + + const action: DeviceServiceShape["action"] = (input) => + Effect.gen(function* () { + const { ready, device } = yield* resolveDevice(input.deviceId); + yield* runDeviceAction(ready, device.platform, input); + return yield* detail({ deviceId: device.id }); + }).pipe(Effect.withSpan("DeviceService.action")); + + const screenshot: DeviceServiceShape["screenshot"] = (deviceId) => + Effect.gen(function* () { + const { ready, device } = yield* resolveDevice(deviceId); + const bytes = yield* ready.devices.screenshot(device); + const extracted = extractPng(bytes); + if (extracted === undefined) { + return yield* new DeviceOperationError({ + operation: "screenshot", + reason: "hub_rejected", + detail: `The device hub returned ${bytes.length} bytes without a complete PNG.`, + }); + } + return { device, ...extracted }; + }).pipe(Effect.withSpan("DeviceService.screenshot")); + + const agentOpen: DeviceServiceShape["agentOpen"] = (input) => + Effect.gen(function* () { + // Consent and agent tools first, before anything boots or a session is recorded. + const agentReady = yield* agentReadiness; + const session = yield* open(input, "agent"); + const device = findDevice(yield* currentState, session.deviceId); + if (!device) return yield* new DeviceNotFoundError({ deviceId: session.deviceId }); + const configPath = agentDeviceConfigPath(path, stateDir); + yield* writeAgentDeviceConfig(configPath, agentReady.agentDevice).pipe( + services, + Effect.mapError( + () => + new DeviceOperationError({ operation: "configure agent", reason: "settings_failed" }), + ), + ); + const agentSession = agentDeviceSession(session.threadId, session.deviceId); + const targetArgs = [ + ...agentTargetArgs(device), + "--config", + configPath, + "--session", + agentSession, + ]; + const command = yield* launcherLock.withPermit( + Effect.gen(function* () { + const { sessions } = yield* currentState; + // Closed while it opened: no launcher for a session that is gone. + if (!sessions.some((live) => sameSession(live, session))) return Option.none(); + return Option.some( + yield* bindAgentDeviceLauncher({ + stateDir, + threadId: session.threadId, + deviceId: session.deviceId, + node: agentReady.nodePath, + entryPath: agentReady.agentDevice.entryPath, + targetArgs, + }).pipe( + services, + Effect.mapError( + () => + new DeviceOperationError({ + operation: "prepare agent-device", + reason: "settings_failed", + }), + ), + ), + ); + }), + ); + if (Option.isNone(command)) { + return yield* new DeviceHostUnavailableError({ + reason: + "The device was closed while it was opening. Call device_open again only if you still need it.", + }); + } + return { device, session, command: command.value, agentSession }; + }).pipe(Effect.withSpan("DeviceService.agentOpen")); + + const sessionsForThread: DeviceServiceShape["sessionsForThread"] = (threadId) => + currentState.pipe( + Effect.map((state) => state.sessions.filter((session) => session.threadId === threadId)), + ); + + const releaseThread: DeviceServiceShape["releaseThread"] = (threadId) => + Effect.sync(() => releaseCounts.set(threadId, (releaseCounts.get(threadId) ?? 0) + 1)).pipe( + Effect.andThen(sessionsForThread(threadId)), + Effect.flatMap((sessions) => lifecycleLock.withPermit(endSessions(sessions))), + Effect.catchCause((cause) => + Effect.logWarning("Could not release a thread's devices", { threadId, cause }), + ), + ); + + const updateTool: DeviceServiceShape["updateTool"] = (tool) => + Effect.gen(function* () { + yield* requireEnabled; + const deviceSettings = yield* readDeviceSettings; + if (tool === "agent" && !deviceSettings.agentAccessEnabled) { + return yield* new DeviceHostUnavailableError({ + reason: "Agent tools install only once Agents can use devices is on.", + }); + } + yield* host.updateTool(tool).pipe( + Effect.mapError( + (error) => + new DeviceOperationError({ + operation: "update device tool", + reason: "command_failed", + detail: error.message, + }), + ), + ); + return yield* publishSummary; + }).pipe(lifecycleLock.withPermit, Effect.withSpan("DeviceService.updateTool")); + + /** "Use devices" off stops the hub and daemon (devices keep running); agent access off stops the daemon. */ + const applySettings = (next: { + readonly enabled: boolean; + readonly agentAccessEnabled: boolean; + }) => + Effect.gen(function* () { + const previous = yield* currentState; + if ( + previous.enabled === next.enabled && + previous.agentAccessEnabled === next.agentAccessEnabled + ) { + return; + } + yield* publish((state) => { + const { hostStatusDetail: _detail, ...rest } = state; + return { + ...rest, + enabled: next.enabled, + agentAccessEnabled: next.agentAccessEnabled, + hostStatus: next.enabled + ? state.hostStatus === "disabled" + ? "idle" + : state.hostStatus + : "disabled", + ...(next.enabled && state.hostStatusDetail + ? { hostStatusDetail: state.hostStatusDetail } + : {}), + devices: next.enabled ? state.devices : [], + sessions: next.enabled ? state.sessions : [], + bootingDevices: next.enabled ? state.bootingDevices : [], + }; + }); + // Published first, then stopped: the agent tools check this state on every + // call, so they refuse from the moment the switch turns off, not after the + // daemon or hub has finished stopping. + if (!next.enabled) { + yield* lifecycleLock.withPermit( + Effect.gen(function* () { + yield* endSessions(previous.sessions); + yield* host.stop; + }), + ); + } else if (!next.agentAccessEnabled && previous.agentAccessEnabled) { + yield* host.stopAgent; + } + // Agent access turned on: fetch agent-device now, in the background, so the + // first device_open is quick. Nothing starts until an agent asks. + if (next.enabled && next.agentAccessEnabled && !previous.agentAccessEnabled) { + yield* host.updateTool("agent").pipe( + Effect.andThen(publishSummary), + Effect.catch((error) => + Effect.logWarning("Could not install agent tools", { detail: error.message }), + ), + Effect.forkIn(scope), + ); + } + }); + + yield* settings.streamChanges.pipe( + Stream.map((value) => ({ + enabled: value.enableDevices, + agentAccessEnabled: value.agentDeviceAccess, + })), + Stream.runForEach(applySettings), + Effect.catchCause((cause) => Effect.logWarning("Device settings watcher stopped", { cause })), + Effect.forkIn(scope), + ); + + // Launchers for sessions that no longer exist (the claims file is the record). + yield* pruneLaunchers.pipe(Effect.ignore); + + // Stop the hub and daemon with the server. Devices keep running. + yield* Effect.addFinalizer(() => host.stop); + + return DeviceService.of({ + state: currentState, + stream: Stream.unwrap( + Effect.gen(function* () { + // Subscribe before reading so no change is lost. + const subscription = yield* PubSub.subscribe(statePubSub); + const initial = yield* currentState; + return Stream.concat( + Stream.make(initial), + Stream.fromSubscription(subscription).pipe( + Stream.filter((state) => state.revision > initial.revision), + ), + ); + }), + ), + list, + updateTool, + open, + close, + shutdown, + detail, + action, + screenshot, + agentOpen, + sessionsForThread, + currentHub: host.current.pipe(Effect.map((ready) => ready?.hub ?? null)), + releaseThread, + }); +}); + +export const layer = Layer.effect(DeviceService, make).pipe(Layer.provide(LocalDeviceHost.layer)); diff --git a/apps/server/src/device/DeviceThreadReactor.ts b/apps/server/src/device/DeviceThreadReactor.ts new file mode 100644 index 000000000..38accba03 --- /dev/null +++ b/apps/server/src/device/DeviceThreadReactor.ts @@ -0,0 +1,62 @@ +/** + * A thread put away lets go of its devices (docs/design/devices.md): archived, + * deleted, or moved to another computer, its device sessions end (and with + * them their agent-device sessions and launchers). The devices keep running. + * + * At startup, sessions reloaded from claims.json whose thread was put away + * while the server was down end the same way, like the secret requests' + * startup sweep (secrets/AgentSecretRequests.ts). + */ +import type { OrchestrationThread, ThreadId } from "@threadlines/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Stream from "effect/Stream"; + +import { OrchestrationEngineService } from "../orchestration/Services/OrchestrationEngine.ts"; +import { ProjectionSnapshotQuery } from "../orchestration/Services/ProjectionSnapshotQuery.ts"; +import { DeviceService } from "./DeviceService.ts"; + +const isPutAway = (thread: OrchestrationThread | undefined) => + thread === undefined || + thread.deletedAt !== null || + thread.archivedAt !== null || + (thread.move ?? null) !== null; + +export const layer = Layer.effectDiscard( + Effect.gen(function* () { + const devices = yield* DeviceService; + const engine = yield* OrchestrationEngineService; + const snapshots = yield* ProjectionSnapshotQuery; + + yield* Stream.runForEach(engine.streamDomainEvents, (event) => { + switch (event.type) { + case "thread.archived": + case "thread.move-started": + case "thread.deleted": + return devices.releaseThread(event.payload.threadId); + default: + return Effect.void; + } + }).pipe( + Effect.catchCause((cause) => Effect.logWarning("device thread cleanup stopped", { cause })), + Effect.forkScoped, + ); + + yield* Effect.gen(function* () { + const { sessions } = yield* devices.state; + const threads = new Set(sessions.map((session) => session.threadId)); + for (const threadId of threads) { + const read = yield* snapshots + .getThreadDetailById(threadId, { includeArchived: true }) + .pipe(Effect.map(Option.getOrUndefined), Effect.result); + // A thread whose read failed keeps its devices: never released blind. + if (read._tag === "Failure") continue; + if (isPutAway(read.success)) yield* devices.releaseThread(threadId); + } + }).pipe( + Effect.catchCause((cause) => Effect.logWarning("device claim recovery failed", { cause })), + Effect.forkScoped, + ); + }), +); diff --git a/apps/server/src/device/DeviceToolchain.ts b/apps/server/src/device/DeviceToolchain.ts new file mode 100644 index 000000000..02275af34 --- /dev/null +++ b/apps/server/src/device/DeviceToolchain.ts @@ -0,0 +1,398 @@ +// @effect-diagnostics nodeBuiltinImport:off - a synchronous check for community agents' Node.js cleanup +/** + * The two device tools (docs/design/devices.md): expo-device-hub streams and + * takes input, agent-device is the command line agents drive devices with. + * + * Each is installed from npm at the version pinned in deviceToolVersions.json + * into `/tools/device//`: npm installs into a staging + * folder beside it, the entry point is checked, a `.install-complete` marker + * is written, and only then is the folder moved into place. Install scripts + * run: the hub's native addons need them. npm and the tools run on + * Threadlines' managed Node.js, never `npx` and never the server's own + * runtime (which is Electron in the desktop app). Both tools load only N-API + * addons, so a later Node.js release runs an earlier install. + * + * Nothing here installs on its own: the hub only after "Use devices", the + * agent tool only after "Agents can use devices" (DeviceService decides). + */ +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; + +import type { DeviceToolVersions } from "@threadlines/contracts"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; + +import { npmPinnedFlags } from "../provider/acpRegistry/AcpRegistryInstaller.ts"; +import { acpRegistryNodes } from "../provider/acpRegistry/AcpRegistryNodes.ts"; +import { managedNodeEnvironment, runNode } from "../provider/acpRegistry/NpmPackage.ts"; +import { + type ManagedNodePaths, + managedNodeReleaseFor, +} from "../provider/managedRuntime/ManagedNode.ts"; +import DEVICE_TOOL_VERSIONS from "./deviceToolVersions.json" with { type: "json" }; + +const DEVICE_HUB_PACKAGE = "expo-device-hub"; +const AGENT_DEVICE_PACKAGE = "agent-device"; +export const DEVICE_HUB_VERSION = DEVICE_TOOL_VERSIONS[DEVICE_HUB_PACKAGE]; +export const AGENT_DEVICE_VERSION = DEVICE_TOOL_VERSIONS[AGENT_DEVICE_PACKAGE]; + +const INSTALL_TIMEOUT_MS = 10 * 60 * 1000; +const VERSION_PATTERN = /^[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.-]+)?$/; +/** One install at a time in this process: two would race for the same staging parent. */ +const installLock = Semaphore.makeUnsafe(1); + +export type DeviceToolKey = "hub" | "agent"; + +interface ToolSpec { + readonly key: DeviceToolKey; + readonly name: string; + readonly version: string; + /** The entry point inside the package, run as `node `. */ + readonly entry: ReadonlyArray; +} + +const SPECS: Record = { + hub: { + key: "hub", + name: DEVICE_HUB_PACKAGE, + version: DEVICE_HUB_VERSION, + entry: ["dist", "server", "cli.mjs"], + }, + agent: { + key: "agent", + name: AGENT_DEVICE_PACKAGE, + version: AGENT_DEVICE_VERSION, + entry: ["bin", "agent-device.mjs"], + }, +}; + +export interface DeviceToolPaths { + readonly installDir: string; + readonly entryPath: string; + readonly sentinelPath: string; +} + +export class DeviceToolchainInstallError extends Schema.TaggedError()( + "DeviceToolchainInstallError", + { + tool: Schema.String, + step: Schema.String, + exitCode: Schema.optional(Schema.Number), + reason: Schema.optional(Schema.Literals(["no_node", "timed_out"])), + /** npm's last lines. Kept in logs only: they can carry registry credentials. */ + tail: Schema.optional(Schema.String), + cause: Schema.optional(Schema.Defect()), + }, +) { + override get message(): string { + if (this.reason === "no_node") { + return "Device support needs Node.js, and Threadlines has no Node.js it can install on this computer."; + } + if (this.reason === "timed_out") + return `Installing ${this.tool} took too long and was stopped.`; + const suffix = this.exitCode === undefined ? "" : ` (exit code ${this.exitCode})`; + return `Installing ${this.tool} failed while ${this.step}${suffix}.`; + } +} + +/** `/tools/device`. */ +export const deviceToolsRoot = (path: Path.Path, stateDir: string) => + path.join(stateDir, "tools", "device"); + +/** `/device`: hub.json, claims.json, agent-device's state, launchers. */ +export const deviceStateDir = (path: Path.Path, stateDir: string) => path.join(stateDir, "device"); + +/** agent-device's own state folder, isolated from the user's `~/.agent-device`. */ +export const agentDeviceStateDir = (path: Path.Path, stateDir: string): string => + path.join(deviceStateDir(path, stateDir), "agent-device"); + +const toolPaths = (path: Path.Path, stateDir: string, spec: ToolSpec): DeviceToolPaths => { + const installDir = path.join(deviceToolsRoot(path, stateDir), spec.name, spec.version); + return { + installDir, + entryPath: path.join(installDir, "node_modules", spec.name, ...spec.entry), + sentinelPath: path.join(installDir, ".install-complete"), + }; +}; + +export const deviceToolPaths = (path: Path.Path, stateDir: string, tool: DeviceToolKey) => + toolPaths(path, stateDir, SPECS[tool]); + +const isInstalledAt = Effect.fn("DeviceToolchain.isInstalledAt")(function* ( + paths: DeviceToolPaths, + version: string, +) { + const fs = yield* FileSystem.FileSystem; + const [entryExists, sentinel] = yield* Effect.all([ + fs.exists(paths.entryPath).pipe(Effect.orElseSucceed(() => false)), + fs.readFileString(paths.sentinelPath).pipe(Effect.orElseSucceed(() => "")), + ]); + return entryExists && sentinel.trim() === version; +}); + +export const isDeviceToolInstalled = Effect.fn("DeviceToolchain.isInstalled")(function* ( + stateDir: string, + tool: DeviceToolKey, +) { + const path = yield* Path.Path; + return yield* isInstalledAt(deviceToolPaths(path, stateDir, tool), SPECS[tool].version); +}); + +/** Is `tool` installed under `stateDir`, without Effect: for community agents' Node.js cleanup. */ +const installedSync = (stateDir: string, tool: DeviceToolKey) => { + const spec = SPECS[tool]; + const installDir = NodePath.join(stateDir, "tools", "device", spec.name, spec.version); + try { + return ( + NodeFS.readFileSync(NodePath.join(installDir, ".install-complete"), "utf8").trim() === + spec.version + ); + } catch { + return false; + } +}; + +/** Threadlines' managed Node.js under `/tools/node`, shared with community agents. */ +const nodesFor = (path: Path.Path, stateDir: string) => { + const nodes = acpRegistryNodes(path.join(stateDir, "tools", "node")); + if (!registeredKeepers.has(stateDir)) { + registeredKeepers.add(stateDir); + // Community agents' cleanup keeps the release device tools run on while one is installed. + nodes.keepAlso( + Effect.sync(() => { + const release = managedNodeReleaseFor(process.platform, process.arch); + return release && (installedSync(stateDir, "hub") || installedSync(stateDir, "agent")) + ? [release] + : []; + }), + ); + } + return nodes; +}; +const registeredKeepers = new Set(); + +/** + * The managed Node.js, installed if missing and leased until `scope` closes, + * so community agents' cleanup leaves it while a device tool runs on it. + */ +export const leaseDeviceNode = Effect.fn("DeviceToolchain.leaseDeviceNode")(function* ( + stateDir: string, +) { + const path = yield* Path.Path; + const release = managedNodeReleaseFor(process.platform, process.arch); + if (!release) { + return yield* new DeviceToolchainInstallError({ + tool: "Node.js", + step: "finding Node.js", + reason: "no_node", + }); + } + const nodes = nodesFor(path, stateDir); + const node = nodes.nodeFor(release); + const installed = yield* node.installed; + if (!installed) { + yield* nodes + .whileInstalling(node.install()) + .pipe( + Effect.mapError( + (cause) => + new DeviceToolchainInstallError({ tool: "Node.js", step: "installing Node.js", cause }), + ), + ); + } + return yield* node.acquire.pipe( + Effect.mapError( + (cause) => + new DeviceToolchainInstallError({ tool: "Node.js", step: "starting Node.js", cause }), + ), + ); +}); + +const installTool = Effect.fn("DeviceToolchain.installTool")(function* ( + spec: ToolSpec, + paths: DeviceToolPaths, + node: ManagedNodePaths, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const fail = (step: string) => (cause: unknown) => + new DeviceToolchainInstallError({ tool: spec.name, step, cause }); + + if (yield* isInstalledAt(paths, spec.version)) return paths; + + const parentDir = path.dirname(paths.installDir); + yield* fs + .remove(paths.installDir, { recursive: true, force: true }) + .pipe(Effect.mapError(fail("removing an incomplete install"))); + yield* fs + .makeDirectory(parentDir, { recursive: true }) + .pipe(Effect.mapError(fail("preparing the install folder"))); + const stagingDir = yield* fs + .makeTempDirectory({ directory: parentDir, prefix: ".staging-" }) + .pipe(Effect.mapError(fail("preparing the install folder"))); + + return yield* Effect.gen(function* () { + const result = yield* Effect.tryPromise({ + try: (signal) => + runNode({ + node: node.node, + args: [ + node.npmCli, + "install", + "--prefix", + stagingDir, + ...npmPinnedFlags(false), + `${spec.name}@${spec.version}`, + ], + cwd: stagingDir, + env: managedNodeEnvironment(process.env, node.binDir, process.platform), + signal, + timeoutMs: INSTALL_TIMEOUT_MS, + }), + catch: fail("running npm install"), + }); + if (result.timedOut) { + return yield* new DeviceToolchainInstallError({ + tool: spec.name, + step: "running npm install", + reason: "timed_out", + }); + } + if (result.exitCode !== 0) { + return yield* new DeviceToolchainInstallError({ + tool: spec.name, + step: "running npm install", + ...(result.exitCode === null ? {} : { exitCode: result.exitCode }), + tail: result.tail, + }); + } + const stagedEntry = path.join(stagingDir, "node_modules", spec.name, ...spec.entry); + if (!(yield* fs.exists(stagedEntry).pipe(Effect.orElseSucceed(() => false)))) { + return yield* new DeviceToolchainInstallError({ + tool: spec.name, + step: "checking the installed entry point", + }); + } + yield* fs + .writeFileString(path.join(stagingDir, ".install-complete"), `${spec.version}\n`) + .pipe(Effect.mapError(fail("recording the finished install"))); + yield* fs.rename(stagingDir, paths.installDir).pipe( + // Another server on this state folder may have published the same version first. + Effect.catch((cause) => + isInstalledAt(paths, spec.version).pipe( + Effect.flatMap((published) => + published ? Effect.void : Effect.fail(fail("publishing the install")(cause)), + ), + ), + ), + ); + return paths; + }).pipe( + Effect.ensuring(fs.remove(stagingDir, { recursive: true, force: true }).pipe(Effect.ignore)), + ); +}); + +/** + * Installs the pinned version of `tool` if it is missing and returns where it + * is. Holds a lease on the managed Node.js until `scope` closes. + */ +export const ensureDeviceTool = Effect.fn("DeviceToolchain.ensure")(function* ( + stateDir: string, + tool: DeviceToolKey, +) { + const path = yield* Path.Path; + const node = yield* leaseDeviceNode(stateDir); + const spec = SPECS[tool]; + const paths = yield* installLock.withPermit( + installTool(spec, toolPaths(path, stateDir, spec), node), + ); + return { paths, node }; +}); + +/** The entry of the first installed agent-device among `versions`; never installs. */ +export const installedAgentDeviceEntry = Effect.fn("DeviceToolchain.installedAgentDeviceEntry")( + function* (stateDir: string, versions: ReadonlyArray) { + const path = yield* Path.Path; + for (const version of versions) { + if (!VERSION_PATTERN.test(version)) continue; + const paths = toolPaths(path, stateDir, { ...SPECS.agent, version }); + if (yield* isInstalledAt(paths, version)) return paths.entryPath; + } + return null; + }, +); + +/** What is on disk for each tool and what runs, for Settings → Devices. */ +export const deviceToolVersions = Effect.fn("DeviceToolchain.versions")(function* ( + stateDir: string, + running: { readonly hub?: string | undefined; readonly agent?: string | undefined } = {}, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const inspect = Effect.fn("DeviceToolchain.inspect")(function* (spec: ToolSpec) { + const directory = path.join(deviceToolsRoot(path, stateDir), spec.name); + const names = yield* fs.readDirectory(directory).pipe(Effect.orElseSucceed(() => [])); + const versions = yield* Effect.filter(names, (version) => + VERSION_PATTERN.test(version) + ? isInstalledAt(toolPaths(path, stateDir, { ...spec, version }), version) + : Effect.succeed(false), + ); + return { + requiredVersion: spec.version, + installedVersions: versions.sort(), + runningVersion: running[spec.key] ?? null, + }; + }); + return { + hub: yield* inspect(SPECS.hub), + agent: yield* inspect(SPECS.agent), + } satisfies DeviceToolVersions; +}); + +/** + * Deletes installed versions of `tool` other than the pinned one and the most + * recent other one, never one a running process was started from (another + * Threadlines server may still run an older pin). Only once the pinned + * version is installed. Best effort. + */ +export const pruneDeviceTool = Effect.fn("DeviceToolchain.prune")(function* ( + stateDir: string, + tool: DeviceToolKey, + runningCommandLines: Effect.Effect, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const spec = SPECS[tool]; + const parent = path.join(deviceToolsRoot(path, stateDir), spec.name); + const names = yield* fs.readDirectory(parent).pipe(Effect.orElseSucceed(() => [])); + const completed: Array<{ version: string; dir: string; modified: number }> = []; + for (const version of names) { + if (!VERSION_PATTERN.test(version)) continue; + const paths = toolPaths(path, stateDir, { ...spec, version }); + if (!(yield* isInstalledAt(paths, version))) continue; + const info = yield* fs.stat(paths.sentinelPath).pipe(Effect.option); + const modified = + info._tag === "Some" && info.value.mtime._tag === "Some" + ? info.value.mtime.value.getTime() + : 0; + completed.push({ version, dir: paths.installDir, modified }); + } + if (!completed.some((entry) => entry.version === spec.version)) return; + const others = completed.filter((entry) => entry.version !== spec.version); + if (others.length <= 1) return; + // A process scan that fails keeps everything: never delete what may be running. + const commandLines = yield* runningCommandLines; + if (commandLines === null) return; + const previous = others.toSorted( + (a, b) => + b.modified - a.modified || b.version.localeCompare(a.version, "en", { numeric: true }), + )[0]?.version; + for (const entry of others) { + if (entry.version === previous) continue; + if (commandLines.includes(entry.dir + path.sep)) continue; + yield* fs.remove(entry.dir, { recursive: true, force: true }).pipe(Effect.ignore); + } +}); diff --git a/apps/server/src/device/LocalDeviceHost.test.ts b/apps/server/src/device/LocalDeviceHost.test.ts new file mode 100644 index 000000000..914a336b3 --- /dev/null +++ b/apps/server/src/device/LocalDeviceHost.test.ts @@ -0,0 +1,96 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { describe, expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; + +import { + findAndroidSdk, + hubArguments, + parseHubStartupLine, + platformReason, +} from "./LocalDeviceHost.ts"; + +describe("parseHubStartupLine", () => { + // expo-device-hub 0.22.1 with --require-token prints exactly this (cli.mjs `startupMessage`). + it("reads the port and token from the pinned hub's Local line", () => { + expect( + parseHubStartupLine( + " Local: http://localhost:47811/?token=e0t5_0X1M1GVlfqrVNtbToKqFRKH6Rr0heuytDHo_HY", + ), + ).toEqual({ port: 47811, token: "e0t5_0X1M1GVlfqrVNtbToKqFRKH6Rr0heuytDHo_HY" }); + }); + + it("ignores every other line, including a Local line without a token", () => { + expect(parseHubStartupLine("Expo Device Hub ready")).toBeNull(); + expect(parseHubStartupLine(" Local: http://localhost:47811")).toBeNull(); + expect( + parseHubStartupLine(" Network: pass --host 0.0.0.0 to expose on your local network"), + ).toBeNull(); + }); +}); + +describe("hubArguments", () => { + it("binds loopback and requires the token", () => { + const args = hubArguments("/tools/cli.mjs", 4000, "darwin"); + expect(args.slice(0, 5)).toEqual(["/tools/cli.mjs", "--port", "4000", "--host", "127.0.0.1"]); + expect(args).toContain("--require-token"); + expect(args).toContain("--hide-sidebar"); + expect(args).toContain("--hide-boot-device"); + expect(hubArguments("/tools/cli.mjs", 4000, "linux")).toContain("android"); + }); +}); + +describe("platformReason", () => { + const sdk = { + root: "/sdk", + adb: true, + emulator: true, + avdmanager: true, + legacyAvdmanager: false, + javaHome: "/jdk", + }; + + it("names what is missing in words the user can act on", () => { + expect(platformReason("ios", "linux", sdk, false)).toBe( + "iOS Simulators need macOS with Xcode.", + ); + expect(platformReason("ios", "darwin", sdk, false)).toBe( + "Xcode command line tools were not found.", + ); + expect(platformReason("android", "darwin", { ...sdk, avdmanager: false }, true)).toContain( + "Command-line Tools (latest) are missing from /sdk", + ); + expect(platformReason("android", "darwin", { ...sdk, root: null }, true)).toContain( + "Android SDK was not found", + ); + }); + + it("is null when the platform can run", () => { + expect(platformReason("ios", "darwin", sdk, true)).toBeNull(); + expect(platformReason("android", "darwin", sdk, true)).toBeNull(); + }); +}); + +describe("findAndroidSdk", () => { + it.live("finds the Java the SDK tools need through JAVA_HOME", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const base = yield* fs.makeTempDirectoryScoped({ prefix: "tl-android-sdk-" }); + const sdkRoot = path.join(base, "sdk"); + const javaHome = path.join(base, "jdk"); + yield* fs.makeDirectory(path.join(sdkRoot, "platform-tools"), { recursive: true }); + yield* fs.writeFileString(path.join(sdkRoot, "platform-tools", "adb"), ""); + yield* fs.makeDirectory(path.join(javaHome, "bin"), { recursive: true }); + yield* fs.writeFileString(path.join(javaHome, "bin", "java"), ""); + + const sdk = yield* findAndroidSdk( + { ANDROID_HOME: sdkRoot, JAVA_HOME: javaHome, HOME: base, PATH: "" }, + "darwin", + ); + expect(sdk.root).toBe(sdkRoot); + expect(sdk.javaHome).toBe(javaHome); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +}); diff --git a/apps/server/src/device/LocalDeviceHost.ts b/apps/server/src/device/LocalDeviceHost.ts new file mode 100644 index 000000000..42ed656f5 --- /dev/null +++ b/apps/server/src/device/LocalDeviceHost.ts @@ -0,0 +1,986 @@ +/** + * This machine as a device host (docs/design/devices.md). Ported from T3 + * Code's LocalDeviceHost. + * + * The hub runs as a supervised child on a loopback port, with + * `--require-token`: it prints a session token once in its startup line, the + * server keeps it in memory and sends it as a bearer on every request, and + * every hub route but its health checks refuses requests without it. A second + * Threadlines server, an older release, or anything else on loopback cannot + * use this hub. + * + * `/device/hub.json` records `{pid, port, entryPath}` so the next + * server stops a hub this one left behind, checked by pid and command line, + * never by name. agent-device's daemon is started by its own CLI (the pid we + * spawn is the CLI's, not the daemon's) and stopped through the CLI's own stop + * command with our isolated state folder. + */ +import * as OS from "node:os"; + +import type { DevicePlatform, DevicePlatformAvailability } from "@threadlines/contracts"; +import { isCommandAvailable } from "@threadlines/shared/shell"; +import * as NetService from "@threadlines/shared/Net"; +import * as Clock from "effect/Clock"; +import * as Deferred from "effect/Deferred"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; +import { HttpClient } from "effect/http"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; + +import { writeFileStringAtomically } from "../atomicWrite.ts"; +import { ServerConfig } from "../config.ts"; +import { ProcessRunner } from "../processRunner.ts"; +import { releaseAgentDeviceSession } from "./AgentDeviceTarget.ts"; +import { setOwnDeviceToolPort } from "../preview/deviceToolPorts.ts"; +import * as DeviceHost from "./DeviceHost.ts"; +import { makeHostDeviceOperations } from "./DeviceHubOperations.ts"; +import { + AGENT_DEVICE_VERSION, + agentDeviceStateDir, + DEVICE_HUB_VERSION, + type DeviceToolchainInstallError, + type DeviceToolPaths, + deviceStateDir, + deviceToolVersions, + ensureDeviceTool, + installedAgentDeviceEntry, + isDeviceToolInstalled, + leaseDeviceNode, + pruneDeviceTool, +} from "./DeviceToolchain.ts"; + +const HUB_READY_TIMEOUT_MS = 30_000; +const DAEMON_READY_TIMEOUT_MS = 30_000; +const DAEMON_POLL_MS = 100; +const HUB_RESTART_STABLE_UPTIME_MS = 60_000; +const HUB_RESTART_MAX_DELAY_MS = 30_000; + +const HubStateFile = Schema.Struct({ + pid: Schema.Int, + port: Schema.Int, + entryPath: Schema.String, +}); +const decodeHubStateFile = Schema.decodeUnknownEffect(Schema.fromJsonString(HubStateFile)); +const encodeHubStateFile = Schema.encodeUnknownEffect(Schema.fromJsonString(HubStateFile)); + +const AgentDeviceDaemonFile = Schema.Struct({ + httpPort: Schema.Int, + token: Schema.String, + pid: Schema.optional(Schema.Int), + version: Schema.optional(Schema.String), +}); +const decodeDaemonFile = Schema.decodeUnknownEffect(Schema.fromJsonString(AgentDeviceDaemonFile)); + +/** + * The hub's startup line with `--require-token` (expo-device-hub 0.22.1, + * `startupMessage` in its cli.mjs): ` Local: http://localhost:/?token=`. + * The token is 32 random bytes in base64url. + */ +export const parseHubStartupLine = ( + line: string, +): { readonly port: number; readonly token: string } | null => { + const match = /^\s*Local:\s+http:\/\/localhost:(\d+)\/\?token=([A-Za-z0-9_-]{16,})\s*$/.exec( + line, + ); + if (!match) return null; + const port = Number(match[1]); + return Number.isInteger(port) && port > 0 && port < 65536 ? { port, token: match[2]! } : null; +}; + +/** A hub output line with any token in it hidden, for the log. */ +const redactHubLine = (line: string) => line.replace(/token=[A-Za-z0-9_-]+/g, "token="); + +interface HubProcess { + readonly nodePath: string; + readonly child: ChildProcessSpawner.ChildProcessHandle; + /** Holds the child and the lease on the managed Node.js it runs on. */ + readonly scope: Scope.Closeable; + readonly origin: string; + readonly token: string; + readonly startedAtMillis: number; +} + +interface RunningHost { + readonly hub: HubProcess; + readonly agentDevice: DeviceHost.AgentDeviceEndpoint | null; + readonly helpers: DeviceHost.DeviceHostReady["helpers"]; +} + +interface AndroidSdk { + readonly root: string | null; + readonly adb: boolean; + readonly emulator: boolean; + readonly avdmanager: boolean; + readonly legacyAvdmanager: boolean; + /** + * A Java the SDK's command-line tools can run on (avdmanager is Java), or + * null. Without one, emulators that are already running can be watched, but + * the hub cannot list stopped ones or start them. + */ + readonly javaHome: string | null; +} + +/** What starting an Android emulator needs when no Java is found. */ +export const ANDROID_JAVA_NOTE = + "Running emulators work. Starting one needs Java, which Android Studio includes. Install Android Studio or a JDK, then restart Threadlines."; + +/** Where the Android SDK is, the way Android Studio and the hub look for it. */ +export const findAndroidSdk = Effect.fn("LocalDeviceHost.findAndroidSdk")(function* ( + environment: NodeJS.ProcessEnv, + platform: NodeJS.Platform, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const windows = platform === "win32"; + const home = environment.HOME ?? environment.USERPROFILE ?? ""; + const explicit = [environment.ANDROID_HOME?.trim(), environment.ANDROID_SDK_ROOT?.trim()].filter( + (root): root is string => !!root, + ); + const exists = (file: string) => fs.exists(file).pipe(Effect.orElseSucceed(() => false)); + const adbBinary = windows ? "adb.exe" : "adb"; + const avdmanagerBinary = windows ? "avdmanager.bat" : "avdmanager"; + // The Java avdmanager runs on: JAVA_HOME, Android Studio's own, or an installed JDK. + // The macOS /usr/bin/java is a stub that asks to install Java, so it never counts. + const javaBinary = windows ? "java.exe" : "java"; + const hasJava = (javaHome: string) => exists(path.join(javaHome, "bin", javaBinary)); + const listDirectory = (directory: string) => + fs.readDirectory(directory).pipe(Effect.orElseSucceed((): Array => [])); + const findJavaHome = Effect.gen(function* () { + const programFiles = environment.ProgramFiles ?? "C:\\Program Files"; + const fixed = [ + environment.JAVA_HOME?.trim(), + "/Applications/Android Studio.app/Contents/jbr/Contents/Home", + path.join(home, "Applications", "Android Studio.app", "Contents", "jbr", "Contents", "Home"), + path.join(programFiles, "Android", "Android Studio", "jbr"), + "/opt/android-studio/jbr", + path.join(home, "android-studio", "jbr"), + ].filter((candidate): candidate is string => !!candidate); + for (const candidate of fixed) { + if (yield* hasJava(candidate)) return candidate; + } + for (const parent of ["/Library/Java/JavaVirtualMachines", "/usr/lib/jvm"]) { + for (const entry of (yield* listDirectory(parent)).toSorted().toReversed()) { + const candidate = + parent === "/usr/lib/jvm" + ? path.join(parent, entry) + : path.join(parent, entry, "Contents", "Home"); + if (yield* hasJava(candidate)) return candidate; + } + } + return null; + }); + const inspect = Effect.fnUntraced(function* (root: string) { + const adb = yield* exists(path.join(root, "platform-tools", adbBinary)); + const emulator = yield* exists( + path.join(root, "emulator", windows ? "emulator.exe" : "emulator"), + ); + const avdmanager = yield* exists( + path.join(root, "cmdline-tools", "latest", "bin", avdmanagerBinary), + ); + const legacyAvdmanager = + !avdmanager && (yield* exists(path.join(root, "tools", "bin", avdmanagerBinary))); + const javaHome = adb || emulator ? yield* findJavaHome : null; + return { root, adb, emulator, avdmanager, legacyAvdmanager, javaHome } satisfies AndroidSdk; + }); + const candidates = [ + ...explicit, + path.join(home, "Library", "Android", "sdk"), + path.join(home, "Android", "Sdk"), + path.join(environment.LOCALAPPDATA ?? path.join(home, "AppData", "Local"), "Android", "Sdk"), + ]; + for (const root of candidates) { + const sdk = yield* inspect(root); + if (sdk.adb || sdk.emulator) return sdk; + } + // PATH entries can be slow to stat (network shares), so only after the standard places miss. + const searchPath = environment.PATH ?? environment.Path ?? ""; + for (const directory of searchPath.split(windows ? ";" : ":")) { + if (!directory) continue; + const adb = path.join(directory, adbBinary); + if (!(yield* exists(adb))) continue; + const resolved = yield* fs.realPath(adb).pipe(Effect.orElseSucceed(() => adb)); + const sdk = yield* inspect(path.dirname(path.dirname(resolved))); + if (sdk.adb || sdk.emulator) return sdk; + } + // No real SDK anywhere: report what the configured root is missing. + if (explicit[0]) return yield* inspect(explicit[0]); + return { + root: null, + adb: false, + emulator: false, + avdmanager: false, + legacyAvdmanager: false, + javaHome: null, + } satisfies AndroidSdk; +}); + +/** Why a platform can't run here, in words the user can act on, or null when it can. */ +export const platformReason = ( + platform: DevicePlatform, + hostPlatform: NodeJS.Platform, + sdk: AndroidSdk, + hasXcrun: boolean, +): string | null => { + if (platform === "ios") { + if (hostPlatform !== "darwin") return "iOS Simulators need macOS with Xcode."; + if (!hasXcrun) return "Xcode command line tools were not found."; + return null; + } + if (!sdk.root) + return "Android SDK was not found. Install it with Android Studio or set ANDROID_HOME to your SDK folder."; + if (!sdk.adb) + return `Android SDK Platform-Tools are missing from ${sdk.root}. Install them in Android Studio's SDK Manager.`; + if (!sdk.emulator) + return `Android Emulator is missing from ${sdk.root}. Install it in Android Studio's SDK Manager.`; + if (!sdk.avdmanager) { + if (sdk.legacyAvdmanager) + return `The Android SDK command-line tools in ${sdk.root} are an older, unsupported version. Install Android SDK Command-line Tools (latest) in Android Studio's SDK Manager under SDK Tools.`; + return `Android SDK Command-line Tools (latest) are missing from ${sdk.root}. Install them in Android Studio's SDK Manager.`; + } + return null; +}; + +/** The environment device tools run in: the SDK's tools first on PATH. */ +const deviceHostEnvironment = ( + environment: NodeJS.ProcessEnv, + sdk: Pick, + hostPlatform: NodeJS.Platform, + path: Path.Path, +): NodeJS.ProcessEnv => { + const { NODE_OPTIONS: _nodeOptions, ...rest } = environment; + const sdkRoot = sdk.root; + return sdkRoot + ? { + ...rest, + ANDROID_HOME: sdkRoot, + // Android Studio users often have its Java and no JAVA_HOME; avdmanager needs one. + ...(sdk.javaHome ? { JAVA_HOME: sdk.javaHome } : {}), + PATH: [ + path.join(sdkRoot, "platform-tools"), + path.join(sdkRoot, "emulator"), + environment.PATH ?? environment.Path ?? "", + ].join(hostPlatform === "win32" ? ";" : ":"), + } + : rest; +}; + +/** Only a Mac can list Apple simulators; elsewhere the hub's attempt surfaces as an error. */ +const hubPlatformArguments = (hostPlatform: NodeJS.Platform) => + hostPlatform === "darwin" ? [] : ["--platform", "android"]; + +/** The hub's arguments: loopback only, token required, no device list or boot controls of its own. */ +export const hubArguments = (entryPath: string, port: number, hostPlatform: NodeJS.Platform) => [ + entryPath, + "--port", + String(port), + "--host", + "127.0.0.1", + "--require-token", + "--hide-sidebar", + "--hide-boot-device", + ...hubPlatformArguments(hostPlatform), +]; + +const isProcessAlive = (pid: number) => + Effect.sync(() => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } + }); + +/** Where the emulator keeps AVDs: `ANDROID_AVD_HOME`, `ANDROID_USER_HOME/avd`, or `~/.android/avd`. */ +const androidAvdHome = (environment: NodeJS.ProcessEnv, path: Path.Path) => + environment.ANDROID_AVD_HOME?.trim() || + path.join(environment.ANDROID_USER_HOME?.trim() || path.join(OS.homedir(), ".android"), "avd"); + +const hostError = + (step: string, detail?: string) => + (cause: unknown): DeviceHost.DeviceHostError => + new DeviceHost.DeviceHostError({ step, ...(detail ? { detail } : {}), cause }); + +const installFailure = (step: string) => (cause: DeviceToolchainInstallError) => + new DeviceHost.DeviceHostError({ step, detail: cause.message, cause }); + +export const make = Effect.fn("LocalDeviceHost.make")(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const config = yield* ServerConfig; + const path = yield* Path.Path; + const fs = yield* FileSystem.FileSystem; + const net = yield* NetService.NetService; + const runner = yield* ProcessRunner; + const httpClient = yield* HttpClient.HttpClient; + const environment = process.env; + const hostPlatform = process.platform; + const stateDir = config.stateDir; + const services = Effect.provideService(FileSystem.FileSystem, fs); + const withPath = Effect.provideService(Path.Path, path); + + // Resolved per use: an SDK installed after the server started is found without a restart. + const currentSdk = findAndroidSdk(environment, hostPlatform).pipe(services, withPath); + const currentHostEnvironment = currentSdk.pipe( + Effect.map((sdk) => deviceHostEnvironment(environment, sdk, hostPlatform, path)), + ); + const startLock = yield* Semaphore.make(1); + const runningRef = yield* Ref.make(null); + const restartDelayRef = yield* Ref.make(0); + const hubStatePath = path.join(deviceStateDir(path, stateDir), "hub.json"); + const daemonFilePath = path.join(agentDeviceStateDir(path, stateDir), "daemon.json"); + + const platformAvailability = Effect.fn("LocalDeviceHost.platformAvailability")(function* ( + platform: DevicePlatform, + ): Effect.fn.Return { + const sdk = yield* currentSdk; + const hasXcrun = platform === "ios" && hostPlatform === "darwin" && isCommandAvailable("xcrun"); + const reason = platformReason(platform, hostPlatform, sdk, hasXcrun); + if (reason !== null) return { platform, available: false, reason }; + // Available, with a note: running emulators work without Java, starting one does not. + return platform === "android" && sdk.javaHome === null + ? { platform, available: true, reason: ANDROID_JAVA_NOTE } + : { platform, available: true }; + }); + + const readDaemonFile = fs + .readFileString(daemonFilePath) + .pipe(Effect.flatMap(decodeDaemonFile), Effect.option); + + const summary: DeviceHost.DeviceHost["Service"]["summary"] = Effect.gen(function* () { + const [platforms, hubInstalled, agentDeviceInstalled] = yield* Effect.all([ + Effect.all([platformAvailability("ios"), platformAvailability("android")]), + isDeviceToolInstalled(stateDir, "hub").pipe(services, withPath), + isDeviceToolInstalled(stateDir, "agent").pipe(services, withPath), + ]); + const running = yield* Ref.get(runningRef); + const hubAlive = running + ? yield* running.hub.child.isRunning.pipe(Effect.orElseSucceed(() => false)) + : false; + const daemon = running?.agentDevice ? yield* readDaemonFile : Option.none(); + const agentAlive = + Option.isSome(daemon) && daemon.value.pid ? yield* isProcessAlive(daemon.value.pid) : false; + const tools = yield* deviceToolVersions(stateDir, { + ...(hubAlive ? { hub: DEVICE_HUB_VERSION } : {}), + ...(agentAlive && Option.isSome(daemon) && daemon.value.version + ? { agent: daemon.value.version } + : {}), + }).pipe(services, withPath, Effect.option); + return { + platforms, + hubInstalled, + agentDeviceInstalled, + tools: Option.getOrUndefined(tools), + toolInspectionError: Option.isNone(tools) + ? "Could not read which device tool versions are installed." + : undefined, + }; + }); + + const stopHub = (hub: HubProcess | undefined) => + hub ? Scope.close(hub.scope, Exit.void).pipe(Effect.ignore) : Effect.void; + + const commandLineOf = (pid: number) => + runner + .run({ + command: "ps", + args: ["-o", "command=", "-p", String(pid)], + timeout: Duration.seconds(5), + timeoutBehavior: "timedOutResult", + }) + .pipe( + Effect.map((result) => result.stdout), + Effect.orElseSucceed(() => ""), + ); + + // A previous server's hub, stopped only if the recorded pid still runs the recorded entry: + // the pid may have been reused by anything since. + const reapStaleHub = Effect.gen(function* () { + const previous = yield* fs + .readFileString(hubStatePath) + .pipe(Effect.flatMap(decodeHubStateFile), Effect.option); + if (Option.isNone(previous)) return; + const { pid, port, entryPath } = previous.value; + if (hostPlatform !== "win32" && (yield* isProcessAlive(pid))) { + if ((yield* commandLineOf(pid)).includes(entryPath)) { + yield* Effect.logWarning("Stopping a device hub left behind by a previous server", { + pid, + port, + }); + yield* Effect.sync(() => { + try { + process.kill(pid, "SIGTERM"); + } catch {} + }); + } + } + yield* fs.remove(hubStatePath, { force: true }).pipe(Effect.ignore); + }).pipe(Effect.ignoreCause); + + const recordHub = (hub: HubProcess, hubTool: DeviceToolPaths) => + encodeHubStateFile({ + pid: Number(hub.child.pid), + port: Number(new URL(hub.origin).port), + entryPath: hubTool.entryPath, + }).pipe( + Effect.flatMap((contents) => writeFileStringAtomically({ filePath: hubStatePath, contents })), + services, + withPath, + Effect.ignore, + ); + + const waitForReady = (origin: string) => + Effect.gen(function* () { + const deadline = (yield* Clock.currentTimeMillis) + HUB_READY_TIMEOUT_MS; + while (true) { + const ok = yield* httpClient + .pipe(HttpClient.withScope) + .get(`${origin}/readyz`) + .pipe( + Effect.flatMap((response) => + response.arrayBuffer.pipe(Effect.as(response.status === 200)), + ), + Effect.scoped, + Effect.timeout(Duration.seconds(2)), + Effect.orElseSucceed(() => false), + ); + if (ok) return; + if ((yield* Clock.currentTimeMillis) > deadline) { + return yield* new DeviceHost.DeviceHostError({ + step: "waiting for the device hub to answer", + detail: `The device hub did not become ready within ${HUB_READY_TIMEOUT_MS / 1000} seconds.`, + }); + } + yield* Effect.sleep(Duration.millis(200)); + } + }); + + const hubEnvironment = (base: NodeJS.ProcessEnv): NodeJS.ProcessEnv => ({ + ...base, + FORCE_COLOR: "0", + NO_COLOR: "1", + // The hub writes `server-.json` there; keep it out of the shared temp folder. + EXPO_DEVICE_HUB_STATE_DIR: path.join(deviceStateDir(path, stateDir), "hub-state"), + }); + + const spawnHub = Effect.fn("LocalDeviceHost.spawnHub")(function* (hubTool: DeviceToolPaths) { + yield* reapStaleHub; + yield* fs + .makeDirectory(deviceStateDir(path, stateDir), { recursive: true }) + .pipe(Effect.ignore); + const reserved = yield* net + .reserveLoopbackPort("127.0.0.1") + .pipe(Effect.mapError(hostError("reserving a port for the device hub"))); + const scope = yield* Scope.make("sequential"); + // Not recorded yet: stop it on failure or interruption from the spawn on. + return yield* Effect.gen(function* () { + const node = yield* leaseDeviceNode(stateDir).pipe( + Effect.provideService(Scope.Scope, scope), + services, + withPath, + Effect.mapError(installFailure("preparing Node.js")), + ); + const child = yield* spawner + .spawn( + ChildProcess.make(node.node, hubArguments(hubTool.entryPath, reserved, hostPlatform), { + cwd: deviceStateDir(path, stateDir), + detached: false, + shell: false, + stdout: "pipe", + stderr: "pipe", + env: hubEnvironment(yield* currentHostEnvironment), + }), + ) + .pipe( + Effect.provideService(Scope.Scope, scope), + Effect.mapError(hostError("starting the device hub")), + ); + const startedAtMillis = yield* Clock.currentTimeMillis; + const started = yield* Deferred.make<{ readonly port: number; readonly token: string }>(); + yield* Effect.forkIn(observeHubOutput(child, started), scope); + const announced = yield* Deferred.await(started).pipe( + Effect.timeoutOption(Duration.millis(HUB_READY_TIMEOUT_MS)), + ); + if (Option.isNone(announced)) { + return yield* new DeviceHost.DeviceHostError({ + step: "waiting for the device hub to start", + detail: "The device hub did not print its startup line. Update it in Settings → Devices.", + }); + } + const origin = `http://127.0.0.1:${announced.value.port}`; + yield* waitForReady(origin); + const hub: HubProcess = { + child, + scope, + origin, + token: announced.value.token, + startedAtMillis, + nodePath: node.node, + }; + yield* recordHub(hub, hubTool); + // The preview tunnel refuses this port while the hub has it, on every platform. + setOwnDeviceToolPort("hub", announced.value.port); + yield* Scope.addFinalizer( + scope, + Effect.sync(() => setOwnDeviceToolPort("hub", null, announced.value.port)), + ); + yield* Effect.logInfo("Device hub started", { + pid: Number(child.pid), + port: announced.value.port, + }); + return hub; + }).pipe(Effect.onError(() => Scope.close(scope, Exit.void).pipe(Effect.ignore))); + }); + + const observeHubOutput = ( + child: ChildProcessSpawner.ChildProcessHandle, + started: Deferred.Deferred<{ readonly port: number; readonly token: string }>, + ) => + child.all.pipe( + Stream.decodeText(), + Stream.splitLines, + Stream.runForEach((line) => + Effect.gen(function* () { + const startup = parseHubStartupLine(line); + if (startup) { + yield* Deferred.succeed(started, startup); + return; + } + const text = line.trim(); + if (text.length > 0) { + yield* Effect.logDebug("Device hub output", { + pid: Number(child.pid), + output: redactHubLine(text), + }); + } + }), + ), + Effect.ignoreCause, + ); + + // Waits for the hub's lifetime and starts a replacement with backoff; a span would nest across restarts. + const superviseHub: (hub: HubProcess, hubTool: DeviceToolPaths) => Effect.Effect = + Effect.fnUntraced( + function* (hub: HubProcess, hubTool: DeviceToolPaths) { + yield* Effect.result(hub.child.exitCode); + const running = yield* Ref.get(runningRef); + if (running?.hub.child.pid !== hub.child.pid) return; + const uptime = (yield* Clock.currentTimeMillis) - hub.startedAtMillis; + const delay = yield* Ref.modify(restartDelayRef, (current) => { + if (uptime >= HUB_RESTART_STABLE_UPTIME_MS) return [0, 0]; + const next = current === 0 ? 1_000 : Math.min(current * 2, HUB_RESTART_MAX_DELAY_MS); + return [current, next]; + }); + yield* Effect.logWarning("Device hub exited; restarting", { + pid: Number(hub.child.pid), + delayMs: delay, + }); + yield* stopHub(hub); + yield* Effect.sleep(Duration.millis(delay)); + yield* startLock.withPermits(1)( + Effect.gen(function* () { + const current = yield* Ref.get(runningRef); + if (current?.hub.child.pid !== hub.child.pid) return; + const replacement = yield* spawnHub(hubTool); + yield* Ref.set(runningRef, { ...current, hub: replacement }); + yield* Effect.forkDetach(superviseHub(replacement, hubTool)); + }), + ); + }, + Effect.catchCause((cause) => Effect.logWarning("Device hub supervisor failed", { cause })), + ); + + /** Every process's command line, for keeping tool versions a running process uses. */ + const runningCommandLines = runner + .run({ + command: "ps", + args: ["-ax", "-o", "command="], + timeout: Duration.seconds(10), + timeoutBehavior: "timedOutResult", + maxOutputBytes: 16 * 1024 * 1024, + }) + .pipe( + Effect.map((result) => (result.code === 0 && !result.timedOut ? result.stdout : null)), + Effect.orElseSucceed(() => null), + ); + + const pruneTool = (tool: "hub" | "agent") => + hostPlatform === "win32" + ? Effect.void + : pruneDeviceTool(stateDir, tool, runningCommandLines).pipe( + services, + withPath, + Effect.ignore, + ); + + const daemonEnvironment = Effect.gen(function* () { + return { + ...(yield* currentHostEnvironment), + AGENT_DEVICE_STATE_DIR: agentDeviceStateDir(path, stateDir), + AGENT_DEVICE_DAEMON_SERVER_MODE: "http", + // No five-minute idle timeout: the server owns shutdown. + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: "0", + AGENT_DEVICE_NO_UPDATE_NOTIFIER: "1", + FORCE_COLOR: "0", + NO_COLOR: "1", + } satisfies NodeJS.ProcessEnv; + }); + + const startAgentDeviceDaemon = Effect.fn("LocalDeviceHost.startAgentDeviceDaemon")(function* ( + entryPath: string, + nodePath: string, + ) { + yield* fs + .makeDirectory(agentDeviceStateDir(path, stateDir), { recursive: true }) + .pipe(Effect.ignore); + const toEndpoint = ( + file: typeof AgentDeviceDaemonFile.Type, + ): DeviceHost.AgentDeviceEndpoint => { + // The preview tunnel refuses the daemon's port too (forgotten when it is stopped). + setOwnDeviceToolPort("agent", file.httpPort); + return { baseUrl: `http://127.0.0.1:${file.httpPort}`, token: file.token, entryPath }; + }; + const existing = yield* readDaemonFile; + if (Option.isSome(existing) && existing.value.version !== AGENT_DEVICE_VERSION) { + // An earlier server's daemon on another pin: stop it with its own CLI and start ours. + yield* stopAgentDeviceDaemon; + } else if (Option.isSome(existing)) { + const alive = yield* httpClient + .pipe(HttpClient.withScope) + .get(`http://127.0.0.1:${existing.value.httpPort}/health`) + .pipe( + Effect.timeout(Duration.seconds(2)), + Effect.flatMap((response) => + response.arrayBuffer.pipe(Effect.as(response.status === 200)), + ), + Effect.scoped, + Effect.orElseSucceed(() => false), + ); + if (alive) return toEndpoint(existing.value); + yield* fs.remove(daemonFilePath, { force: true }).pipe(Effect.ignore); + } + // agent-device has no `daemon start`; `devices` is the cheapest command that starts and awaits it. + const runResult = yield* runner + .run({ + command: nodePath, + args: [entryPath, "devices", "--json"], + env: yield* daemonEnvironment, + timeout: Duration.millis(DAEMON_READY_TIMEOUT_MS), + timeoutBehavior: "timedOutResult", + }) + .pipe(Effect.result); + // Discovery may fail after startup; a published endpoint is enough. + const started = yield* readDaemonFile; + if (Option.isSome(started)) return toEndpoint(started.value); + if (runResult._tag === "Failure") { + return yield* hostError("starting agent tools")(runResult.failure); + } + if (runResult.success.timedOut) { + return yield* new DeviceHost.DeviceHostError({ + step: "starting agent tools", + detail: `Agent tools did not start within ${DAEMON_READY_TIMEOUT_MS / 1000} seconds.`, + }); + } + if (runResult.success.code !== 0) { + return yield* new DeviceHost.DeviceHostError({ + step: "starting agent tools", + detail: `agent-device could not start (exit code ${runResult.success.code}). Its log is in ${agentDeviceStateDir(path, stateDir)}.`, + }); + } + const deadline = (yield* Clock.currentTimeMillis) + DAEMON_READY_TIMEOUT_MS; + while (true) { + const file = yield* readDaemonFile; + if (Option.isSome(file)) return toEndpoint(file.value); + if ((yield* Clock.currentTimeMillis) > deadline) { + return yield* new DeviceHost.DeviceHostError({ + step: "starting agent tools", + detail: `Agent tools did not start within ${DAEMON_READY_TIMEOUT_MS / 1000} seconds.`, + }); + } + yield* Effect.sleep(Duration.millis(DAEMON_POLL_MS)); + } + }); + + // The daemon may predate this process (an earlier server was killed, possibly on an + // older pin), so it is stopped with the CLI of its recorded version when that is installed. + const stopAgentDeviceDaemon = Effect.gen(function* () { + setOwnDeviceToolPort("agent", null); + if (!(yield* fs.exists(daemonFilePath).pipe(Effect.orElseSucceed(() => false)))) return; + const recorded = yield* readDaemonFile; + if ( + Option.isSome(recorded) && + recorded.value.pid && + !(yield* isProcessAlive(recorded.value.pid)) + ) { + return yield* fs.remove(daemonFilePath, { force: true }); + } + const entryPath = yield* installedAgentDeviceEntry(stateDir, [ + ...(Option.isSome(recorded) && recorded.value.version ? [recorded.value.version] : []), + AGENT_DEVICE_VERSION, + ]).pipe(withPath, services); + if (!entryPath) return; + yield* Effect.scoped( + Effect.gen(function* () { + const node = yield* leaseDeviceNode(stateDir).pipe(services, withPath); + yield* runner.run({ + command: node.node, + args: [entryPath, "daemon", "stop", "--state-dir", agentDeviceStateDir(path, stateDir)], + env: { ...(yield* currentHostEnvironment), AGENT_DEVICE_NO_UPDATE_NOTIFIER: "1" }, + timeout: Duration.seconds(10), + timeoutBehavior: "timedOutResult", + }); + }), + ); + }).pipe(Effect.ignore); + + const serveSimHelpers = Effect.fn("LocalDeviceHost.serveSimHelpers")(function* ( + hubTool: DeviceToolPaths, + ) { + const serveSimDist = path.join( + hubTool.installDir, + "node_modules", + "expo-device-hub", + "vendor", + "serve-sim", + "dist", + ); + const candidate = { + serveSimAxSettings: path.join(serveSimDist, "simax", "serve-sim-ax-settings"), + serveSimCli: path.join(serveSimDist, "serve-sim.js"), + }; + const [axExists, cliExists] = yield* Effect.all([ + fs.exists(candidate.serveSimAxSettings).pipe(Effect.orElseSucceed(() => false)), + fs.exists(candidate.serveSimCli).pipe(Effect.orElseSucceed(() => false)), + ]); + return { + serveSimAxSettings: axExists ? candidate.serveSimAxSettings : null, + serveSimCli: cliExists ? candidate.serveSimCli : null, + }; + }); + + const ensureHubReady = Effect.fn("LocalDeviceHost.ensureHubReady")(function* ( + onPhase: (phase: DeviceHost.DeviceHostPhase, detail?: string) => Effect.Effect, + ) { + const running = yield* Ref.get(runningRef); + // A dead hub's record stays until a replacement is published: the agent + // daemon outlives the hub, and close needs its endpoint if the restart fails. + if (running) { + const alive = yield* running.hub.child.isRunning.pipe(Effect.orElseSucceed(() => false)); + if (alive) return running; + } + if (!(yield* isDeviceToolInstalled(stateDir, "hub").pipe(services, withPath))) { + yield* onPhase("installing", `Installing the device hub ${DEVICE_HUB_VERSION}…`); + } + const { paths: hubTool } = yield* Effect.scoped( + ensureDeviceTool(stateDir, "hub").pipe( + services, + withPath, + Effect.mapError(installFailure("installing device support")), + ), + ); + yield* onPhase("starting"); + const hub = yield* spawnHub(hubTool); + const next = yield* Effect.gen(function* () { + const nextRunning: RunningHost = { + hub, + agentDevice: running?.agentDevice ?? null, + helpers: yield* serveSimHelpers(hubTool), + }; + yield* Ref.set(runningRef, nextRunning); + return nextRunning; + }).pipe(Effect.onError(() => stopHub(hub))); + if (running) yield* stopHub(running.hub); + yield* Ref.set(restartDelayRef, 0); + yield* Effect.forkDetach(superviseHub(hub, hubTool)); + yield* pruneTool("hub").pipe(Effect.forkDetach); + return next; + }); + + const run: DeviceHost.DeviceHostReady["run"] = (command, args, options) => + currentSdk.pipe( + Effect.flatMap((sdk) => + runner.run({ + command: + command === "emulator" && sdk.root + ? path.join( + sdk.root, + "emulator", + hostPlatform === "win32" ? "emulator.exe" : "emulator", + ) + : command, + args, + env: deviceHostEnvironment(environment, sdk, hostPlatform, path), + timeout: Duration.millis(options?.timeoutMs ?? 20_000), + timeoutBehavior: "timedOutResult", + ...(options?.stdin === undefined ? {} : { stdin: options.stdin }), + ...(options?.maxOutputBytes === undefined + ? {} + : { maxOutputBytes: options.maxOutputBytes }), + }), + ), + Effect.map((result) => ({ + stdout: result.stdout, + stderr: result.stderr, + code: result.timedOut ? 124 : Number(result.code ?? 127), + })), + Effect.catch((cause) => Effect.succeed({ stdout: "", stderr: String(cause), code: 127 })), + ); + + // Made once per host: its model cache outlives every hub start and every refresh. + const hubOperations = makeHostDeviceOperations({ + http: httpClient, + run, + platformAvailable: (platform) => + platformAvailability(platform).pipe(Effect.map((availability) => availability.available)), + scratchDir: path.join(deviceStateDir(path, stateDir), "tmp"), + androidAvds: { + home: androidAvdHome(environment, path), + readTextFile: (file) => + fs.readFileString(file).pipe( + Effect.map((text) => (text.length > 256 * 1024 ? undefined : text)), + Effect.orElseSucceed(() => undefined), + ), + }, + }); + const toReady = (running: RunningHost): DeviceHost.DeviceHostReady => ({ + hub: { origin: running.hub.origin, token: running.hub.token }, + devices: hubOperations({ origin: running.hub.origin, token: running.hub.token }), + nodePath: running.hub.nodePath, + run, + helpers: running.helpers, + ...(running.agentDevice + ? { releaseAgentSession: releaseAgentDeviceSession(httpClient, running.agentDevice) } + : {}), + }); + + const ensureReady: DeviceHost.DeviceHost["Service"]["ensureReady"] = (onPhase) => + startLock.withPermits(1)(ensureHubReady(onPhase).pipe(Effect.map(toReady))); + + const ensureAgentReady: DeviceHost.DeviceHost["Service"]["ensureAgentReady"] = (onPhase) => + startLock.withPermits(1)( + Effect.gen(function* () { + const running = yield* ensureHubReady(onPhase); + if (!(yield* isDeviceToolInstalled(stateDir, "agent").pipe(services, withPath))) { + yield* onPhase("installing", `Installing agent tools ${AGENT_DEVICE_VERSION}…`); + } + const { paths: agentTool } = yield* Effect.scoped( + ensureDeviceTool(stateDir, "agent").pipe( + services, + withPath, + Effect.mapError(installFailure("installing agent tools")), + ), + ); + if (!running.agentDevice) yield* onPhase("starting"); + const agentDevice = yield* startAgentDeviceDaemon( + agentTool.entryPath, + running.hub.nodePath, + ); + const next = { ...running, agentDevice }; + yield* Ref.set(runningRef, next); + yield* pruneTool("agent").pipe(Effect.forkDetach); + return { ...toReady(next), agentDevice }; + }), + ); + + const current: DeviceHost.DeviceHost["Service"]["current"] = Ref.get(runningRef).pipe( + Effect.map((running) => (running ? toReady(running) : null)), + ); + + const releaseAgentSession: DeviceHost.DeviceHost["Service"]["releaseAgentSession"] = (session) => + Effect.gen(function* () { + const running = yield* Ref.get(runningRef); + const endpoint = + running?.agentDevice ?? + (yield* readDaemonFile.pipe( + Effect.map((file) => + Option.isSome(file) + ? { baseUrl: `http://127.0.0.1:${file.value.httpPort}`, token: file.value.token } + : null, + ), + )); + if (!endpoint) return true; + yield* releaseAgentDeviceSession(httpClient, endpoint)(session); + return true; + }).pipe( + Effect.catch((cause) => + Effect.logWarning("Could not end the agent-device session", { session, cause }).pipe( + Effect.as(false), + ), + ), + ); + + const stopAgent: DeviceHost.DeviceHost["Service"]["stopAgent"] = startLock.withPermits(1)( + Effect.gen(function* () { + yield* stopAgentDeviceDaemon; + yield* Ref.update(runningRef, (running) => + running ? { ...running, agentDevice: null } : running, + ); + }), + ); + + const stop: DeviceHost.DeviceHost["Service"]["stop"] = startLock.withPermits(1)( + Effect.gen(function* () { + const running = yield* Ref.getAndSet(runningRef, null); + // Hub cleanup waits for its exit; it must not strand the detached agent daemon. + yield* Effect.all( + [ + stopHub(running?.hub).pipe( + Effect.andThen(fs.remove(hubStatePath, { force: true }).pipe(Effect.ignore)), + ), + stopAgentDeviceDaemon, + ], + { concurrency: "unbounded", discard: true }, + ); + }), + ); + + /** + * Installs the pinned version (a repair when its files went missing). The + * hub always starts on the pinned version; a daemon an earlier server left + * running on an older pin is restarted on this one. + */ + const updateTool: DeviceHost.DeviceHost["Service"]["updateTool"] = (tool) => + Effect.gen(function* () { + yield* Effect.scoped( + ensureDeviceTool(stateDir, tool).pipe( + services, + withPath, + Effect.mapError( + installFailure(tool === "hub" ? "updating the device hub" : "updating agent tools"), + ), + ), + ); + if (tool !== "agent") return; + const running = yield* Ref.get(runningRef); + if (!running?.agentDevice) return; + const daemon = yield* readDaemonFile; + if (Option.isSome(daemon) && daemon.value.version === AGENT_DEVICE_VERSION) return; + yield* stopAgent; + yield* ensureAgentReady(() => Effect.void); + }); + + yield* Effect.addFinalizer(() => stop); + + return DeviceHost.DeviceHost.of({ + summary, + platformAvailability, + ensureReady, + ensureAgentReady, + updateTool, + current, + releaseAgentSession, + stopAgent, + stop, + }); +}); + +export const layer = Layer.effect(DeviceHost.DeviceHost, make()); diff --git a/apps/server/src/device/deviceToolVersions.json b/apps/server/src/device/deviceToolVersions.json new file mode 100644 index 000000000..ce61669b2 --- /dev/null +++ b/apps/server/src/device/deviceToolVersions.json @@ -0,0 +1,4 @@ +{ + "agent-device": "0.21.36", + "expo-device-hub": "0.22.1" +} diff --git a/apps/server/src/device/mjpegStillFrame.ts b/apps/server/src/device/mjpegStillFrame.ts new file mode 100644 index 000000000..7cbe30075 --- /dev/null +++ b/apps/server/src/device/mjpegStillFrame.ts @@ -0,0 +1,121 @@ +/** + * Browsers paint a multipart/x-mixed-replace part only once the next part begins, and the hub + * writes a part only when the screen changes. Left alone, an `` viewer never shows a still + * screen (clients such as the iOS app then give up waiting for the first frame) and always hides + * the last frame of a change. Re-sending the last part after a short quiet period makes it paint. + */ +import * as Arr from "effect/Array"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Option from "effect/Option"; +import * as Stream from "effect/Stream"; + +const STILL_FRAME_DELAY = Duration.millis(100); + +const HEADER_END = Buffer.from("\r\n\r\n"); +const MAX_HEADER_BYTES = 16 * 1024; +// Simulator frames are far smaller; a larger claim is unknown framing, not something to retain. +const MAX_PART_BYTES = 16 * 1024 * 1024; + +type Part = Arr.NonEmptyReadonlyArray; + +const partLength = (headers: string) => { + if (!headers.startsWith("--")) return undefined; + const length = Number(/^content-length:\s*(\d+)\s*$/im.exec(headers)?.[1]); + return length <= MAX_PART_BYTES ? length : undefined; +}; + +const endsWithCrlf = (part: Part) => { + const last = Arr.lastNonEmpty(part); + const cr = last.length > 1 ? last.at(-2) : part.at(-2)?.at(-1); + return cr === 0x0d && last.at(-1) === 0x0a; +}; + +/** + * Follows the hub's part framing (`--frame`, `Content-Length`, CRLF CRLF, body, CRLF; each part + * is one write) and keeps the last complete part as views into the forwarded chunks, so frame + * bytes are never copied. Any other framing turns tracking off and the rest passes through. + */ +const partTracker = () => { + let pieces: Array = []; + let size = 0; + let partEnd: number | null = null; + let tracking = true; + return { + /** The last complete part, while the stream sits at a part boundary. */ + still: null as Part | null, + feed(chunk: Uint8Array) { + if (!tracking) return; + let rest = Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength); + let part: Part | undefined; + while (rest.length > 0) { + if (partEnd === null) { + // A header split across chunks is re-scanned per chunk. Only a header-sized + // prefix of the new chunk joins the scan, so frame bytes are never copied. + const head = + size === 0 + ? rest + : Buffer.concat([...pieces, rest.subarray(0, MAX_HEADER_BYTES + HEADER_END.length)]); + const headerEnd = head.indexOf(HEADER_END); + if (headerEnd < 0 && head.length <= MAX_HEADER_BYTES) { + pieces.push(rest); + size += rest.length; + break; + } + const length = + headerEnd < 0 || headerEnd > MAX_HEADER_BYTES + ? undefined + : partLength(head.subarray(0, headerEnd).toString("latin1")); + if (length === undefined) { + tracking = false; + break; + } + partEnd = headerEnd + HEADER_END.length + length + 2; + } + const taken = rest.subarray(0, partEnd - size); + pieces.push(taken); + size += taken.length; + rest = rest.subarray(taken.length); + if (size < partEnd) break; + const complete = pieces; + if (!Arr.isArrayNonEmpty(complete) || !endsWithCrlf(complete)) { + tracking = false; + break; + } + part = complete; + pieces = []; + size = 0; + partEnd = null; + } + if (!tracking) pieces = []; + this.still = tracking && size === 0 ? (part ?? this.still) : null; + }, + }; +}; + +/** Re-sends the latest frame once when the stream goes quiet between frames. */ +export const repeatStillFrame = (body: Stream.Stream) => + Stream.transformPull(body, (pull, scope) => + Effect.sync(() => { + const tracker = partTracker(); + let pending: Fiber.Fiber, Effect.Error> | null = + null; + const forward = (chunks: Effect.Success) => { + pending = null; + for (const chunk of chunks) tracker.feed(chunk); + return chunks; + }; + return Effect.gen(function* () { + const still = tracker.still; + if (still === null && pending === null) return forward(yield* pull); + pending ??= yield* Effect.forkIn(pull, scope); + if (still === null) return forward(yield* Fiber.join(pending)); + // Timing out the join leaves the pull running, so a frame arriving mid-wait is kept. + const next = yield* Fiber.join(pending).pipe(Effect.timeoutOption(STILL_FRAME_DELAY)); + if (Option.isSome(next)) return forward(next.value); + tracker.still = null; + return still; + }); + }), + ); diff --git a/apps/server/src/environment/Layers/ServerEnvironment.ts b/apps/server/src/environment/Layers/ServerEnvironment.ts index 33ca82d6e..576ff9445 100644 --- a/apps/server/src/environment/Layers/ServerEnvironment.ts +++ b/apps/server/src/environment/Layers/ServerEnvironment.ts @@ -120,6 +120,7 @@ export const makeServerEnvironment = Effect.fn("makeServerEnvironment")(function projectLead: true, childStartSelection: true, projectContext: true, + devices: true, deviceNames: true, draftGoalSetup: true, pullRequestSwitchesByNumber: true, diff --git a/apps/server/src/guide/topics.ts b/apps/server/src/guide/topics.ts index b2661c21f..372b84a0b 100644 --- a/apps/server/src/guide/topics.ts +++ b/apps/server/src/guide/topics.ts @@ -393,6 +393,33 @@ The project menu's Project brief and notes, the command palette and the lead cha Show device toolbar, in the browser's options menu (⋮), sets the page's size: Responsive fills the panel, and a Phone, Tablet, Laptop or Desktop size shrinks to fit it. New tabs open at {place:general.browserPageSize}, which the size menu can also set. Words typed in the browser's address bar are searched for, with the search engine set in {place:general.browserSearchEngine}. An address loads as it always did.`, + }, + { + id: "device-tab", + title: "The Device tab", + keywords: [ + "device", + "simulator", + "emulator", + "ios", + "android", + "iphone", + "xcode", + "android sdk", + "app preview", + "device_open", + "agent-device", + "iphone duo", + "foldable", + "accessibility frames", + ], + body: `The Device tab in the right panel shows a live iOS Simulator or Android Emulator running on the thread's computer. You tap, swipe and type on its screen as on a phone. The screen sits in a drawing of the device's model (an unknown model gets a plain frame). The toolbar above it has Home, the app switcher, Back and Recents on Android, rotate, screenshot, accessibility frames and power off. Accessibility frames outline each element on screen and show its label on hover; their Inspect button makes a tap show the label instead of touching the device. An iPhone Duo shows its stance buttons beside the screen (closed, book, open, laptop, tent, and table mode where offered), and an Android foldable shows fold and unfold. The Tools drawer sets appearance, text size, accessibility options, location and permissions, opens a URL, launches or quits an app, shakes the device, sends a test push, and on Android sets the network. + +Open it from the right panel's + menu or its launcher, or with Open device in the command palette. It shows in project threads, not General Chats. + +iOS needs a Mac with Xcode, and Android needs the Android SDK with the emulator. Opening the tab downloads nothing: {place:devices.enable} has to be on first, and then the device hub installs the first time the tab or an agent asks. {place:devices.tools} shows the installed versions and an Update button when a newer one is needed. + +Agents get device tools only while {place:devices.agentAccess} is on, and only in sessions started after it was turned on. An agent that opens a device brings up the Device tab for anyone watching the thread. Closing a device leaves the simulator running; power off shuts it down.`, }, { id: "file-viewer", diff --git a/apps/server/src/mcp/McpDeviceServer.ts b/apps/server/src/mcp/McpDeviceServer.ts new file mode 100644 index 000000000..fe9a1805a --- /dev/null +++ b/apps/server/src/mcp/McpDeviceServer.ts @@ -0,0 +1,236 @@ +/** + * The device tools' endpoint, `threadlines_device` at `/mcp/device` + * (docs/design/devices.md). + * + * Its own MCP server, like McpCommandsServer: a provider lists it only for + * sessions started with `deviceTools`, and Codex approves calls per server. + * The same bearer credentials as `/mcp` (McpSessionRegistry); a credential + * reaches this endpoint only if it carries the `device` grant, and each call + * checks again that "Use devices" and "Agents can use devices" are still on. + * + * Hand-registered rather than a toolkit so `device_screenshot` can answer + * with an MCP image block instead of base64 the model cannot see. + */ +import * as Cause from "effect/Cause"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import type * as Types from "effect/Types"; +import { McpProtocol, McpSchema, McpServer, Tool } from "effect/ai"; +import { HttpRouter, HttpServerRequest, type HttpServerResponse } from "effect/http"; + +import { DeviceService } from "../device/DeviceService.ts"; +import { deviceClose, deviceList, deviceOpen, deviceScreenshot } from "./deviceToolHandlers.ts"; +import { makeOutsideToolLimits } from "./outsideToolLimit.ts"; +import { + DEVICE_CLOSE_TOOL_NAME, + DEVICE_LIST_TOOL_NAME, + DEVICE_OPEN_TOOL_NAME, + DEVICE_SCREENSHOT_TOOL_NAME, + DEVICE_TOOLS, + DeviceToolListInput, +} from "./deviceTools.ts"; +import { + DeviceToolCloseInput, + DeviceToolOpenInput, + DeviceToolTargetInput, +} from "@threadlines/contracts"; +import { + MCP_DEVICE_ROUTE_PATH, + normalizeMcpHttpResponse, + readBearerToken, + unauthorizedMcpResponse, +} from "./McpHttpServer.ts"; +import { type McpInvocationScope, mcpSessionRegistry } from "./McpSessionRegistry.ts"; + +/** Which runtime a device tool call came from, with the outside-work limit read for this request. */ +export class McpDeviceInvocation extends Context.Service()( + "@threadlines/server/mcp/McpDeviceInvocation", +) {} + +const authenticate = Effect.gen(function* () { + const limits = yield* makeOutsideToolLimits; + return Effect.fn("McpDeviceServer.authenticate")(function* ( + handler: Effect.Effect< + HttpServerResponse.HttpServerResponse, + Types.unhandled, + McpDeviceInvocation + >, + ) { + const request = yield* HttpServerRequest.HttpServerRequest; + const scope = yield* mcpSessionRegistry.resolve(readBearerToken(request)); + if (scope === null || !scope.device) { + return unauthorizedMcpResponse; + } + // Read on every request: outside work can arrive in a session that started without it. + const outsideLimit = yield* limits.forScope(scope); + return yield* handler.pipe( + Effect.provideService(McpDeviceInvocation, { + ...scope, + ...(outsideLimit !== null ? { outsideLimit } : {}), + }), + Effect.map(normalizeMcpHttpResponse), + ); + }); +}); + +const AuthenticationLive = HttpRouter.middleware<{ + provides: McpDeviceInvocation; +}>()(authenticate).layer; + +const describe = (tool: Tool.Any) => + new McpSchema.Tool({ + name: tool.name, + description: Tool.getDescription(tool), + inputSchema: Tool.getJsonSchema(tool), + annotations: { + ...Context.getOption(tool.annotations, Tool.Title).pipe( + Option.map((title) => ({ title })), + Option.getOrUndefined, + ), + readOnlyHint: Context.get(tool.annotations, Tool.Readonly), + destructiveHint: Context.get(tool.annotations, Tool.Destructive), + idempotentHint: Context.get(tool.annotations, Tool.Idempotent), + openWorldHint: Context.get(tool.annotations, Tool.OpenWorld), + }, + }); + +const textResult = (text: string, isError: boolean, structured?: Record) => + new McpSchema.CallToolResult({ + isError, + ...(structured !== undefined ? { structuredContent: structured } : {}), + content: [{ type: "text", text }], + }); + +/** A failure the model can read: the error's own words. Interrupts and defects stay what they are. */ +const failureResult = (cause: Cause.Cause) => { + if (Cause.hasInterrupts(cause) || cause.reasons.some(Cause.isDieReason)) { + return Effect.failCause(cause).pipe(Effect.orDie); + } + const error = cause.reasons.find(Cause.isFailReason)?.error; + const message = + typeof error === "object" && error !== null && "message" in error + ? String(error.message) + : "The device tool failed."; + return Effect.succeed(textResult(message, true)); +}; + +const screenshotResult = (shot: { + readonly device: { readonly name: string; readonly version: string }; + readonly png: Uint8Array; + readonly width: number; + readonly height: number; +}) => + new McpSchema.CallToolResult({ + isError: false, + structuredContent: { width: shot.width, height: shot.height }, + content: [ + { + type: "text", + text: `Screenshot of ${shot.device.name} (${shot.device.version}), ${shot.width}x${shot.height} pixels.`, + }, + { type: "image", data: shot.png, mimeType: "image/png" }, + ], + }); + +const registerDeviceTools = Effect.fn("McpDeviceServer.registerDeviceTools")(function* () { + const server = yield* McpServer.McpServer; + const devices = yield* DeviceService; + const decodeList = Schema.decodeUnknownEffect(DeviceToolListInput); + const decodeOpen = Schema.decodeUnknownEffect(DeviceToolOpenInput); + const decodeTarget = Schema.decodeUnknownEffect(DeviceToolTargetInput); + const decodeClose = Schema.decodeUnknownEffect(DeviceToolCloseInput); + + // The caller comes off the request's own fiber: a hand-registered handler + // does not run under the router middleware that provides it. + const callerOf = (fiber: { readonly context: Context.Context }) => + Context.getUnsafe(fiber.context as Context.Context, McpDeviceInvocation); + + const badArguments = (error: unknown) => + textResult( + `The arguments do not fit the tool's schema: ${error instanceof Error ? error.message : String(error)}`, + true, + ); + + const handlers: Record< + string, + ( + caller: McpInvocationScope, + payload: unknown, + ) => Effect.Effect + > = { + [DEVICE_LIST_TOOL_NAME]: (caller, payload) => + decodeList(payload ?? {}).pipe( + Effect.matchEffect({ + onFailure: (error) => Effect.succeed(badArguments(error)), + onSuccess: (input) => + deviceList(caller, input).pipe( + Effect.map((result) => + textResult(JSON.stringify(result, null, 2), false, { ...result }), + ), + Effect.catchCause(failureResult), + ), + }), + ), + [DEVICE_OPEN_TOOL_NAME]: (caller, payload) => + decodeOpen(payload ?? {}).pipe( + Effect.matchEffect({ + onFailure: (error) => Effect.succeed(badArguments(error)), + onSuccess: (input) => + deviceOpen(caller, input).pipe( + Effect.map((result) => textResult(result.quickStart, false, { ...result })), + Effect.catchCause(failureResult), + ), + }), + ), + [DEVICE_SCREENSHOT_TOOL_NAME]: (caller, payload) => + decodeTarget(payload ?? {}).pipe( + Effect.matchEffect({ + onFailure: (error) => Effect.succeed(badArguments(error)), + onSuccess: (input) => + deviceScreenshot(caller, input).pipe( + Effect.map(screenshotResult), + Effect.catchCause(failureResult), + ), + }), + ), + [DEVICE_CLOSE_TOOL_NAME]: (caller, payload) => + decodeClose(payload ?? {}).pipe( + Effect.matchEffect({ + onFailure: (error) => Effect.succeed(badArguments(error)), + onSuccess: (input) => + deviceClose(caller, input).pipe( + Effect.as(textResult("Closed.", false, {})), + Effect.catchCause(failureResult), + ), + }), + ), + }; + + for (const tool of DEVICE_TOOLS) { + const handle = handlers[tool.name]!; + yield* server.addTool({ + tool: describe(tool), + annotations: tool.annotations, + handle: (payload) => + Effect.withFiber((fiber) => + handle(callerOf(fiber), payload).pipe(Effect.provideService(DeviceService, devices)), + ), + }); + } +}); + +export const layer = Layer.fresh( + Layer.effectDiscard(registerDeviceTools()).pipe( + Layer.provideMerge( + McpServer.layerHttp({ + name: "threadlines-device", + version: "1", + path: MCP_DEVICE_ROUTE_PATH, + protocols: [McpProtocol.v2025_06_18], + }).pipe(Layer.provide(AuthenticationLive)), + ), + ), +); diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index d4f7289ca..739c7fa44 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -79,6 +79,15 @@ export function mcpCommandsEndpointUrl(port: number): string { return `http://127.0.0.1:${port}${MCP_COMMANDS_ROUTE_PATH}`; } +/** The device tools' endpoint (McpDeviceServer, docs/design/devices.md): its + * own server, so a provider lists them only for sessions with `deviceTools`. */ +export const MCP_DEVICE_ROUTE_PATH = "/mcp/device"; + +/** The device tools' address, on loopback for the same reason. */ +export function mcpDeviceEndpointUrl(port: number): string { + return `http://127.0.0.1:${port}${MCP_DEVICE_ROUTE_PATH}`; +} + /** * The same endpoint, in the shape Codex takes it. * diff --git a/apps/server/src/mcp/McpSessionRegistry.ts b/apps/server/src/mcp/McpSessionRegistry.ts index 99f3a86ae..80f37a533 100644 --- a/apps/server/src/mcp/McpSessionRegistry.ts +++ b/apps/server/src/mcp/McpSessionRegistry.ts @@ -54,6 +54,8 @@ export interface McpInvocationScope { readonly pages: boolean; /** May ask the user to run commands at `/mcp/commands` (McpCommandsServer). */ readonly commands: boolean; + /** May use the device tools at `/mcp/device` (McpDeviceServer). */ + readonly device: boolean; /** Room tools this credential may call at `/mcp/room`. Empty: none. */ readonly roomTools: ReadonlySet; /** @@ -80,6 +82,8 @@ export interface McpCredentialRequest { readonly pages?: boolean | undefined; /** `ask_user_to_run`. Never granted to a side runtime, whatever is asked. */ readonly commands?: boolean | undefined; + /** The device tools. Never granted to a side runtime, whatever is asked. */ + readonly device?: boolean | undefined; /** * A side runtime's kind. A side key without one is treated as a review, * the narrowest. @@ -156,6 +160,7 @@ export const makeMcpSessionRegistry = (): McpSessionRegistryShape => { browser: request.browser && sideKind === undefined, pages: request.pages === true && sideKind === undefined, commands: request.commands === true && sideKind === undefined, + device: request.device === true && sideKind === undefined, roomTools: new Set( request.room ? roomToolsFor(sideKind, { diff --git a/apps/server/src/mcp/alertAssistantsToolHandlers.test.ts b/apps/server/src/mcp/alertAssistantsToolHandlers.test.ts index 34cde7710..06f9aae49 100644 --- a/apps/server/src/mcp/alertAssistantsToolHandlers.test.ts +++ b/apps/server/src/mcp/alertAssistantsToolHandlers.test.ts @@ -47,6 +47,7 @@ const caller = (side?: "ask"): McpInvocationScope => ({ browser: true, pages: true, commands: false, + device: false, roomTools: new Set(roomToolsFor(side, { guideOnly: side === undefined })), ...(side !== undefined ? { side: { sideTurnId: SideTurnId.make("side-1"), kind: side } } : {}), }); diff --git a/apps/server/src/mcp/deviceToolHandlers.test.ts b/apps/server/src/mcp/deviceToolHandlers.test.ts new file mode 100644 index 000000000..f09a57a22 --- /dev/null +++ b/apps/server/src/mcp/deviceToolHandlers.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from "@effect/vitest"; +import { type DeviceId, type DeviceSummary, ThreadId } from "@threadlines/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; + +import { DeviceService } from "../device/DeviceService.ts"; +import { deviceList, pickDevice } from "./deviceToolHandlers.ts"; +import type { McpInvocationScope } from "./McpSessionRegistry.ts"; + +const device = (id: string, overrides: Partial): DeviceSummary => ({ + id: id as DeviceId, + platform: "android", + name: id, + version: "Android 16", + booted: false, + physical: false, + ...overrides, +}); + +describe("pickDevice", () => { + it.effect("never picks a plugged-in phone by default, only when named", () => + Effect.gen(function* () { + const phone = device("R5CT1234", { booted: true, physical: true }); + const emulator = device("emulator-5554", {}); + const state = { devices: [phone, emulator], hostStatus: "ready" as const }; + + expect((yield* pickDevice(state, { platform: "android" })).id).toBe("emulator-5554"); + expect((yield* pickDevice(state, { deviceId: phone.id })).id).toBe("R5CT1234"); + }), + ); +}); + +describe("device tools under outside work", () => { + it.effect( + "refuse while an outside connection's work limits the session, even with devices on", + () => + Effect.gen(function* () { + const thread = ThreadId.make("thread-outside"); + const caller: McpInvocationScope = { + threadId: thread, + sessionKey: thread, + participantId: null, + generation: 1, + agentId: "agent", + browser: true, + pages: true, + commands: true, + device: true, + roomTools: new Set(), + outsideLimit: { names: ["Helper laptop"], projectIds: null }, + }; + const refused = yield* deviceList(caller, {}).pipe(Effect.flip); + expect(refused.message).toContain("Devices are off for this agent while Helper laptop"); + }).pipe( + Effect.provide( + Layer.mock(DeviceService)({ + state: Effect.succeed({ enabled: true, agentAccessEnabled: true } as never), + }), + ), + ), + ); +}); diff --git a/apps/server/src/mcp/deviceToolHandlers.ts b/apps/server/src/mcp/deviceToolHandlers.ts new file mode 100644 index 000000000..461c55254 --- /dev/null +++ b/apps/server/src/mcp/deviceToolHandlers.ts @@ -0,0 +1,213 @@ +/** + * What the device tools do (docs/design/devices.md), for McpDeviceServer. + * Ported from T3 Code's device toolkit handlers, with one host per server. + */ +import { + type DeviceError, + type DeviceId, + type DevicePlatform, + type DeviceServiceState, + type DeviceSummary, + type DeviceToolCloseInput, + type DeviceToolListResult, + type DeviceToolOpenInput, + type DeviceToolOpenResult, + type DeviceToolTargetInput, + DeviceToolUnavailableError, + type ThreadId, +} from "@threadlines/contracts"; +import * as Effect from "effect/Effect"; + +import { DeviceService, type DeviceScreenshot } from "../device/DeviceService.ts"; +import type { McpInvocationScope } from "./McpSessionRegistry.ts"; +import { outsideDeviceDetail } from "./outsideToolLimit.ts"; + +const shellWord = (value: string) => + /^[a-zA-Z0-9_./:-]+$/.test(value) ? value : "'" + value.replaceAll("'", "'\"'\"'") + "'"; + +/** How to drive the device with its bound launcher: returned by `device_open`, never kept in a prompt. */ +export function agentDeviceQuickStart(device: DeviceSummary, command: string): string { + const executable = shellWord(command); + const run = (args: string) => `${executable} ${args}`; + return [ + `The user is watching ${device.name} (${device.version}) in the Device tab.`, + `Drive it with ${executable}. It is bound to this device and session, so it needs no --platform, --udid, --serial, --config or --session flags. Use this exact path; login shells may reset PATH.`, + // Screenshots and input need the session `open` starts, and an iOS snapshot also needs its app. + // device_open does not run open itself: on iOS that can start a minutes-long XCTest runner build. + "Every app launch, deep link, tap, text entry, alert, setting, screenshot and recording on this device goes through it. Run open (apps lists them) before snapshots, CLI screenshots or input; it starts the session they need. Use device_screenshot just to look.", + ` ${run("open --relaunch")} # restarts the app; Expo dev clients connect to their Metro, --metro-port picks it`, + ` ${run("open ")}`, + ` ${run("snapshot -i")} # accessibility tree with @eN refs`, + ` ${run("press @e3 --settle")}`, + ` ${run('fill @e5 "text" --settle')}`, + ` ${run("alert accept")}`, + ` ${run("settings appearance dark")} # also text-size, permission grant `, + ` ${run("screenshot /tmp/shot.png")} # ${device.platform === "ios" ? "iOS saves 1x points; add --pixel-density (3 on most iPhones) for device resolution, " : ""}or call device_screenshot to look`, + ` ${run("record start --scope device /tmp/run.mp4")} # then record stop`, + ` ${run("install ")}`, + `Prefer snapshot refs over coordinates. Run ${executable} help for flags and ${executable} help workflow for guides.`, + "Do not use simctl, idb, adb or ssh for those actions. They stay fine for builds, installs, code sync, Metro, port forwards, logs, test runners and what this command cannot do; come back to it for every UI action and screenshot so the user can watch.", + "Each device_open returns its own command; opening another device does not retarget this one.", + "The command is removed when the device closes or the thread is put away. If it is missing, call device_open again.", + device.platform === "ios" + ? "First use builds an XCTest runner and can take a couple of minutes; later commands are fast." + : "The Android snapshot helper installs itself on first use.", + ].join("\n"); +} + +const NO_ACCESS = + "Agents can't use devices on this computer right now. Ask the user to turn on Use devices and Agents can use devices in Settings → Devices; it applies to agent sessions started after that, such as in a new thread."; + +/** The caller may use the device tools: its credential has the grant and the settings still allow it. */ +const requireAccess = (caller: McpInvocationScope) => + Effect.gen(function* () { + if (!caller.device) return yield* new DeviceToolUnavailableError({ reason: NO_ACCESS }); + // Work from an outside connection must not reach this machine's devices. + if (caller.outsideLimit !== undefined) { + return yield* new DeviceToolUnavailableError({ + reason: outsideDeviceDetail(caller.outsideLimit), + }); + } + const state = yield* (yield* DeviceService).state; + if (!state.enabled || !state.agentAccessEnabled) { + return yield* new DeviceToolUnavailableError({ reason: NO_ACCESS }); + } + return state; + }); + +/** + * The device an agent means: the one it named, else the booted or first + * simulator or emulator of its platform. A physical phone is never picked by + * default; an agent gets one only by naming its id. + */ +export const pickDevice = ( + state: Pick, + input: { + readonly deviceId?: DeviceId | undefined; + readonly platform?: DevicePlatform | undefined; + }, +): Effect.Effect => { + if (input.deviceId !== undefined) { + const found = state.devices.find((device) => device.id === input.deviceId); + if (found) return Effect.succeed(found); + return Effect.fail( + new DeviceToolUnavailableError({ + reason: [ + `No device ${input.deviceId} on this computer.`, + ...(state.hostStatus === "failed" && state.hostStatusDetail + ? [`Device support failed to start: ${state.hostStatusDetail}`] + : []), + "Call device_list for current ids.", + ].join(" "), + }), + ); + } + const candidates = state.devices.filter( + (device) => + !device.physical && (input.platform === undefined || device.platform === input.platform), + ); + if (candidates.length === 0) { + return Effect.fail( + new DeviceToolUnavailableError({ + reason: + state.hostStatus === "failed" && state.hostStatusDetail + ? state.hostStatusDetail + : input.platform === undefined + ? "No simulators or emulators were found. Call device_list to see why." + : `No ${input.platform === "ios" ? "iOS" : "Android"} devices were found. Call device_list to see why.`, + }), + ); + } + if ( + input.platform === undefined && + new Set(candidates.map((device) => device.platform)).size > 1 + ) { + return Effect.fail( + new DeviceToolUnavailableError({ + reason: "Both iOS and Android devices are available; pass platform or deviceId.", + }), + ); + } + return Effect.succeed(candidates.find((device) => device.booted) ?? candidates[0]!); +}; + +const toolError = (error: DeviceError | DeviceToolUnavailableError) => error; + +export const deviceList = ( + caller: McpInvocationScope, + input: { readonly refresh?: boolean | undefined }, +) => + Effect.gen(function* () { + yield* requireAccess(caller); + const devices = yield* DeviceService; + const state = yield* devices.list({ refresh: input.refresh !== false }); + return { + hostStatus: state.hostStatus, + ...(state.hostStatusDetail ? { hostStatusDetail: state.hostStatusDetail } : {}), + platforms: state.platforms, + devices: state.devices, + open: state.sessions + .filter((session) => session.threadId === caller.threadId) + .map((session) => ({ deviceId: session.deviceId })), + } satisfies DeviceToolListResult; + }).pipe(Effect.mapError(toolError)); + +export const deviceOpen = (caller: McpInvocationScope, input: DeviceToolOpenInput) => + Effect.gen(function* () { + yield* requireAccess(caller); + const devices = yield* DeviceService; + const listed = yield* devices.list({ refresh: true }); + const target = yield* pickDevice(listed, input); + const opened = yield* devices.agentOpen({ + threadId: caller.threadId, + deviceId: target.id, + platform: target.platform, + }); + return { + device: opened.device, + agentDevice: { command: opened.command, session: opened.agentSession }, + quickStart: agentDeviceQuickStart(opened.device, opened.command), + } satisfies DeviceToolOpenResult; + }).pipe(Effect.mapError(toolError)); + +/** The session a screenshot or close means: the one named, else the last one this thread opened. */ +const sessionFor = (threadId: ThreadId, deviceId: DeviceId | undefined) => + Effect.gen(function* () { + const sessions = yield* (yield* DeviceService).sessionsForThread(threadId); + const session = + deviceId === undefined + ? sessions.at(-1) + : sessions.find((candidate) => candidate.deviceId === deviceId); + if (!session) { + return yield* new DeviceToolUnavailableError({ + reason: + deviceId === undefined + ? "No device is open in this thread. Call device_open first." + : `Device ${deviceId} is not open in this thread. Call device_open first.`, + }); + } + return session; + }); + +export const deviceScreenshot = ( + caller: McpInvocationScope, + input: DeviceToolTargetInput, +): Effect.Effect => + Effect.gen(function* () { + yield* requireAccess(caller); + const session = yield* sessionFor(caller.threadId, input.deviceId); + return yield* (yield* DeviceService).screenshot(session.deviceId); + }).pipe(Effect.mapError(toolError)); + +export const deviceClose = (caller: McpInvocationScope, input: DeviceToolCloseInput) => + Effect.gen(function* () { + yield* requireAccess(caller); + const devices = yield* DeviceService; + if (input.deviceId !== undefined) yield* sessionFor(caller.threadId, input.deviceId); + yield* devices.close({ + threadId: caller.threadId, + ...(input.deviceId === undefined ? {} : { deviceId: input.deviceId }), + ...(input.shutdown === undefined ? {} : { shutdown: input.shutdown }), + }); + return {}; + }).pipe(Effect.mapError(toolError)); diff --git a/apps/server/src/mcp/deviceTools.ts b/apps/server/src/mcp/deviceTools.ts new file mode 100644 index 000000000..a1ac01e2c --- /dev/null +++ b/apps/server/src/mcp/deviceTools.ts @@ -0,0 +1,93 @@ +/** + * The device tools (docs/design/devices.md), served at `/mcp/device` by + * McpDeviceServer: list, open, screenshot and close a simulator or emulator + * on this server's machine. Driving the device happens through the + * `agent-device` launcher `device_open` returns, not through more tools. + * + * Descriptions from T3 Code's device toolkit, without its host selection: a + * Threadlines server has one device host, its own machine. + */ +import { + DeviceToolCloseInput, + DeviceToolListResult, + DeviceToolOpenInput, + DeviceToolOpenResult, + DeviceToolScreenshotResult, + DeviceToolTargetInput, +} from "@threadlines/contracts"; +import * as Schema from "effect/Schema"; +import { Tool } from "effect/ai"; + +export const DEVICE_MCP_SERVER_NAME = "threadlines_device"; + +export const DEVICE_LIST_TOOL_NAME = "device_list"; +export const DEVICE_OPEN_TOOL_NAME = "device_open"; +export const DEVICE_SCREENSHOT_TOOL_NAME = "device_screenshot"; +export const DEVICE_CLOSE_TOOL_NAME = "device_close"; + +/** The tools as Claude names them, for its allow list. */ +export const CLAUDE_DEVICE_TOOL_IDS = [ + DEVICE_LIST_TOOL_NAME, + DEVICE_OPEN_TOOL_NAME, + DEVICE_SCREENSHOT_TOOL_NAME, + DEVICE_CLOSE_TOOL_NAME, +].map((name) => `mcp__${DEVICE_MCP_SERVER_NAME}__${name}`); + +// An empty struct serializes as `anyOf [object, array]`, which some providers +// reject and then drop every tool on the server with it. +export const DeviceToolListInput = Schema.Struct({ + refresh: Schema.optional( + Schema.Boolean.annotate({ + description: "Ask the device hub again instead of answering from the last listing.", + }), + ), +}); + +export const DeviceListTool = Tool.make(DEVICE_LIST_TOOL_NAME, { + description: + "List iOS Simulators and Android Emulators on this computer, what it can run, and what is open in this thread's Device tab. Call before `device_open` when you do not know a device id.", + parameters: DeviceToolListInput, + success: DeviceToolListResult, +}) + .annotate(Tool.Title, "List devices") + .annotate(Tool.Readonly, true) + .annotate(Tool.Destructive, false) + .annotate(Tool.Idempotent, true) + .annotate(Tool.OpenWorld, false); + +export const DeviceOpenTool = Tool.make(DEVICE_OPEN_TOOL_NAME, { + description: + "Open a simulator or emulator for this thread. Pass `deviceId` from `device_list`. Boots it if needed and shows it in the user's Device tab. Returns the `agent-device` command bound to that device. Drive the device with that command.", + parameters: DeviceToolOpenInput, + success: DeviceToolOpenResult, +}) + .annotate(Tool.Title, "Open device") + .annotate(Tool.Readonly, false) + .annotate(Tool.Destructive, false) + .annotate(Tool.Idempotent, true) + .annotate(Tool.OpenWorld, true); + +export const DeviceScreenshotTool = Tool.make(DEVICE_SCREENSHOT_TOOL_NAME, { + description: + "Get a PNG of an open device's screen. For taps and text, use the `agent-device` command.", + parameters: DeviceToolTargetInput, + success: DeviceToolScreenshotResult, +}) + .annotate(Tool.Title, "Screenshot device") + .annotate(Tool.Readonly, true) + .annotate(Tool.Destructive, false) + .annotate(Tool.Idempotent, true) + .annotate(Tool.OpenWorld, true); + +export const DeviceCloseTool = Tool.make(DEVICE_CLOSE_TOOL_NAME, { + description: "Remove a device from this thread's Device tab. `shutdown=true` also turns it off.", + parameters: DeviceToolCloseInput, + success: Schema.Struct({}), +}) + .annotate(Tool.Title, "Close device") + .annotate(Tool.Readonly, false) + .annotate(Tool.Destructive, true) + .annotate(Tool.Idempotent, true) + .annotate(Tool.OpenWorld, true); + +export const DEVICE_TOOLS = [DeviceListTool, DeviceOpenTool, DeviceScreenshotTool, DeviceCloseTool]; diff --git a/apps/server/src/mcp/outsideToolLimit.ts b/apps/server/src/mcp/outsideToolLimit.ts index 766055bd8..51209dc7e 100644 --- a/apps/server/src/mcp/outsideToolLimit.ts +++ b/apps/server/src/mcp/outsideToolLimit.ts @@ -51,6 +51,9 @@ export const outsideLimitDetail = (limit: OutsideToolLimit) => export const outsideBrowserDetail = (limit: OutsideToolLimit) => `The browser panel is off for this agent while ${namesText(limit.names)}'s work limits its session. It's back once the agent starts fresh.`; +export const outsideDeviceDetail = (limit: OutsideToolLimit) => + `Devices are off for this agent while ${namesText(limit.names)}'s work limits its session. They're back once the agent starts fresh.`; + /** Whether a read of `projectId` stays inside the limit. */ export const limitReaches = (limit: OutsideToolLimit, projectId: ProjectId) => limit.projectIds === null || limit.projectIds.has(projectId); diff --git a/apps/server/src/mcp/projectNoteToolHandlers.test.ts b/apps/server/src/mcp/projectNoteToolHandlers.test.ts index 5088292ca..b3d61de41 100644 --- a/apps/server/src/mcp/projectNoteToolHandlers.test.ts +++ b/apps/server/src/mcp/projectNoteToolHandlers.test.ts @@ -104,6 +104,7 @@ const caller = (thread: OrchestrationThread, side?: "ask"): McpInvocationScope = browser: true, pages: true, commands: false, + device: false, roomTools: new Set(roomToolsFor(side)), ...(side !== undefined ? { side: { sideTurnId: SideTurnId.make("side-1"), kind: side } } : {}), }); diff --git a/apps/server/src/mcp/roomToolHandlers.test.ts b/apps/server/src/mcp/roomToolHandlers.test.ts index d55da1769..eda0aa0c8 100644 --- a/apps/server/src/mcp/roomToolHandlers.test.ts +++ b/apps/server/src/mcp/roomToolHandlers.test.ts @@ -102,6 +102,7 @@ const mainCaller = ( browser: true, pages: true, commands: false, + device: false, roomTools: new Set(tools), }); diff --git a/apps/server/src/mcp/secretRequestToolHandlers.test.ts b/apps/server/src/mcp/secretRequestToolHandlers.test.ts index d69220576..53f291615 100644 --- a/apps/server/src/mcp/secretRequestToolHandlers.test.ts +++ b/apps/server/src/mcp/secretRequestToolHandlers.test.ts @@ -39,6 +39,7 @@ const mainCaller = (): McpInvocationScope => ({ browser: true, pages: true, commands: false, + device: false, roomTools: new Set(roomToolsFor(undefined)), }); diff --git a/apps/server/src/mcp/settingsChangeToolHandlers.test.ts b/apps/server/src/mcp/settingsChangeToolHandlers.test.ts index 4c1a90e85..a582fac0f 100644 --- a/apps/server/src/mcp/settingsChangeToolHandlers.test.ts +++ b/apps/server/src/mcp/settingsChangeToolHandlers.test.ts @@ -52,6 +52,7 @@ const mainCaller = (): McpInvocationScope => ({ browser: true, pages: true, commands: false, + device: false, roomTools: new Set(roomToolsFor(undefined)), }); diff --git a/apps/server/src/mcp/threadToolHandlers.test.ts b/apps/server/src/mcp/threadToolHandlers.test.ts index eda15706f..2ffe79a58 100644 --- a/apps/server/src/mcp/threadToolHandlers.test.ts +++ b/apps/server/src/mcp/threadToolHandlers.test.ts @@ -194,6 +194,7 @@ const mainCaller = (): McpInvocationScope => ({ browser: true, pages: true, commands: false, + device: false, roomTools: new Set(roomToolsFor(undefined)), }); diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts index 9a6f4b133..8ae4c51c1 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts @@ -1733,6 +1733,14 @@ const make = Effect.gen(function* () { ? { artifacts: true } : {}), ...(toolSettings?.enableCommandRequests !== false ? { commandRequests: true } : {}), + // "Use devices" and "Agents can use devices", both off by default + // (docs/design/devices.md). Drivers leave them off a side runtime, and a + // fenced thread gets none: they act on this machine's simulators. + ...(!fencedOff && + toolSettings?.enableDevices === true && + toolSettings.agentDeviceAccess === true + ? { deviceTools: true } + : {}), runtimeMode: desiredRuntimeMode, }); diff --git a/apps/server/src/pages/AgentPages.test.ts b/apps/server/src/pages/AgentPages.test.ts index 6ef53377f..05d5b9383 100644 --- a/apps/server/src/pages/AgentPages.test.ts +++ b/apps/server/src/pages/AgentPages.test.ts @@ -53,6 +53,7 @@ const caller = (overrides: Partial = {}): McpInvocationScope browser: true, pages: true, commands: false, + device: false, roomTools: new Set(), ...overrides, }); diff --git a/apps/server/src/preview/LocalServers.ts b/apps/server/src/preview/LocalServers.ts index 8fdf65f36..4614bc744 100644 --- a/apps/server/src/preview/LocalServers.ts +++ b/apps/server/src/preview/LocalServers.ts @@ -9,15 +9,18 @@ */ import type { PreviewLocalServer } from "@threadlines/contracts"; -import { execFile } from "node:child_process"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; -import { parseListeningPorts, type ListeningPort } from "./parseListeningPorts.ts"; +import { findDeviceToolProcessIds } from "./deviceToolPorts.ts"; +import { + parseListeningPorts, + readListeningSockets, + type ListeningPort, +} from "./parseListeningPorts.ts"; import { probeHttpServer } from "./probeHttpServer.ts"; -const LSOF_TIMEOUT_MS = 5_000; /** * How long a probe result stands. Long enough that a stable server is not * re-fetched on every poll, short enough that restarting one on the same port @@ -68,26 +71,19 @@ export const make = Effect.gen(function* LocalServersMake() { scan: Effect.fn("LocalServers.scan")(function* () { // Never fails the caller: an empty list renders as "nothing running", // which is the truth whether lsof is missing or genuinely found nothing. - return yield* Effect.promise( - () => - new Promise>((resolve) => { - if (process.platform === "win32") { - resolve([]); - return; - } - execFile( - "lsof", - ["-iTCP", "-sTCP:LISTEN", "-P", "-n", "-F", "pcn"], - { timeout: LSOF_TIMEOUT_MS, maxBuffer: 4_000_000 }, - (_error, stdout) => { - // lsof exits non-zero when some descriptors are unreadable - // while still printing the ones it could read, so stdout is - // used even on error rather than discarding a good result. - void probeAll(parseListeningPorts(stdout ?? "")).then(resolve); - }, - ); - }), - ); + return yield* Effect.promise(async () => { + const [output, devicePids] = await Promise.all([ + readListeningSockets(), + findDeviceToolProcessIds(), + ]); + // Device tools' ports are never offered: the tunnel refuses them. When + // they can't be told apart the list keeps everything, and the tunnel + // still refuses. + const listeners = parseListeningPorts(output ?? "").filter( + (entry) => devicePids?.has(entry.pid) !== true, + ); + return await probeAll(listeners); + }); }), }); }).pipe(Effect.withSpan("LocalServers.make")); diff --git a/apps/server/src/preview/PreviewTunnel.test.ts b/apps/server/src/preview/PreviewTunnel.test.ts index 60009a9b7..dfc27813f 100644 --- a/apps/server/src/preview/PreviewTunnel.test.ts +++ b/apps/server/src/preview/PreviewTunnel.test.ts @@ -3,8 +3,13 @@ import { PREVIEW_TUNNEL_WINDOW_BYTES, type PreviewTunnelEvent } from "@threadlin import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Stream from "effect/Stream"; +import { spawn } from "node:child_process"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import * as Net from "node:net"; +import { tmpdir } from "node:os"; +import * as Path from "node:path"; +import { setOwnDeviceToolPort } from "./deviceToolPorts.ts"; import { loopbackAddresses, makePreviewTunnel } from "./PreviewTunnel.ts"; const sessionId = "session-a"; @@ -46,6 +51,67 @@ const pullEvents = (stream: Stream.Stream) => }); }); +/** + * A stand-in device hub: a real process run from the hub's entry-point path, + * listening on one port, with a child of its own (an emulator, say) listening + * on another. Both exit when their stdin closes, so neither outlives the test. + */ +const HUB_CHILD = ` +const server = require("node:net").createServer(); +server.listen(0, "127.0.0.1", () => process.stdout.write(server.address().port + "\\n")); +process.stdin.on("end", () => process.exit(0)); +process.stdin.resume(); +`; +const HUB_SCRIPT = ` +import { spawn } from "node:child_process"; +import * as Net from "node:net"; +const server = Net.createServer(); +await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); +const child = spawn(process.execPath, ["-e", ${JSON.stringify(HUB_CHILD)}], { + stdio: ["pipe", "pipe", "inherit"], +}); +let heard = ""; +child.stdout.on("data", (chunk) => { + heard += chunk; + if (heard.includes("\\n")) { + const ports = { hub: server.address().port, child: Number.parseInt(heard, 10) }; + process.stdout.write(JSON.stringify(ports) + "\\n"); + } +}); +process.stdin.on("end", () => process.exit(0)); +process.stdin.resume(); +`; + +const startStandInHub = Effect.acquireRelease( + Effect.promise(async () => { + const root = await mkdtemp(Path.join(tmpdir(), "preview-tunnel-hub-")); + const entry = Path.join(root, "node_modules/expo-device-hub/dist/server/cli.mjs"); + await mkdir(Path.dirname(entry), { recursive: true }); + await writeFile(entry, HUB_SCRIPT); + const hub = spawn(process.execPath, [entry, "--port", "0"], { + stdio: ["pipe", "pipe", "inherit"], + }); + const ports = await new Promise<{ hub: number; child: number }>((resolve, reject) => { + let heard = ""; + hub.stdout.on("data", (chunk) => { + heard += chunk; + if (heard.includes("\n")) resolve(JSON.parse(heard)); + }); + hub.once("exit", (code) => reject(new Error(`The stand-in hub exited (${code}).`))); + }); + return { root, hub, ports }; + }), + ({ root, hub }) => + Effect.promise(async () => { + // Its child exits on its own once the hub's end of the pipe closes. + hub.stdin.end(); + if (hub.exitCode === null) { + await new Promise((resolve) => hub.once("exit", resolve)); + } + await rm(root, { recursive: true, force: true }); + }), +); + const decode = (event: PreviewTunnelEvent) => event._tag === "data" ? Buffer.from(event.data, "base64").toString("utf8") : `<${event._tag}>`; @@ -200,6 +266,65 @@ describe("PreviewTunnel", () => { ), ); + it.live.skipIf(process.platform === "win32")( + "refuses a device tool's ports and its children's, and opens a dev server's", + () => + Effect.scoped( + Effect.gen(function* () { + const { ports } = yield* startStandInHub; + const { port: devServer } = yield* listen(() => {}); + const tunnels = yield* makePreviewTunnel({ ownPort: -1 }); + + for (const [tunnelId, port] of [ + ["hub", ports.hub], + ["child", ports.child], + ] as const) { + const refused = yield* tunnels + .open(sessionId, { tunnelId, host: "localhost", port }) + .pipe(Effect.flip); + assert.strictEqual( + refused.detail, + "That port belongs to device support and can't be previewed.", + tunnelId, + ); + } + + const events = yield* tunnels.open(sessionId, { + tunnelId: "dev", + host: "localhost", + port: devServer, + }); + const next = yield* pullEvents(events); + assert.strictEqual((yield* next)._tag, "opened"); + }), + ), + ); + + it.live("refuses the ports this server's own device tools hold, where no scan finds them", () => + Effect.scoped( + Effect.gen(function* () { + // A plain listener: nothing about its process marks it as a device tool. + const { port } = yield* listen(() => {}); + const tunnels = yield* makePreviewTunnel({ ownPort: -1 }); + setOwnDeviceToolPort("hub", port); + yield* Effect.addFinalizer(() => Effect.sync(() => setOwnDeviceToolPort("hub", null))); + const refused = yield* tunnels + .open(sessionId, { tunnelId: "own-hub", host: "localhost", port }) + .pipe(Effect.flip); + assert.strictEqual( + refused.detail, + "That port belongs to device support and can't be previewed.", + ); + // A replaced hub stopping late does not forget its successor's port. + setOwnDeviceToolPort("hub", null, port + 1); + const still = yield* tunnels + .open(sessionId, { tunnelId: "own-hub-2", host: "localhost", port }) + .pipe(Effect.flip); + assert.strictEqual(still.reason, "notLoopback"); + }), + ), + ); + it.live("says plainly when nothing is listening", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/preview/PreviewTunnel.ts b/apps/server/src/preview/PreviewTunnel.ts index 05c0a3577..7e34ab428 100644 --- a/apps/server/src/preview/PreviewTunnel.ts +++ b/apps/server/src/preview/PreviewTunnel.ts @@ -8,7 +8,9 @@ * in contracts) and this opens it on our own loopback and passes the bytes. * * Every part is about staying a pipe and nothing more: only loopback, so no - * client gets a way into the network this machine sits on; bytes in the order + * client gets a way into the network this machine sits on; never a device + * tool's port, which takes actions from anyone on loopback (see + * `deviceToolPorts.ts`); bytes in the order * the page wrote them, however the calls carrying them were scheduled; and no * more in flight than the page is reading, so a stalled page holds the dev * server up instead of filling this process with its output. @@ -34,6 +36,7 @@ import * as Stream from "effect/Stream"; import * as Net from "node:net"; import { ServerConfig } from "../config.ts"; +import { currentDeviceToolPorts } from "./deviceToolPorts.ts"; /** Tunnels one session may hold open at once; a busy page uses a few dozen. */ export const PREVIEW_TUNNEL_MAX_PER_SESSION = 256; @@ -115,7 +118,8 @@ export interface PreviewTunnelShape { /** * Opens one connection and hands back what it sends. Scoped: the scope * closing -- the client unsubscribing, its socket dropping -- destroys it. - * Fails when the host is not loopback or nothing answers there. + * Fails when the host is not loopback, the port is a device tool's, or + * nothing answers there. */ readonly open: ( sessionId: string, @@ -300,6 +304,19 @@ const makeTunnels = (options: { readonly ownPort: number }): PreviewTunnelShape }), ); + // Asked afresh for every connection: a port that served a dev server a + // moment ago may be the device hub's now. + const devicePorts = yield* Effect.promise(() => currentDeviceToolPorts()); + if (devicePorts === null || devicePorts.has(input.port)) { + return yield* new PreviewTunnelError({ + reason: "notLoopback", + detail: + devicePorts === null + ? "Threadlines couldn't check what is listening on that port, so it didn't open it. Try again." + : "That port belongs to device support and can't be previewed.", + }); + } + let lastError: unknown = null; let socket: Net.Socket | null = null; for (const address of addresses) { diff --git a/apps/server/src/preview/deviceToolPorts.ts b/apps/server/src/preview/deviceToolPorts.ts new file mode 100644 index 000000000..9dc727621 --- /dev/null +++ b/apps/server/src/preview/deviceToolPorts.ts @@ -0,0 +1,159 @@ +/** + * Which listeners on this machine belong to device tools, so a preview never + * reaches one. + * + * The device hub (and serve-sim/serve-emu inside it) and the agent-device + * daemon listen on loopback and accept actions there: touch input, app + * installs, shell commands. A client must reach them only through the device + * channel, never through the loopback tunnel (docs/design/devices.md, "The hub + * is never reachable from a client"). That holds for every copy on this + * machine, whoever started it -- this server, another Threadlines server, an + * older release -- so they are found by command line rather than from what + * this server spawned. Matching by name is for refusing only; nothing here + * ever signals a process it found this way. + * + * A process is a device tool's when its own command line matches, or one of + * its ancestors' does: an emulator the hub boots is the hub's child and + * listens on ports of its own. + */ +import { listProcessTreesAsync } from "@threadlines/shared/processTree"; +import { execFile } from "node:child_process"; + +import { parseListenerPortNumbers, readListeningSockets } from "./parseListeningPorts.ts"; + +const SCAN_TIMEOUT_MS = 5_000; + +/** + * The command lines of device tools, as an extended regular expression for + * `pgrep -f`: the hub's entry point, the agent-device daemon's, and + * serve-sim/serve-emu run on their own (their vendored entry points, or a bin + * by that name) in case a hub version spawns them as helpers. + */ +export const DEVICE_TOOL_COMMAND_PATTERN = [ + "expo-device-hub/dist/server/cli\\.mjs", + "agent-device/dist/src/internal/daemon\\.js", + "vendor/serve-(sim|emu)/", + "(^|[/ ])serve-(sim|emu)( |$)", +].join("|"); + +/** + * Pids whose command line matches DEVICE_TOOL_COMMAND_PATTERN. Empty when + * none does, or when this machine has no `pgrep` (a slim container, where no + * device tool runs either); null when the search failed or ran out of time. + */ +function matchingProcessIds(): Promise { + return new Promise((resolve) => { + execFile( + "pgrep", + ["-f", DEVICE_TOOL_COMMAND_PATTERN], + { encoding: "utf8", timeout: SCAN_TIMEOUT_MS }, + (error, stdout) => { + const code = (error as { code?: unknown } | null)?.code; + if (error !== null && error.killed !== true && (code === 1 || code === "ENOENT")) { + // 1: nothing matched. + resolve([]); + return; + } + if (error !== null) { + resolve(null); + return; + } + resolve( + stdout + .split("\n") + .map((line) => Number.parseInt(line.trim(), 10)) + .filter((pid) => Number.isInteger(pid) && pid > 0), + ); + }, + ); + }); +} + +/** + * Every process that belongs to a device tool: the matches and all their + * descendants. Null when that can't be known. Empty on Windows, where this + * check does not run. + */ +export async function findDeviceToolProcessIds(): Promise | null> { + if (process.platform === "win32") return new Set(); + const roots = await matchingProcessIds(); + if (roots === null) return null; + if (roots.length === 0) return new Set(); + const tree = await listProcessTreesAsync(roots, SCAN_TIMEOUT_MS); + return tree === null ? null : new Set(tree); +} + +/** Every port a device tool listens on, at any address. Null when that can't be known. */ +async function scanDeviceToolPorts(): Promise | null> { + const pids = await findDeviceToolProcessIds(); + if (pids === null) return null; + if (pids.size === 0) return new Set(); + const output = await readListeningSockets({ pids: [...pids] }); + return output === null ? null : new Set(parseListenerPortNumbers(output)); +} + +/** + * Runs `scan` for callers that each need an answer from after they asked. + * A caller arriving mid-scan waits for the next one, which every caller + * arriving meanwhile shares: a page opening dozens of connections at once + * costs two scans, not dozens, and none is answered from before it asked. + */ +function freshScans(scan: () => Promise): () => Promise { + let running: Promise | null = null; + let queued: Promise | null = null; + const start = (): Promise => { + const current: Promise = scan().finally(() => { + if (running === current) running = null; + }); + running = current; + return current; + }; + return () => { + if (running === null) return start(); + if (queued === null) { + const next = running.then( + () => undefined, + () => undefined, + ); + queued = next.then(() => { + queued = null; + return start(); + }); + } + return queued; + }; +} + +/** + * Ports this process's own device tools listen on, by tool, as the device host + * starts and stops them. Refused on every platform, including where the scan + * above cannot run (Windows, a machine without `pgrep`). + */ +const ownToolPorts = new Map(); + +/** + * The device host records the port one of its tools listens on, or with null + * forgets it. `onlyIf` forgets only while that port is still the recorded one, + * so a replaced hub stopping after its successor started leaves the new port. + */ +export function setOwnDeviceToolPort( + tool: "hub" | "agent", + port: number | null, + onlyIf?: number, +): void { + if (port !== null) ownToolPorts.set(tool, port); + else if (onlyIf === undefined || ownToolPorts.get(tool) === onlyIf) ownToolPorts.delete(tool); +} + +const scanOnce = freshScans(scanDeviceToolPorts); + +/** + * The ports device tools listen on right now: this server's own, plus a scan + * begun after the call. Never cached: a port that was a dev server's a moment + * ago may be the hub's now. Null when the scan can't tell, which callers treat + * as a refusal. + */ +export const currentDeviceToolPorts = async (): Promise | null> => { + const scanned = await scanOnce(); + return scanned === null ? null : new Set([...scanned, ...ownToolPorts.values()]); +}; diff --git a/apps/server/src/preview/parseListeningPorts.ts b/apps/server/src/preview/parseListeningPorts.ts index 2556dfb9f..86ba7aa70 100644 --- a/apps/server/src/preview/parseListeningPorts.ts +++ b/apps/server/src/preview/parseListeningPorts.ts @@ -1,5 +1,5 @@ /** - * Parses `lsof -iTCP -sTCP:LISTEN -P -n -F pcn`. + * Lists and parses `lsof -iTCP -sTCP:LISTEN -P -n -F pcn`. * * `-F` emits one field per line behind a single-character tag, which is the * only lsof output worth depending on: the human-readable table changes shape @@ -7,6 +7,9 @@ * (command) describe the process, and every `n` (address) line after them * belongs to that process until the next `p`. */ +import { execFile } from "node:child_process"; + +const LSOF_TIMEOUT_MS = 5_000; export interface ListeningPort { port: number; @@ -40,6 +43,18 @@ function loopbackProbeHost(address: string): "127.0.0.1" | "::1" | null { } } +/** + * The port of an lsof `n` address, or null when it is not a listener's. IPv6 + * arrives bracketed, and lsof writes "->" for established connections; a + * listener has no peer. + */ +function listenerPort(address: string): number | null { + if (address.includes("->")) return null; + const port = Number.parseInt(address.slice(address.lastIndexOf(":") + 1), 10); + return Number.isNaN(port) || port <= 0 ? null : port; +} + +/** Listeners this machine's browser can reach, one per port, sorted by port. */ export function parseListeningPorts(lsofOutput: string): ListeningPort[] { const byPort = new Map(); let pid: number | null = null; @@ -63,14 +78,9 @@ export function parseListeningPorts(lsofOutput: string): ListeningPort[] { if (tag !== "n" || pid === null) { continue; } - // IPv6 arrives bracketed, and lsof writes "->" for established - // connections; a listener has no peer. - const probeHost = value.includes("->") ? null : loopbackProbeHost(value); - if (probeHost === null) { - continue; - } - const port = Number.parseInt(value.slice(value.lastIndexOf(":") + 1), 10); - if (Number.isNaN(port) || port <= 0) { + const port = listenerPort(value); + const probeHost = port === null ? null : loopbackProbeHost(value); + if (port === null || probeHost === null) { continue; } // The same port often appears twice, once per address family: one entry, @@ -85,3 +95,59 @@ export function parseListeningPorts(lsofOutput: string): ListeningPort[] { return [...byPort.values()].sort((a, b) => a.port - b.port); } + +/** + * Every port listened on in lsof output, at any address. For refusing + * ports: a listener bound only to 127.0.0.2 is not offered as a server, but a + * tunnel can still reach it. + */ +export function parseListenerPortNumbers(lsofOutput: string): number[] { + const ports = new Set(); + for (const line of lsofOutput.split("\n")) { + if (line[0] !== "n") continue; + const port = listenerPort(line.slice(1)); + if (port !== null) ports.add(port); + } + return [...ports].sort((a, b) => a - b); +} + +/** + * Runs `lsof -iTCP -sTCP:LISTEN -P -n -F pcn`, for every process or only + * `pids`, and hands back what it printed. Null when lsof is missing or ran out + * of time; empty on Windows, which has no lsof. + */ +export function readListeningSockets(options?: { + readonly pids?: ReadonlyArray; +}): Promise { + if (process.platform === "win32") return Promise.resolve(""); + const pids = options?.pids; + if (pids !== undefined && pids.length === 0) return Promise.resolve(""); + const args = [ + ...(pids === undefined ? [] : ["-a", "-p", pids.join(",")]), + "-iTCP", + "-sTCP:LISTEN", + "-P", + "-n", + "-F", + "pcn", + ]; + return new Promise((resolve) => { + execFile( + "lsof", + args, + { encoding: "utf8", timeout: LSOF_TIMEOUT_MS, maxBuffer: 4_000_000 }, + (error, stdout) => { + // lsof exits 1 when some descriptors are unreadable, or some of the + // given pids hold no listener, while still printing the ones it found, + // so its output stands then. Not when it never ran or was cut off: + // half a list read as a whole one would leave listeners out. + const exitedWithCode = typeof (error as { code?: unknown } | null)?.code === "number"; + if (error !== null && (!exitedWithCode || error.killed === true)) { + resolve(null); + return; + } + resolve(stdout); + }, + ); + }); +} diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.ts b/apps/server/src/provider/Layers/ClaudeAdapter.ts index 89d4ece63..3357298d5 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.ts @@ -159,6 +159,7 @@ import { ServerConfig } from "../../config.ts"; import { BROWSER_MCP_SERVER_NAME, mcpCommandsEndpointUrl, + mcpDeviceEndpointUrl, mcpEndpointUrl, mcpPagesEndpointUrl, mcpRoomEndpointUrl, @@ -168,6 +169,8 @@ import { CLAUDE_COMMAND_TOOL_IDS, COMMANDS_MCP_SERVER_NAME, } from "../../mcp/commandTools.ts"; +import { CLAUDE_DEVICE_TOOL_IDS, DEVICE_MCP_SERVER_NAME } from "../../mcp/deviceTools.ts"; +import { buildDeviceInstructions } from "../deviceInstructions.ts"; import { mcpSessionRegistry } from "../../mcp/McpSessionRegistry.ts"; import { outsideWorkLookup } from "../outsideWorkLookup.ts"; import { ROOM_TOOL_CALL_TIMEOUT } from "../../mcp/roomToolTimeouts.ts"; @@ -1008,6 +1011,8 @@ export function buildClaudeSystemPromptAppend(input: { readonly pageAssetsDir: string | undefined; readonly artifacts: boolean; readonly commandRequests: boolean; + /** The device tools (ProviderSessionStartInput.deviceTools). */ + readonly deviceTools: boolean; /** The room endpoint's tools this session may call, when it is attached. */ readonly room: | { readonly tools: ReadonlyArray; readonly schedules: boolean } @@ -1031,6 +1036,7 @@ export function buildClaudeSystemPromptAppend(input: { : []), ...(input.artifacts ? [CLAUDE_ARTIFACT_INSTRUCTIONS] : []), ...(input.commandRequests ? [buildCommandRequestInstructions({})] : []), + ...(input.deviceTools ? [buildDeviceInstructions()] : []), ...(input.room?.schedules === true ? [buildScheduleInstructions({ toolPrefix: roomPrefix, claude: true })] : []), @@ -9613,6 +9619,8 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( ); // `ask_user_to_run`: a command card the user runs from the chat. const commandRequests = input.commandRequests === true && !lockdown; + // The device tools: simulators and emulators on this computer. + const deviceTools = input.deviceTools === true && !lockdown; const credential = !lockdown || roomGrant.attach ? yield* mcpSessionRegistry.credentialFor({ @@ -9621,6 +9629,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( room: roomGrant.attach, pages: agentPages, commands: commandRequests, + device: deviceTools, sideKind, scheduledRun: input.scheduledRunTools === true, guideOnly: roomGrant.guideOnly, @@ -9687,6 +9696,19 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( }, } : {}), + ...(credential !== undefined && deviceTools + ? { + [DEVICE_MCP_SERVER_NAME]: { + type: "http" as const, + url: mcpDeviceEndpointUrl(serverConfig.port), + headers: { Authorization: `Bearer ${credential.token}` }, + // Like the others: Claude Code waits for the server before + // the first turn, and each tool's own mark decides whether + // it is in the prompt (toolLoading.ts). + alwaysLoad: true, + }, + } + : {}), }, ...(apiModelId ? { model: apiModelId } : {}), pathToClaudeCodeExecutable: claudeBinaryPath, @@ -9698,6 +9720,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( pageAssetsDir: agentPages ? pageAssets : undefined, artifacts, commandRequests: credential !== undefined && commandRequests, + deviceTools: credential !== undefined && deviceTools, room: roomServer !== undefined ? { @@ -9739,6 +9762,9 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( ...(credential !== undefined && agentPages ? CLAUDE_PAGE_TOOL_IDS : []), // Asking the user to run a command only shows a card; they decide. ...(credential !== undefined && commandRequests ? CLAUDE_COMMAND_TOOL_IDS : []), + // The user turned on "Agents can use devices"; driving one goes + // through the agent-device command, which asks like any command. + ...(credential !== undefined && deviceTools ? CLAUDE_DEVICE_TOOL_IDS : []), ], ...(effectiveEffort ? { diff --git a/apps/server/src/provider/Layers/CodexAdapter.ts b/apps/server/src/provider/Layers/CodexAdapter.ts index f5b587833..1db5ccff1 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.ts @@ -3437,6 +3437,7 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( ...(input.commandRequests === true && sideAnswer === undefined ? { commandRequests: true } : {}), + ...(input.deviceTools === true && sideAnswer === undefined ? { deviceTools: true } : {}), runtimeMode: input.runtimeMode, ...(input.modelSelection?.instanceId === boundInstanceId ? { model: input.modelSelection.model } diff --git a/apps/server/src/provider/Layers/CodexSessionRuntime.ts b/apps/server/src/provider/Layers/CodexSessionRuntime.ts index cf86efda2..237e2f35d 100644 --- a/apps/server/src/provider/Layers/CodexSessionRuntime.ts +++ b/apps/server/src/provider/Layers/CodexSessionRuntime.ts @@ -2,6 +2,7 @@ import { parseSessionKey } from "@threadlines/shared/threadParticipants"; import { BROWSER_MCP_SERVER_NAME, mcpCommandsEndpointUrl, + mcpDeviceEndpointUrl, mcpEndpointUrl, mcpPagesEndpointUrl, mcpRoomEndpointUrl, @@ -10,6 +11,8 @@ import { buildCommandRequestInstructions, COMMANDS_MCP_SERVER_NAME, } from "../../mcp/commandTools.ts"; +import { DEVICE_MCP_SERVER_NAME } from "../../mcp/deviceTools.ts"; +import { buildDeviceInstructions } from "../deviceInstructions.ts"; import { buildAgentPageInstructions, CODEX_PAGE_RIVAL, @@ -255,6 +258,8 @@ export interface CodexSessionRuntimeOptions { readonly agentPages?: boolean; /** Attach `ask_user_to_run` (ProviderSessionStartInput.commandRequests). Never for a side runtime. */ readonly commandRequests?: boolean; + /** Attach the device tools (ProviderSessionStartInput.deviceTools). Never for a side runtime. */ + readonly deviceTools?: boolean; /** * The project's brief (ProviderSessionStartInput.projectInstructions). The * provider thread keeps it as its developer instructions, set at every @@ -748,6 +753,8 @@ function buildCodexCollaborationMode(input: { readonly agentPageAssetsDir?: string; /** The session has `ask_user_to_run`. */ readonly commandRequests?: boolean; + /** The session has the device tools. */ + readonly deviceTools?: boolean; /** The thread's own runtime has the room tools, schedule tools among them. */ readonly scheduleTools?: boolean; /** The thread's own runtime has the room endpoint, so the app guide. */ @@ -779,6 +786,7 @@ function buildCodexCollaborationMode(input: { ] : []), ...(input.commandRequests ? [buildCommandRequestInstructions({})] : []), + ...(input.deviceTools ? [buildDeviceInstructions()] : []), ...(input.scheduleTools ? [buildScheduleInstructions({ toolPrefix: `mcp__${ROOM_MCP_SERVER_NAME}__` })] : []), @@ -809,6 +817,8 @@ export function buildTurnStartParams(input: { readonly agentPageAssetsDir?: string; /** The session has `ask_user_to_run`. */ readonly commandRequests?: boolean; + /** The session has the device tools. */ + readonly deviceTools?: boolean; /** The thread's own runtime has the room tools, schedule tools among them. */ readonly scheduleTools?: boolean; /** The thread's own runtime has the room endpoint, so the app guide. */ @@ -845,6 +855,7 @@ export function buildTurnStartParams(input: { ? { agentPageAssetsDir: input.agentPageAssetsDir } : {}), ...(input.commandRequests ? { commandRequests: true } : {}), + ...(input.deviceTools ? { deviceTools: true } : {}), ...(input.scheduleTools && !input.lockdown ? { scheduleTools: true } : {}), ...(input.guide && !input.lockdown ? { guide: true } : {}), }); @@ -2104,6 +2115,7 @@ const makeCodexSessionRuntimeIn = ( const roomGrant = roomEndpointGrant(options, lockdown?.kind); const agentPages = options.agentPages === true && lockdown === undefined; const commandRequests = options.commandRequests === true && lockdown === undefined; + const deviceTools = options.deviceTools === true && lockdown === undefined; const credential = lockdown === undefined || roomGrant.attach ? yield* mcpSessionRegistry.credentialFor({ @@ -2112,6 +2124,7 @@ const makeCodexSessionRuntimeIn = ( room: roomGrant.attach, pages: agentPages, commands: commandRequests, + device: deviceTools, ...(lockdown !== undefined ? { sideKind: lockdown.kind } : {}), scheduledRun: options.scheduledRunTools === true, guideOnly: roomGrant.guideOnly, @@ -2162,6 +2175,14 @@ const makeCodexSessionRuntimeIn = ( }, } : {}), + ...(deviceTools + ? { + device: { + url: mcpDeviceEndpointUrl(options.serverPort), + serverName: DEVICE_MCP_SERVER_NAME, + }, + } + : {}), }), env, ); @@ -3123,6 +3144,7 @@ const makeCodexSessionRuntimeIn = ( ...((yield* runsInManagedWorktree) ? { managedWorktree: true } : {}), ...(agentPageAssetsDir !== undefined ? { agentPageAssetsDir } : {}), ...(commandRequests ? { commandRequests: true } : {}), + ...(deviceTools ? { deviceTools: true } : {}), ...(roomTools && options.scheduleTools === true && lockdown === undefined ? { scheduleTools: true } : {}), diff --git a/apps/server/src/provider/Layers/MuseAdapter.ts b/apps/server/src/provider/Layers/MuseAdapter.ts index a3fa55c39..921f07b68 100644 --- a/apps/server/src/provider/Layers/MuseAdapter.ts +++ b/apps/server/src/provider/Layers/MuseAdapter.ts @@ -84,6 +84,7 @@ import { ServerConfig } from "../../config.ts"; import { BROWSER_MCP_SERVER_NAME, mcpCommandsEndpointUrl, + mcpDeviceEndpointUrl, mcpEndpointUrl, mcpPagesEndpointUrl, mcpRoomEndpointUrl, @@ -92,6 +93,8 @@ import { buildCommandRequestInstructions, COMMANDS_MCP_SERVER_NAME, } from "../../mcp/commandTools.ts"; +import { DEVICE_MCP_SERVER_NAME } from "../../mcp/deviceTools.ts"; +import { buildDeviceInstructions } from "../deviceInstructions.ts"; import { mcpSessionRegistry } from "../../mcp/McpSessionRegistry.ts"; import { buildAgentPageInstructions, PAGES_MCP_SERVER_NAME } from "../../mcp/pageTools.ts"; import { buildGuideInstructions } from "../../mcp/guideTools.ts"; @@ -314,6 +317,7 @@ interface SessionContext { readonly scheduleTools: boolean; readonly agentPageAssetsDir: string | undefined; readonly commandRequests: boolean; + readonly deviceTools: boolean; readonly managedWorktree: boolean; readonly mcpToken: string; museSessionId: string; @@ -1800,6 +1804,9 @@ export function makeMuseAdapter(options: MuseAdapterOptions) { ...(ctx.commandRequests ? { [COMMANDS_MCP_SERVER_NAME]: server(mcpCommandsEndpointUrl(serverConfig.port)) } : {}), + ...(ctx.deviceTools + ? { [DEVICE_MCP_SERVER_NAME]: server(mcpDeviceEndpointUrl(serverConfig.port)) } + : {}), }, }; }; @@ -1812,6 +1819,7 @@ export function makeMuseAdapter(options: MuseAdapterOptions) { : []), // Muse's own names for outside tools are not ours to guess; the bare name is unambiguous. ...(ctx.commandRequests ? [buildCommandRequestInstructions({ toolPrefix: "" })] : []), + ...(ctx.deviceTools ? [buildDeviceInstructions("")] : []), ...(ctx.scheduleTools ? [buildScheduleInstructions({ toolPrefix: "" })] : []), ...(ctx.roomEndpoint ? [buildGuideInstructions({ toolPrefix: "" })] : []), FILE_LINK_INSTRUCTIONS, @@ -2212,6 +2220,7 @@ export function makeMuseAdapter(options: MuseAdapterOptions) { const roomTools = input.roomTools === true; const roomGrant = roomEndpointGrant(input, undefined); const commandRequests = input.commandRequests === true; + const deviceTools = input.deviceTools === true; const agentPageAssetsDir = input.agentPages === true ? yield* ensurePageAssetsDir(parseSessionKey(input.threadId).threadId).pipe( @@ -2225,6 +2234,7 @@ export function makeMuseAdapter(options: MuseAdapterOptions) { room: roomGrant.attach, pages: agentPageAssetsDir !== undefined, commands: commandRequests, + device: deviceTools, scheduledRun: input.scheduledRunTools === true, guideOnly: roomGrant.guideOnly, }); @@ -2260,6 +2270,7 @@ export function makeMuseAdapter(options: MuseAdapterOptions) { scheduleTools: roomTools && input.scheduleTools === true, agentPageAssetsDir, commandRequests, + deviceTools, managedWorktree, mcpToken: credential.token, museSessionId: "", diff --git a/apps/server/src/provider/Layers/OpenCodeAdapter.ts b/apps/server/src/provider/Layers/OpenCodeAdapter.ts index 4b4043907..dd5c13243 100644 --- a/apps/server/src/provider/Layers/OpenCodeAdapter.ts +++ b/apps/server/src/provider/Layers/OpenCodeAdapter.ts @@ -74,11 +74,13 @@ import { ServerConfig } from "../../config.ts"; import { mcpEndpointUrl, mcpCommandsEndpointUrl, + mcpDeviceEndpointUrl, mcpPagesEndpointUrl, mcpRoomEndpointUrl, } from "../../mcp/McpHttpServer.ts"; import { buildAgentPageInstructions } from "../../mcp/pageTools.ts"; import { buildCommandRequestInstructions } from "../../mcp/commandTools.ts"; +import { buildDeviceInstructions } from "../deviceInstructions.ts"; import { buildGuideInstructions } from "../../mcp/guideTools.ts"; import { roomEndpointGrant } from "../../mcp/roomToolAccess.ts"; import { buildScheduleInstructions } from "../../mcp/scheduleTools.ts"; @@ -123,6 +125,7 @@ import { openCodeBrowserServerName, openCodeRoomServerName, openCodeCommandsServerName, + openCodeDeviceServerName, openCodePagesServerName, openCodeSessionGrantRules, openCodeSessionRules, @@ -260,6 +263,8 @@ interface SessionContext { readonly agentPageAssetsDir: string | undefined; /** The session has `ask_user_to_run`. */ readonly commandRequests: boolean; + /** The session has the device tools. */ + readonly deviceTools: boolean; /** The project's brief (ProviderSessionStartInput.projectInstructions), last in the instructions entry. */ readonly projectInstructions: string | undefined; readonly runtimeMode: RuntimeMode; @@ -600,6 +605,7 @@ export function makeOpenCodeAdapter(options: OpenCodeAdapterOptions) { roomTools: ctx.roomEndpoint, agentPages: ctx.agentPageAssetsDir !== undefined, commandRequests: ctx.commandRequests, + deviceTools: ctx.deviceTools, }); }); @@ -1993,6 +1999,14 @@ export function makeOpenCodeAdapter(options: OpenCodeAdapterOptions) { }, ] : []), + ...(ctx.deviceTools + ? [ + { + name: openCodeDeviceServerName(ctx.toolKey), + url: mcpDeviceEndpointUrl(serverConfig.port), + }, + ] + : []), ]; for (const server of servers) { yield* runOpenCode("mcp.add", (signal) => @@ -2044,6 +2058,9 @@ export function makeOpenCodeAdapter(options: OpenCodeAdapterOptions) { }), ] : []), + ...(ctx.deviceTools + ? [buildDeviceInstructions(`${openCodeDeviceServerName(ctx.toolKey)}_`)] + : []), ...(ctx.scheduleTools ? [buildScheduleInstructions({ toolPrefix: `${openCodeRoomServerName(ctx.toolKey)}_` })] : []), @@ -2294,6 +2311,7 @@ export function makeOpenCodeAdapter(options: OpenCodeAdapterOptions) { scheduleTools: roomTools && input.scheduleTools === true, agentPageAssetsDir, commandRequests: input.commandRequests === true, + deviceTools: input.deviceTools === true, projectInstructions: input.projectInstructions, runtimeMode: input.runtimeMode, agent: BUILD_AGENT, @@ -2335,6 +2353,7 @@ export function makeOpenCodeAdapter(options: OpenCodeAdapterOptions) { room: roomGrant.attach, pages: agentPageAssetsDir !== undefined, commands: input.commandRequests === true, + device: input.deviceTools === true, scheduledRun: input.scheduledRunTools === true, guideOnly: roomGrant.guideOnly, // OpenCode registers tool servers per folder: every session in diff --git a/apps/server/src/provider/Layers/PiAdapter.ts b/apps/server/src/provider/Layers/PiAdapter.ts index 8090cab0e..b390afaf7 100644 --- a/apps/server/src/provider/Layers/PiAdapter.ts +++ b/apps/server/src/provider/Layers/PiAdapter.ts @@ -78,6 +78,7 @@ import { ServerConfig } from "../../config.ts"; import { BROWSER_MCP_SERVER_NAME, mcpCommandsEndpointUrl, + mcpDeviceEndpointUrl, mcpEndpointUrl, mcpPagesEndpointUrl, mcpRoomEndpointUrl, @@ -86,6 +87,8 @@ import { buildCommandRequestInstructions, COMMANDS_MCP_SERVER_NAME, } from "../../mcp/commandTools.ts"; +import { DEVICE_MCP_SERVER_NAME } from "../../mcp/deviceTools.ts"; +import { buildDeviceInstructions } from "../deviceInstructions.ts"; import { mcpSessionRegistry } from "../../mcp/McpSessionRegistry.ts"; import { buildAgentPageInstructions, PAGES_MCP_SERVER_NAME } from "../../mcp/pageTools.ts"; import { ROOM_MCP_SERVER_NAME, roomEndpointGrant } from "../../mcp/roomToolAccess.ts"; @@ -308,6 +311,7 @@ interface SessionContext { readonly roomEndpoint: boolean; readonly agentPageAssetsDir: string | undefined; readonly commandRequests: boolean; + readonly deviceTools: boolean; readonly managedWorktree: boolean; readonly mcpToken: string; readonly launchArgs: ReadonlyArray; @@ -491,6 +495,7 @@ export function makePiAdapter(options: PiAdapterOptions) { const instructionsFor = (input: { readonly agentPageAssetsDir: string | undefined; readonly commandRequests: boolean; + readonly deviceTools: boolean; readonly scheduleTools: boolean; readonly roomEndpoint: boolean; readonly managedWorktree: boolean; @@ -503,6 +508,7 @@ export function makePiAdapter(options: PiAdapterOptions) { ...(input.commandRequests ? [buildCommandRequestInstructions({ toolPrefix: `mcp__${COMMANDS_MCP_SERVER_NAME}__` })] : []), + ...(input.deviceTools ? [buildDeviceInstructions(`mcp__${DEVICE_MCP_SERVER_NAME}__`)] : []), ...(input.scheduleTools ? [buildScheduleInstructions({ toolPrefix: `mcp__${ROOM_MCP_SERVER_NAME}__` })] : []), @@ -517,6 +523,7 @@ export function makePiAdapter(options: PiAdapterOptions) { readonly roomEndpoint: boolean; readonly agentPageAssetsDir: string | undefined; readonly commandRequests: boolean; + readonly deviceTools: boolean; }): ReadonlyArray => [ { name: BROWSER_MCP_SERVER_NAME, url: mcpEndpointUrl(serverConfig.port) }, ...(ctx.roomEndpoint @@ -528,6 +535,9 @@ export function makePiAdapter(options: PiAdapterOptions) { ...(ctx.commandRequests ? [{ name: COMMANDS_MCP_SERVER_NAME, url: mcpCommandsEndpointUrl(serverConfig.port) }] : []), + ...(ctx.deviceTools + ? [{ name: DEVICE_MCP_SERVER_NAME, url: mcpDeviceEndpointUrl(serverConfig.port) }] + : []), ]; // ── Turns ──────────────────────────────────────────────────────────── @@ -1734,6 +1744,7 @@ export function makePiAdapter(options: PiAdapterOptions) { const roomTools = input.roomTools === true; const roomGrant = roomEndpointGrant(input, undefined); const commandRequests = input.commandRequests === true; + const deviceTools = input.deviceTools === true; const agentPageAssetsDir = input.agentPages === true ? yield* ensurePageAssetsDir(parseSessionKey(input.threadId).threadId).pipe( @@ -1747,6 +1758,7 @@ export function makePiAdapter(options: PiAdapterOptions) { room: roomGrant.attach, pages: agentPageAssetsDir !== undefined, commands: commandRequests, + device: deviceTools, scheduledRun: input.scheduledRunTools === true, guideOnly: roomGrant.guideOnly, }); @@ -1763,6 +1775,7 @@ export function makePiAdapter(options: PiAdapterOptions) { instructionsFor({ agentPageAssetsDir, commandRequests, + deviceTools, scheduleTools: roomTools && input.scheduleTools === true, roomEndpoint: roomGrant.attach, managedWorktree, @@ -1808,6 +1821,7 @@ export function makePiAdapter(options: PiAdapterOptions) { roomEndpoint: roomGrant.attach, agentPageAssetsDir, commandRequests, + deviceTools, managedWorktree, mcpToken: credential.token, launchArgs: launchArgs.args, diff --git a/apps/server/src/provider/acp/AcpAdapter.ts b/apps/server/src/provider/acp/AcpAdapter.ts index d1caff225..3d5097d07 100644 --- a/apps/server/src/provider/acp/AcpAdapter.ts +++ b/apps/server/src/provider/acp/AcpAdapter.ts @@ -67,11 +67,13 @@ import { BROWSER_MCP_SERVER_NAME, mcpEndpointUrl, mcpCommandsEndpointUrl, + mcpDeviceEndpointUrl, mcpPagesEndpointUrl, mcpRoomEndpointUrl, } from "../../mcp/McpHttpServer.ts"; import { PAGES_MCP_SERVER_NAME } from "../../mcp/pageTools.ts"; import { COMMANDS_MCP_SERVER_NAME } from "../../mcp/commandTools.ts"; +import { DEVICE_MCP_SERVER_NAME } from "../../mcp/deviceTools.ts"; import { mcpSessionRegistry } from "../../mcp/McpSessionRegistry.ts"; import { ROOM_MCP_SERVER_NAME, @@ -1269,6 +1271,8 @@ export function makeAcpAdapter( const agentPagesRequested = input.agentPages === true && lockdown === undefined; // `ask_user_to_run` likewise: its description carries the rules. const commandRequestsRequested = input.commandRequests === true && lockdown === undefined; + // The device tools likewise. + const deviceToolsRequested = input.deviceTools === true && lockdown === undefined; const credential = reachesHost && (lockdown === undefined || roomGrant.attach) ? yield* mcpSessionRegistry.credentialFor({ @@ -1277,6 +1281,7 @@ export function makeAcpAdapter( room: roomGrant.attach, pages: agentPagesRequested, commands: commandRequestsRequested, + device: deviceToolsRequested, ...(sideKind !== undefined ? { sideKind } : {}), scheduledRun: lockdown === undefined && input.scheduledRunTools === true, guideOnly: roomGrant.guideOnly, @@ -1335,6 +1340,16 @@ export function makeAcpAdapter( }, ] : []), + ...(deviceToolsRequested + ? [ + { + type: "http" as const, + name: DEVICE_MCP_SERVER_NAME, + url: mcpDeviceEndpointUrl(serverConfig.port), + headers: authorization, + }, + ] + : []), ]; // A side runtime's process is the descriptor's locked-down one; what diff --git a/apps/server/src/provider/acpRegistry/AcpRegistryInstaller.ts b/apps/server/src/provider/acpRegistry/AcpRegistryInstaller.ts index dc854c41e..cd0f0d89c 100644 --- a/apps/server/src/provider/acpRegistry/AcpRegistryInstaller.ts +++ b/apps/server/src/provider/acpRegistry/AcpRegistryInstaller.ts @@ -263,7 +263,7 @@ const SHIM_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u; * configuration, say). * - `update-notifier`: npm's last lines are about this install. */ -const npmPinnedFlags = (ignoreScripts: boolean) => [ +export const npmPinnedFlags = (ignoreScripts: boolean) => [ "--global=false", "--package-lock=true", "--no-audit", diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index 3ac47430b..28e0c3ebc 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { - "claudeSystemAppendBytes": 7281, + "claudeSystemAppendBytes": 7992, "upFrontTools": [ "app_guide", "ask_user_to_run", diff --git a/apps/server/src/provider/agentContextBudget.test.ts b/apps/server/src/provider/agentContextBudget.test.ts index 834b31413..ca8dea255 100644 --- a/apps/server/src/provider/agentContextBudget.test.ts +++ b/apps/server/src/provider/agentContextBudget.test.ts @@ -89,7 +89,7 @@ it("what a Claude thread is sent on every request matches the committed budget", const onDemand = threadTools.filter((tool) => !isLoadedUpFront(tool)); const measured = { // Every optional section on, as in a thread with pages, artifacts, - // command requests, schedules and the room tools, in a + // command requests, devices, schedules and the room tools, in a // Threadlines-managed worktree. claudeSystemAppendBytes: utf8Bytes( onLinux(() => @@ -97,6 +97,7 @@ it("what a Claude thread is sent on every request matches the committed budget", pageAssetsDir: "/tmp/threadlines-page-assets/thread", artifacts: true, commandRequests: true, + deviceTools: true, room: { tools: roomToolsFor(undefined), schedules: true }, lockdown: false, managedWorktree: true, diff --git a/apps/server/src/provider/codexAppServerArgs.ts b/apps/server/src/provider/codexAppServerArgs.ts index e6d1fb99e..c39db102f 100644 --- a/apps/server/src/provider/codexAppServerArgs.ts +++ b/apps/server/src/provider/codexAppServerArgs.ts @@ -122,6 +122,7 @@ export function codexAppServerArgs(servers?: { readonly room?: CodexRoomServer; readonly pages?: { readonly url: string; readonly serverName: string }; readonly commands?: { readonly url: string; readonly serverName: string }; + readonly device?: { readonly url: string; readonly serverName: string }; }): ReadonlyArray { return [ ...CODEX_APP_SERVER_ARGS, @@ -131,14 +132,17 @@ export function codexAppServerArgs(servers?: { ...(servers?.room !== undefined ? codexRoomServerArgs(servers.room) : []), ...(servers?.pages !== undefined ? codexPagesServerArgs(servers.pages) : []), ...(servers?.commands !== undefined ? codexPagesServerArgs(servers.commands) : []), + ...(servers?.device !== undefined ? codexPagesServerArgs(servers.device) : []), ]; } /** - * The page tools (McpPagesServer), and `ask_user_to_run` (McpCommandsServer). - * They never ask: a page shows only in the caller's own thread, and a command - * card only asks the user. That is why each has a server of its own, since - * Codex approves per server and the browser tools must still ask. + * The page tools (McpPagesServer), `ask_user_to_run` (McpCommandsServer) and + * the device tools (McpDeviceServer). They never ask: a page shows only in the + * caller's own thread, a command card only asks the user, and the user turned + * on "Agents can use devices" for the device tools. That is why each has a + * server of its own, since Codex approves per server and the browser tools + * must still ask. */ function codexPagesServerArgs(pages: { readonly url: string; diff --git a/apps/server/src/provider/deviceInstructions.ts b/apps/server/src/provider/deviceInstructions.ts new file mode 100644 index 000000000..f415058ba --- /dev/null +++ b/apps/server/src/provider/deviceInstructions.ts @@ -0,0 +1,26 @@ +/** + * The device tools, introduced to a model (docs/design/devices.md). + * + * Sent only to sessions started with `deviceTools`. An agent picks between + * these tools, `simctl`/`adb` and its own habits before it opens anything, so + * this says which to reach for. How to drive a device comes back from + * `device_open` itself: it costs nothing in threads that never open one and + * cannot drift from the pinned agent-device version. + * + * ACP providers get the tool descriptions only, like every other Threadlines + * tool there. + */ +import { DEVICE_MCP_SERVER_NAME } from "../mcp/deviceTools.ts"; + +/** + * `toolPrefix` is what precedes each device tool's own name for this + * provider: `mcp__threadlines_device__` for Claude and Codex, `` for Muse, + * `_` for OpenCode. + */ +export function buildDeviceInstructions( + toolPrefix: string = `mcp__${DEVICE_MCP_SERVER_NAME}__`, +): string { + return ` +The user can watch iOS Simulators and Android Emulators on this computer in the Device tab beside this chat. For any work on a simulator or emulator's screen (opening an app, tapping, typing, reading what is shown, taking a screenshot), call \`${toolPrefix}device_open\` and drive the device with the \`agent-device\` command it returns, so the user sees each step. \`${toolPrefix}device_list\` shows what can be opened, \`${toolPrefix}device_screenshot\` shows you the screen, and \`${toolPrefix}device_close\` ends your hold on a device. \`simctl\`, \`adb\` and build tools stay fine for builds, installs, logs and test runners. +`; +} diff --git a/apps/server/src/provider/opencode/OpenCodeRules.test.ts b/apps/server/src/provider/opencode/OpenCodeRules.test.ts index 716094df5..48c41c860 100644 --- a/apps/server/src/provider/opencode/OpenCodeRules.test.ts +++ b/apps/server/src/provider/opencode/OpenCodeRules.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "vite-plus/test"; import { type OpenCodeRule, openCodeBrowserServerName, + openCodeDeviceServerName, openCodePagesServerName, openCodeSessionGrantRules, openCodeSessionRules, @@ -114,6 +115,24 @@ describe("openCodeSessionRules", () => { expect(effect(rules, `${openCodePagesServerName(otherKey)}_show_page`, "*")).toBe("deny"); }); + it("lets only a session with devices reach its own thread's device tools", () => { + const rulesWith = (deviceTools: boolean) => [ + ...BUILD_AGENT, + ...openCodeSessionRules({ + runtimeMode: "approval-required", + agentRules: BUILD_AGENT, + toolKey: ownKey, + roomTools: false, + deviceTools, + }), + ]; + const own = `${openCodeDeviceServerName(ownKey)}_device_open`; + const other = `${openCodeDeviceServerName(otherKey)}_device_open`; + expect(effect(rulesWith(true), own, "*")).toBe("allow"); + expect(effect(rulesWith(true), other, "*")).toBe("deny"); + expect(effect(rulesWith(false), own, "*")).toBe("deny"); + }); + it("honours a user's deny of Threadlines' own tools", () => { const agent: ReadonlyArray = [ ...BUILD_AGENT, diff --git a/apps/server/src/provider/opencode/OpenCodeRules.ts b/apps/server/src/provider/opencode/OpenCodeRules.ts index adcb42dcd..0ea190090 100644 --- a/apps/server/src/provider/opencode/OpenCodeRules.ts +++ b/apps/server/src/provider/opencode/OpenCodeRules.ts @@ -35,6 +35,7 @@ const BROWSER_SERVER_PREFIX = "threadlines_b_"; const ROOM_SERVER_PREFIX = "threadlines_r_"; const PAGES_SERVER_PREFIX = "threadlines_p_"; const COMMANDS_SERVER_PREFIX = "threadlines_c_"; +const DEVICE_SERVER_PREFIX = "threadlines_d_"; /** Short, stable, tool-name-safe key for one thread's tool servers. */ export function openCodeThreadToolKey(threadKey: string): string { @@ -57,6 +58,10 @@ export function openCodeCommandsServerName(toolKey: string): string { return `${COMMANDS_SERVER_PREFIX}${toolKey}`; } +export function openCodeDeviceServerName(toolKey: string): string { + return `${DEVICE_SERVER_PREFIX}${toolKey}`; +} + /** OpenCode's own wildcard: `*` is any run of characters, `?` any one. */ function wildcardMatch(input: string, pattern: string): boolean { const escaped = pattern @@ -122,6 +127,8 @@ export function openCodeSessionRules(input: { readonly agentPages?: boolean; /** `ask_user_to_run`: it only shows the user a card, so it never asks. */ readonly commandRequests?: boolean; + /** The device tools: the user turned on "Agents can use devices", so they never ask. */ + readonly deviceTools?: boolean; }): ReadonlyArray { const mode = modeRules(input.runtimeMode); return [ @@ -130,6 +137,7 @@ export function openCodeSessionRules(input: { { action: `${ROOM_SERVER_PREFIX}*`, resource: "*", effect: "deny" }, { action: `${PAGES_SERVER_PREFIX}*`, resource: "*", effect: "deny" }, { action: `${COMMANDS_SERVER_PREFIX}*`, resource: "*", effect: "deny" }, + { action: `${DEVICE_SERVER_PREFIX}*`, resource: "*", effect: "deny" }, { action: `${openCodeBrowserServerName(input.toolKey)}_*`, resource: "*", @@ -162,6 +170,15 @@ export function openCodeSessionRules(input: { } satisfies OpenCodeRule, ] : []), + ...(input.deviceTools === true + ? [ + { + action: `${openCodeDeviceServerName(input.toolKey)}_*`, + resource: "*", + effect: "allow", + } satisfies OpenCodeRule, + ] + : []), ...agentRestrictions(input.agentRules, mode), ]; } diff --git a/apps/server/src/provider/opencode/OpenCodeToolItems.ts b/apps/server/src/provider/opencode/OpenCodeToolItems.ts index cf987034b..332c5ee1a 100644 --- a/apps/server/src/provider/opencode/OpenCodeToolItems.ts +++ b/apps/server/src/provider/opencode/OpenCodeToolItems.ts @@ -18,6 +18,7 @@ import { BROWSER_MCP_SERVER_NAME } from "../../mcp/McpHttpServer.ts"; import { ROOM_MCP_SERVER_NAME } from "../../mcp/roomToolAccess.ts"; import { PAGES_MCP_SERVER_NAME } from "../../mcp/pageTools.ts"; import { COMMANDS_MCP_SERVER_NAME } from "../../mcp/commandTools.ts"; +import { DEVICE_MCP_SERVER_NAME } from "../../mcp/deviceTools.ts"; const DETAIL_MAX_CHARS = 400; const OUTPUT_MAX_CHARS = 20_000; @@ -59,12 +60,13 @@ function truncate(value: string, max: number): string { /** * Threadlines' own tool servers are registered per thread under - * `threadlines_b_` / `threadlines_r_` / `threadlines_p_` / `threadlines_c_`; + * `threadlines_b_` / `threadlines_r_` / `threadlines_p_` / `threadlines_c_` / + * `threadlines_d_`; * the chat knows them by their * stable names, so that is what it is told. */ function threadlinesMcpTool(name: string): OpenCodeToolIdentity["mcp"] | undefined { - const match = /^threadlines_([brpc])_[0-9a-f]+_(.+)$/u.exec(name); + const match = /^threadlines_([brpcd])_[0-9a-f]+_(.+)$/u.exec(name); if (!match) return undefined; return { server: @@ -74,7 +76,9 @@ function threadlinesMcpTool(name: string): OpenCodeToolIdentity["mcp"] | undefin ? PAGES_MCP_SERVER_NAME : match[1] === "c" ? COMMANDS_MCP_SERVER_NAME - : ROOM_MCP_SERVER_NAME, + : match[1] === "d" + ? DEVICE_MCP_SERVER_NAME + : ROOM_MCP_SERVER_NAME, tool: match[2]!, }; } diff --git a/apps/server/src/scheduled/ScheduledTasks.test.ts b/apps/server/src/scheduled/ScheduledTasks.test.ts index 64941f90f..3960b89e0 100644 --- a/apps/server/src/scheduled/ScheduledTasks.test.ts +++ b/apps/server/src/scheduled/ScheduledTasks.test.ts @@ -1346,6 +1346,7 @@ it.layer(NodeServices.layer)("schedule tools", (it) => { browser: true, pages: false, commands: false, + device: false, roomTools: new Set(roomToolsFor(undefined, { scheduledRun })), }); diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 1aecc2894..8308e51fb 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -184,6 +184,8 @@ import * as PreviewTunnel from "./preview/PreviewTunnel.ts"; import { AgentPages } from "./pages/AgentPages.ts"; import { McpApps } from "./mcpApps/McpApps.ts"; import { CommandRuns } from "./commandRuns/CommandRuns.ts"; +import { DeviceChannels } from "./device/DeviceChannels.ts"; +import { DeviceService } from "./device/DeviceService.ts"; import { AgentSettingsChanges } from "./settingsChanges/AgentSettingsChanges.ts"; import { AgentSecretRequests } from "./secrets/AgentSecretRequests.ts"; import { layer as SecretScrubberLive } from "./secrets/SecretScrubber.ts"; @@ -798,6 +800,9 @@ const buildAppUnderTest = (options?: { Layer.mock(AgentSettingsChanges)({}), // So are agents' requests for keys (AgentSecretRequests.test.ts). Layer.mock(AgentSecretRequests)({ destinations: {} as never }), + // Devices are tested on their own (device/*.test.ts); the routes only need them built. + Layer.mock(DeviceService)({}), + Layer.mock(DeviceChannels)({ releaseConnection: () => Effect.void }), // The watched keys' registry, empty: the routes' doors read it. SecretScrubberLive, Layer.mock(CommandRuns)({ diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 66f28cc7e..2ca612a9c 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -107,6 +107,10 @@ import * as McpHttpServer from "./mcp/McpHttpServer.ts"; import * as McpRoomServer from "./mcp/McpRoomServer.ts"; import * as McpPagesServer from "./mcp/McpPagesServer.ts"; import * as McpCommandsServer from "./mcp/McpCommandsServer.ts"; +import * as McpDeviceServer from "./mcp/McpDeviceServer.ts"; +import * as DeviceChannels from "./device/DeviceChannels.ts"; +import * as DeviceService from "./device/DeviceService.ts"; +import * as DeviceThreadReactor from "./device/DeviceThreadReactor.ts"; import * as McpConnectServer from "./outsideAgents/McpConnectServer.ts"; import { AssistantMcpServerLive } from "./assistants/AssistantMcpServer.ts"; import { OtherComputersLive } from "./outsideAgents/OtherComputers.ts"; @@ -632,6 +636,14 @@ const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe( // Answers to agents' requests for keys; it settles at startup what a // previous run left mid-save and sets saved files' expiry timers. AgentSecretRequests.layer.pipe(Layer.provide(ProcessRunnerLive)), + // Simulators and emulators (docs/design/devices.md): the service, the + // Device tab's channels to the hub, and the reactor that lets a thread + // put away let go of its devices (and sweeps claims at startup). + DeviceThreadReactor.layer.pipe( + Layer.provideMerge(DeviceChannels.layer), + Layer.provideMerge(DeviceService.layer), + Layer.provide(ProcessRunnerLive), + ), // The calls' table also serves thread deletion's cleanup. McpApps.layer.pipe(Layer.provideMerge(McpAppCallRepositoryLive)), // Outside agents' starts and sends by request id (`/mcp/connect`); @@ -773,6 +785,7 @@ export const makeRoutesLayer = Layer.mergeAll( McpRoomServer.layer, McpPagesServer.layer, McpCommandsServer.layer, + McpDeviceServer.layer, // Outside agents' tools (docs/design/outside-agents.md). McpConnectServer.layer, // The same tools for assistants, over the relay (docs/design/assistants.md). diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 58ebac1e5..3fad00147 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -70,6 +70,7 @@ import { WsMovesRpcGroup, WsThemeShowcaseRpcGroup, WsThemeVsCodeRpcGroup, + WsDeviceRpcGroup, WsThemesRpcGroup, WsOwnerRequiredError, OutsideAgentError, @@ -119,6 +120,8 @@ import { import { LocalServers } from "./preview/LocalServers.ts"; import { PreviewAutomationBroker } from "./preview/PreviewAutomationBroker.ts"; import { PreviewTunnel } from "./preview/PreviewTunnel.ts"; +import { DeviceChannels } from "./device/DeviceChannels.ts"; +import { DeviceService } from "./device/DeviceService.ts"; import { ProjectFaviconResolver } from "./project/Services/ProjectFaviconResolver.ts"; import { PullRequestService } from "./pullRequest/PullRequestService.ts"; import { PullRequestObservations } from "./pullRequest/PullRequestObservations.ts"; @@ -833,6 +836,51 @@ const ThemeVsCodeRpcLayer = WsThemeVsCodeRpcGroup.toLayer( }), ); +/** + * Devices (docs/design/devices.md): its own RPC group and layer + * (WsDeviceRpcGroup), merged in makeWsRpcLayer. Open to every paired device, + * as terminals are. Channels belong to this connection (`viewerId`) and close + * with it. + */ +const makeDeviceRpcLayer = (viewerId: string) => + WsDeviceRpcGroup.toLayer( + Effect.gen(function* () { + const devices = yield* DeviceService; + const channels = yield* DeviceChannels; + const aggregate = { "rpc.aggregate": "devices" } as const; + return WsDeviceRpcGroup.of({ + [WS_METHODS.deviceList]: (input) => + observeRpcEffect(WS_METHODS.deviceList, devices.list(input), aggregate), + [WS_METHODS.deviceUpdateTool]: (input) => + observeRpcEffect(WS_METHODS.deviceUpdateTool, devices.updateTool(input.tool), aggregate), + [WS_METHODS.deviceOpen]: (input) => + observeRpcEffect(WS_METHODS.deviceOpen, devices.open(input, "user"), aggregate), + [WS_METHODS.deviceClose]: (input) => + observeRpcEffect(WS_METHODS.deviceClose, devices.close(input), aggregate), + [WS_METHODS.deviceShutdown]: (input) => + observeRpcEffect(WS_METHODS.deviceShutdown, devices.shutdown(input), aggregate), + [WS_METHODS.deviceDetail]: (input) => + observeRpcEffect(WS_METHODS.deviceDetail, devices.detail(input), aggregate), + [WS_METHODS.deviceAction]: (input) => + observeRpcEffect(WS_METHODS.deviceAction, devices.action(input), aggregate), + [WS_METHODS.subscribeDeviceState]: (_input) => + observeRpcStream(WS_METHODS.subscribeDeviceState, devices.stream, aggregate), + [WS_METHODS.deviceChannelOpen]: (input) => + // Scoped to the subscription: the client letting go, or its socket + // dropping, closes the request or socket to the hub too. + observeRpcStreamEffect( + WS_METHODS.deviceChannelOpen, + channels.open(viewerId, input), + aggregate, + ), + // A channel's messages and credits are its traffic itself: left out of + // the RPC metrics and traces they would drown. + [WS_METHODS.deviceChannelSend]: (input) => channels.send(viewerId, input), + [WS_METHODS.deviceChannelCredit]: (input) => channels.credit(viewerId, input), + }); + }), + ); + /** Every RPC the WebSocket serves, for one connection. */ const makeWsRpcLayer = ( currentSession: { @@ -852,6 +900,7 @@ const makeWsRpcLayer = ( ThemeShowcaseRpcLayer, MovesRpcLayer, ThemeVsCodeRpcLayer, + makeDeviceRpcLayer(viewerId), ); const makeCoreRpcLayer = ( @@ -3675,6 +3724,7 @@ export const websocketRpcRouteLayer = Layer.unwrap( const threadBootstrap = yield* ThreadBootstrap; const terminalManager = yield* TerminalManager; const pushAlerts = yield* PushAlerts; + const deviceChannels = yield* DeviceChannels; return HttpRouter.add( "GET", "/ws", @@ -3770,6 +3820,7 @@ export const websocketRpcRouteLayer = Layer.unwrap( .pipe( Effect.andThen(terminalManager.releaseViewer(viewerId)), Effect.andThen(pushAlerts.releaseViewer(viewerId)), + Effect.andThen(deviceChannels.releaseConnection(viewerId)), ), ); }).pipe(Effect.scoped, Effect.catchTag("AuthError", respondToAuthError)), diff --git a/apps/web/src/components/ChatView.browser-harness.tsx b/apps/web/src/components/ChatView.browser-harness.tsx index d11409a14..86a2a588b 100644 --- a/apps/web/src/components/ChatView.browser-harness.tsx +++ b/apps/web/src/components/ChatView.browser-harness.tsx @@ -402,6 +402,7 @@ export function createMockEnvironmentApi(input: { attachments: {} as EnvironmentApi["attachments"], previewAutomation: input.previewAutomation ?? ({} as EnvironmentApi["previewAutomation"]), previewLoopback: {} as EnvironmentApi["previewLoopback"], + device: {} as EnvironmentApi["device"], filesystem: { browse: input.browse, }, diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 7be7461ef..8c05d6f65 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -99,6 +99,7 @@ import { useRoomRecipientStore, } from "../rooms"; import { ELECTRON_HEADER_HEIGHT_CLASS } from "../desktopChrome"; +import { useDevicesSupported } from "./device/deviceState"; import { isElectron } from "../env"; import { ensureLocalApi, readLocalApi } from "../localApi"; import { parseDiffRouteSearch } from "../diffRouteSearch"; @@ -1606,13 +1607,15 @@ export default function ChatView(props: ChatViewProps) { // Which surfaces this thread's sidebar offers. Resolved from the same inputs // the route uses, so the header's entry points can never land on a tab the // strip would refuse to show. + const devicesSupported = useDevicesSupported(environmentId); const rightPanelAvailableTabs = useMemo( () => availableRightPanelTabs({ isGeneralChat: isGeneralChatThread, isDraft: routeKind === "draft" || !isServerThread, + devicesSupported, }), - [isGeneralChatThread, isServerThread, routeKind], + [devicesSupported, isGeneralChatThread, isServerThread, routeKind], ); /** * A picked element becomes a context attached to the message, not text in it. diff --git a/apps/web/src/components/CommandPaletteDialog.tsx b/apps/web/src/components/CommandPaletteDialog.tsx index b1bdbb8fd..30c1538be 100644 --- a/apps/web/src/components/CommandPaletteDialog.tsx +++ b/apps/web/src/components/CommandPaletteDialog.tsx @@ -73,6 +73,7 @@ import { PinOffIcon, PlugIcon, CameraIcon, + SmartphoneIcon, } from "lucide-react"; import { useCallback, @@ -154,7 +155,11 @@ import { import { useGitStatus } from "../lib/gitStatusState"; import { useScheduledTaskEnvironments } from "../lib/scheduledTasksState"; import { resolveThreadPullRequest, threadViewBranch } from "./pull-requests/pullRequests.logic"; -import { focusRightPanelTab, pullRequestTabSearchParams } from "../rightPanelTabs"; +import { + focusRightPanelTab, + pullRequestTabSearchParams, + rightPanelTabSearchParams, +} from "../rightPanelTabs"; import { cn, isMacPlatform, newCommandId, newProjectId, newThreadId } from "../lib/utils"; import { selectProjectsAcrossEnvironments, @@ -201,6 +206,7 @@ import { APP_VERSION } from "../branding"; import { useProjectSetupTargets } from "../environments/projectSetups/projectSetupTargets"; import { useMoveThreadsOffered, useThreadMoveOffered } from "../environments/links/threadMoves"; import { openMoveThreadsDialog } from "./MoveThreadsDialog"; +import { environmentSupportsDevices } from "./device/deviceState"; import { startAssistantCode, useOutsideAgentsSnapshot, @@ -2033,6 +2039,32 @@ export function OpenCommandPaletteDialog() { }); } + // A project thread on a computer whose server has devices: the same rule as + // the Device tab itself (availableRightPanelTabs). + if ( + activeThread && + activeThreadProject !== null && + activeThreadProject.kind !== "general-chat" && + environmentSupportsDevices(activeThread.environmentId) + ) { + const threadRef = scopeThreadRef(activeThread.environmentId, activeThread.id); + actionItems.push({ + kind: "action", + value: "action:thread-device", + searchTerms: ["device", "simulator", "emulator", "ios", "android", "phone"], + title: "Open device", + icon: , + run: async () => { + focusRightPanelTab(scopedThreadKey(threadRef), "device"); + await navigate({ + to: "/$environmentId/$threadId", + params: buildThreadRouteParams(threadRef), + search: (previous) => rightPanelTabSearchParams(previous, "device"), + }); + }, + }); + } + // Where find in chat can open: a saved chat on screen, off phones. if (chatFindAvailable) { actionItems.push({ diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index 9505af168..b2e8af5ca 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -17,6 +17,8 @@ import type { ThreadId, TurnId, } from "@threadlines/contracts"; +import { saveComposerDraftAttachments } from "../../composerDraftAttachmentSave"; +import { publishComposerImageSupport } from "./composerImageSupport"; import { type PromptRecall, recallablePrompts, stepPromptRecall } from "./composerPromptRecall"; import { containsWorkflowKeyword } from "@threadlines/shared/workflowRuns"; import { ComposerWorkflowKeywordHint } from "./ComposerWorkflowKeywordHint"; @@ -66,11 +68,7 @@ import { expandCollapsedComposerCursor, replaceTextRange, } from "../../composer-logic"; -import { - deriveComposerSendState, - desktopCapturedScreenshotToFile, - readFileAsDataUrl, -} from "../ChatView.logic"; +import { deriveComposerSendState, desktopCapturedScreenshotToFile } from "../ChatView.logic"; import { type ComposerAttachment, type DraftId, @@ -868,13 +866,6 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) const updateComposerDraftTranscriptHighlightContextNote = useComposerDraftStore( (store) => store.updateTranscriptHighlightContextNote, ); - const clearComposerDraftPersistedAttachments = useComposerDraftStore( - (store) => store.clearPersistedAttachments, - ); - const syncComposerDraftPersistedAttachments = useComposerDraftStore( - (store) => store.syncPersistedAttachments, - ); - const getComposerDraft = useComposerDraftStore((store) => store.getComposerDraft); const clearStashableComposerContent = useComposerDraftStore( (store) => store.clearStashableComposerContent, ); @@ -1027,6 +1018,10 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) ), [selectedProvider, selectedProviderModels, selectedModel], ); + // A device recording asks before it attaches screenshots to this thread. + useEffect(() => { + publishComposerImageSupport(scopedThreadKey(routeThreadRef), selectedModelSupportsImages); + }, [routeThreadRef, selectedModelSupportsImages]); const composerProviderState = useMemo( () => @@ -2719,67 +2714,17 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) // ------------------------------------------------------------------ // Image persist effect // ------------------------------------------------------------------ + // Saves the attachments whenever they change; a stale run skips its write. useEffect(() => { let cancelled = false; - void (async () => { - if (composerAttachments.length === 0) { - clearComposerDraftPersistedAttachments(composerDraftTarget); - return; - } - const getPersistedAttachmentsForThread = () => - getComposerDraft(composerDraftTarget)?.persistedAttachments ?? []; - try { - const currentPersistedAttachments = getPersistedAttachmentsForThread(); - const existingPersistedById = new Map( - currentPersistedAttachments.map((attachment) => [attachment.id, attachment]), - ); - const stagedAttachmentById = new Map(); - await Promise.all( - composerAttachments.map(async (image) => { - try { - const dataUrl = await readFileAsDataUrl(image.file); - stagedAttachmentById.set(image.id, { - id: image.id, - name: image.name, - mimeType: image.mimeType, - sizeBytes: image.sizeBytes, - dataUrl, - }); - } catch { - const existingPersisted = existingPersistedById.get(image.id); - if (existingPersisted) { - stagedAttachmentById.set(image.id, existingPersisted); - } - } - }), - ); - const serialized = Array.from(stagedAttachmentById.values()); - if (cancelled) return; - syncComposerDraftPersistedAttachments(composerDraftTarget, serialized); - } catch { - const currentImageIds = new Set(composerAttachments.map((image) => image.id)); - const fallbackPersistedAttachments = getPersistedAttachmentsForThread(); - const fallbackPersistedIds = fallbackPersistedAttachments - .map((attachment) => attachment.id) - .filter((id) => currentImageIds.has(id)); - const fallbackPersistedIdSet = new Set(fallbackPersistedIds); - const fallbackAttachments = fallbackPersistedAttachments.filter((attachment) => - fallbackPersistedIdSet.has(attachment.id), - ); - if (cancelled) return; - syncComposerDraftPersistedAttachments(composerDraftTarget, fallbackAttachments); - } - })(); + void saveComposerDraftAttachments(composerDraftTarget, { + attachments: composerAttachments, + cancelled: () => cancelled, + }); return () => { cancelled = true; }; - }, [ - composerDraftTarget, - clearComposerDraftPersistedAttachments, - composerAttachments, - getComposerDraft, - syncComposerDraftPersistedAttachments, - ]); + }, [composerDraftTarget, composerAttachments]); // ------------------------------------------------------------------ // Callbacks: prompt change diff --git a/apps/web/src/components/chat/RightPanelTabStrip.tsx b/apps/web/src/components/chat/RightPanelTabStrip.tsx index c376f3c1c..dd41a604e 100644 --- a/apps/web/src/components/chat/RightPanelTabStrip.tsx +++ b/apps/web/src/components/chat/RightPanelTabStrip.tsx @@ -25,6 +25,7 @@ import { ImagesIcon, ListTreeIcon, PlusIcon, + SmartphoneIcon, XIcon, } from "lucide-react"; import { memo, useCallback, useEffect, useLayoutEffect, useRef, useState } from "react"; @@ -56,6 +57,7 @@ export const RIGHT_PANEL_TAB_ICONS: Readonly> = { agents: BotIcon, media: ImagesIcon, threads: ListTreeIcon, + device: SmartphoneIcon, }; /** Labelled while they fit, all icons once they do not. Never a mix: dropping diff --git a/apps/web/src/components/chat/composerImageSupport.ts b/apps/web/src/components/chat/composerImageSupport.ts new file mode 100644 index 000000000..3eb67536d --- /dev/null +++ b/apps/web/src/components/chat/composerImageSupport.ts @@ -0,0 +1,16 @@ +/** + * Whether each thread's composer would take images right now, as the composer + * itself works it out from the selected model. Published by the mounted + * composer so code outside it (a device recording handing off screenshots) + * makes the same call the composer would. Unknown threads count as taking + * images, the composer's own default for a model with no listed modalities. + */ +const imagesByThreadKey = new Map(); + +export function publishComposerImageSupport(threadKey: string, acceptsImages: boolean) { + imagesByThreadKey.set(threadKey, acceptsImages); +} + +export function composerAcceptsImages(threadKey: string): boolean { + return imagesByThreadKey.get(threadKey) ?? true; +} diff --git a/apps/web/src/components/device/DeviceBody.tsx b/apps/web/src/components/device/DeviceBody.tsx new file mode 100644 index 000000000..4e35b2d29 --- /dev/null +++ b/apps/web/src/components/device/DeviceBody.tsx @@ -0,0 +1,202 @@ +import type { CSSProperties, ReactNode } from "react"; + +import { type BodyBox, DEVICE_BUTTON_DEPTH, type QuarterTurn } from "./deviceFrameLayout"; +import type { DeviceBody } from "./deviceShapes"; + +/** The metal band's width, in points; the glass inside it is the bezel. */ +const BAND = 2.5; +/** The band and the side buttons: a step off the panel in both themes, flat. */ +const BAND_COLOR = "color-mix(in oklab, var(--foreground) 24%, var(--background))"; +/** The hinge's knuckles on a Duo's band. */ +const HINGE_COLOR = "color-mix(in oklab, var(--foreground) 38%, var(--background))"; + +const px = (value: number) => `${value}px`; + +const cornerRadii = (radii: readonly number[], scale: number, grow: number) => + radii.map((radius) => px(Math.max(0, radius + grow) * scale)).join(" "); + +/** + * One device body drawn upright at `scale` px per point: the band, the black + * glass, side buttons, the screen with the stream in it, and whatever the + * hardware puts over the screen (Dynamic Island, notch, punch hole, a fold + * line). Flat: no shadows, no gloss, theme colors for everything but the glass. + */ +export function DeviceBodyView(props: { + readonly body: DeviceBody; + readonly box: BodyBox; + readonly scale: number; + /** Top left in the drawing, in points. */ + readonly x: number; + readonly y: number; + /** The stream's turn inside the screen. */ + readonly canvasTurn: QuarterTurn; + /** Mounts the stream's canvas. */ + readonly mountCanvas: (node: HTMLDivElement | null) => (() => void) | undefined; + /** iPhone Duo: draw the knuckles of a hinge down the middle. */ + readonly hinge?: boolean; + /** A display not in front: its last picture, dimmed. */ + readonly inactive?: boolean; + readonly children?: ReactNode; + readonly testId?: string; +}) { + const { body, box, scale, mountCanvas } = props; + const { bezel } = body; + const outer = Math.min(bezel.top, bezel.right, bezel.bottom, bezel.left); + // The body's corners follow the screen's, out by the bezel; a square screen keeps a phone's rounding. + const bodyRadii = body.screenRadius.map((radius) => + radius > 0 ? radius + outer : Math.min(61, outer * 2.2), + ); + const screen = box.screen; + const sideways = props.canvasTurn === 90 || props.canvasTurn === -90; + const mediaStyle: CSSProperties = sideways + ? { + width: screen.height * scale, + height: screen.width * scale, + left: ((screen.width - screen.height) * scale) / 2, + top: ((screen.height - screen.width) * scale) / 2, + transform: `rotate(${props.canvasTurn}deg)`, + } + : { + width: screen.width * scale, + height: screen.height * scale, + ...(props.canvasTurn ? { transform: `rotate(${props.canvasTurn}deg)` } : {}), + }; + const depth = DEVICE_BUTTON_DEPTH * scale; + const cutout = body.cutout; + return ( +
+ {body.buttons.map((button) => { + const along = button.offset * scale; + const length = button.length * scale; + const style: CSSProperties = + button.edge === "top" + ? { left: along, top: -depth, width: length, height: depth + scale * 2 } + : { + top: along, + height: length, + width: depth + scale * 2, + ...(button.edge === "left" ? { left: -depth } : { right: -depth }), + }; + return ( + + ); + })} +
+
+ {props.hinge ? ( + <> + + + + ) : null} +
+
+ {body.crease ? ( + + ) : null} + {cutout?.kind === "island" ? ( + + ) : cutout?.kind === "notch" ? ( + + ) : cutout?.kind === "punch" ? ( + + ) : null} + {props.children} +
+ {body.homeButton ? ( + + ) : null} +
+ ); +} diff --git a/apps/web/src/components/device/DeviceChips.tsx b/apps/web/src/components/device/DeviceChips.tsx new file mode 100644 index 000000000..e48055bff --- /dev/null +++ b/apps/web/src/components/device/DeviceChips.tsx @@ -0,0 +1,171 @@ +import { ChevronDownIcon, PictureInPicture2Icon, PlusIcon } from "lucide-react"; +import { useRef, useState } from "react"; + +import { cn } from "~/lib/utils"; +import { Button } from "../ui/button"; +import { Menu, MenuItem, MenuPopup, MenuTrigger } from "../ui/menu"; +import { DEVICE_CHIP_NAME_MAX_CHARS } from "./deviceViewStore"; + +export interface DeviceChip { + readonly deviceId: string; + readonly name: string; + /** The device's own name, shown when a chip was renamed. */ + readonly deviceName: string; + readonly floating: boolean; +} + +/** + * The thread's open devices as a row of chips at the top of the Device tab + * (docs/design/devices.md, "Several devices open at once"). A press shows + * one; double-click or the chip's menu renames it; the menu also closes it; + * "+" shows the picker. Plain until hover, the shown chip filled. + */ +export function DeviceChips(props: { + readonly chips: ReadonlyArray; + /** The chip on screen, or null while the picker shows. */ + readonly selectedId: string | null; + readonly onSelect: (deviceId: string | null) => void; + readonly onRename: (deviceId: string, name: string) => void; + readonly onClose: (deviceId: string) => void; +}) { + const [renaming, setRenaming] = useState(null); + // Rename from a menu: the menu must not take focus back from the new field. + const renameFromMenu = useRef(false); + return ( +
+ {props.chips.map((chip) => { + const selected = chip.deviceId === props.selectedId; + if (renaming === chip.deviceId) { + return ( + { + setRenaming(null); + if (name !== null) props.onRename(chip.deviceId, name); + }} + /> + ); + } + return ( +
+ + + + } + > + + { + const keepFocus = renameFromMenu.current; + renameFromMenu.current = false; + return !keepFocus; + }} + > + { + renameFromMenu.current = true; + setRenaming(chip.deviceId); + }} + > + Rename… + + props.onClose(chip.deviceId)}>Close device + + +
+ ); + })} + +
+ ); +} + +function RenameField(props: { + readonly chip: DeviceChip; + readonly onDone: (name: string | null) => void; +}) { + const [value, setValue] = useState(props.chip.name); + const [done, setDone] = useState(false); + const finish = (name: string | null) => { + if (done) return; + setDone(true); + props.onDone(name); + }; + return ( + setValue(event.target.value)} + onFocus={(event) => event.currentTarget.select()} + onBlur={() => finish(value)} + onKeyDown={(event) => { + if (event.key === "Enter") { + event.preventDefault(); + finish(value); + } else if (event.key === "Escape") { + event.preventDefault(); + event.stopPropagation(); + finish(null); + } + }} + /> + ); +} diff --git a/apps/web/src/components/device/DeviceDuoGlyph.tsx b/apps/web/src/components/device/DeviceDuoGlyph.tsx new file mode 100644 index 000000000..789899906 --- /dev/null +++ b/apps/web/src/components/device/DeviceDuoGlyph.tsx @@ -0,0 +1,45 @@ +import type { DuoPose } from "@threadlines/client-runtime/device"; + +import { cn } from "~/lib/utils"; + +// Ported from T3 Code's DeviceDuoGlyph (MIT, see NOTICE). +const stancePaths = { + laptop: + "M2.7351 12.9857 L15.8128 11.4159 L14.4114 3.3787 C14.1943 2.134 13.1275 1.2319 12.0283 1.3639 L2.9308 2.4559 C1.8317 2.5879 1.1167 3.7038 1.3337 4.9485 L2.7351 12.9857 Z M2.7351 12.9857 L15.8128 11.4159 L19.9103 12.8914 C20.5449 13.1199 20.1683 13.4121 19.0692 13.5441 L9.9717 14.6361 C8.8725 14.7681 7.4672 14.6898 6.8326 14.4612 L2.7351 12.9857 Z", + tent: "M3.2522 2.7267 L18.7478 1.4816 L16.696 9.7017 C16.3782 10.9748 15.0649 12.0915 13.7626 12.1962 L2.983 13.0623 C1.6807 13.167 0.8826 12.2199 1.2004 10.9468 L3.2522 2.7267 Z M3.2522 2.7267 L18.7478 1.4816 L20.7996 10.6909 C21.1174 12.1172 20.3193 13.3581 19.017 13.4628 L8.2374 14.3289 C6.9351 14.4336 5.6218 13.3623 5.304 11.936 L3.2522 2.7267 Z", +} as const; + +/** A line drawing of a folding phone in one stance, sized like a toolbar icon. */ +export function DeviceDuoGlyph(props: { readonly pose: DuoPose; readonly className?: string }) { + const { pose } = props; + const stance = pose === "laptop" || pose === "tent"; + return ( + + {stance ? : null} + {pose === "closed" ? ( + <> + + + + ) : null} + {pose === "book" ? ( + + ) : null} + {pose === "open" ? ( + <> + + + + ) : null} + + ); +} diff --git a/apps/web/src/components/device/DeviceEventLog.tsx b/apps/web/src/components/device/DeviceEventLog.tsx new file mode 100644 index 000000000..7ca9ff236 --- /dev/null +++ b/apps/web/src/components/device/DeviceEventLog.tsx @@ -0,0 +1,77 @@ +import { + appendDeviceEventLog, + type DeviceEventLogEntry, + type DeviceHubTransport, + subscribeDeviceEventLog, +} from "@threadlines/client-runtime/device"; +import { useEffect, useState } from "react"; + +/** "14:02:07" from an event's ISO time; the raw text when it is not one. */ +function clockTime(timestamp: string): string { + const date = new Date(timestamp); + if (Number.isNaN(date.getTime())) return timestamp; + return date.toLocaleTimeString(undefined, { + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + hour12: false, + }); +} + +/** + * An iOS Simulator's recent events from serve-sim, newest first. Follows the + * log while mounted (the Tools drawer is open) and cancels the read when it + * unmounts. Mount it keyed by device. + */ +export function DeviceEventLog(props: { + readonly transport: DeviceHubTransport; + readonly deviceId: string; +}) { + const { transport, deviceId } = props; + const [entries, setEntries] = useState | null>(null); + const [failed, setFailed] = useState(false); + useEffect( + () => + subscribeDeviceEventLog( + transport, + deviceId, + (incoming, reset) => + setEntries((current) => appendDeviceEventLog(current ?? [], incoming, reset)), + setFailed, + ), + [deviceId, transport], + ); + + if (entries === null) { + return ( +

+ {failed ? "The event log is not answering. Trying again…" : "Reading the event log…"} +

+ ); + } + if (entries.length === 0) { + return

No events yet.

; + } + return ( +
    + {entries.toReversed().map((entry) => ( +
  1. + + + {entry.kind ? {entry.kind} : null} + {entry.summary} + +
  2. + ))} +
+ ); +} diff --git a/apps/web/src/components/device/DeviceFloatingWindow.tsx b/apps/web/src/components/device/DeviceFloatingWindow.tsx new file mode 100644 index 000000000..e3a624791 --- /dev/null +++ b/apps/web/src/components/device/DeviceFloatingWindow.tsx @@ -0,0 +1,299 @@ +import type { DeviceSummary, EnvironmentId, ThreadId } from "@threadlines/contracts"; +import { threadDeviceSessions } from "@threadlines/client-runtime/deviceState"; +import { GripVerticalIcon, PanelRightIcon, XIcon } from "lucide-react"; +import { + type PointerEvent, + useCallback, + useEffect, + useRef, + useState, + useSyncExternalStore, +} from "react"; +import { createPortal } from "react-dom"; + +import { useIsMobile } from "~/hooks/useMediaQuery"; +import { cn } from "~/lib/utils"; +import { Button } from "../ui/button"; +import { + DEVICE_FLOAT_DEFAULT_WIDTH, + DEVICE_FLOAT_HEADER_HEIGHT, + DEVICE_FLOAT_MARGIN, + type DeviceFloatEdge, + type DeviceFloatRect, + dragDeviceFloat, + fitDeviceFloat, + resizeDeviceFloat, +} from "./deviceFloatLayout"; +import { deviceScreenAspect } from "./deviceFrameLayout"; +import { useDevicesSupported, useDeviceState } from "./deviceState"; +import { DeviceStreamView } from "./DeviceStreamView"; +import { + dockDeviceFloat, + placeDeviceFloat, + useDeviceFloat, + useThreadDeviceViews, +} from "./deviceViewStore"; +import { useDeviceStream } from "./useDeviceStream"; + +/** Below the chat header, clear of the window's right edge, when first floated. */ +const DEFAULT_TOP_OFFSET = 52; + +/** + * The device floating over a thread's chat, when the phone layout is not in + * use: the phone layout has no room for one, so there the device stays in + * the Device tab. + */ +export function useFloatingDevice(threadKey: string | null) { + const float = useDeviceFloat(threadKey); + const phone = useIsMobile(); + return phone ? null : float; +} + +const subscribeViewport = (listener: () => void) => { + window.addEventListener("resize", listener); + return () => window.removeEventListener("resize", listener); +}; +const readViewportWidth = () => window.innerWidth; +const readViewportHeight = () => window.innerHeight; + +/** + * The floating device window (docs/design/devices.md, "Float the device over + * the chat"). Render it inside the chat column; it measures the column for + * its first spot and draws in a layer over the app, kept inside the window. + * It streams and takes touch while the thread is on screen, whatever the + * right panel shows. Drag it by its title strip, resize it from its sides + * and bottom. "Open in right panel" docks it; close dismisses the float and + * leaves the device running. + */ +export function DeviceFloatingWindow(props: { + readonly environmentId: EnvironmentId; + readonly threadId: ThreadId; + readonly threadKey: string; + readonly onOpenInPanel: (deviceId: string) => void; +}) { + const { environmentId, threadId, threadKey } = props; + const float = useFloatingDevice(threadKey); + const supported = useDevicesSupported(environmentId); + const state = useDeviceState(environmentId, supported && float !== null); + const views = useThreadDeviceViews(threadKey); + // The chat column this renders in, known once mounted; the float's first spot comes from it. + const [column, setColumn] = useState(null); + const measureColumn = useCallback((node: HTMLSpanElement | null) => { + setColumn(node?.parentElement ?? null); + }, []); + + const session = + float && state + ? threadDeviceSessions(state, threadId).find((entry) => entry.deviceId === float.deviceId) + : undefined; + const device = + session && state ? state.devices.find((entry) => entry.id === session.deviceId) : undefined; + // A device closed or gone (here, by an agent, or on another client) takes its float with it. + const gone = float !== null && state !== null && (!supported || !session); + useEffect(() => { + if (gone) dockDeviceFloat(threadKey); + }, [gone, threadKey]); + + return ( + <> +