From b037fefe3ce25adce0e8d3ac7eddb4897c7a5e30 Mon Sep 17 00:00:00 2001 From: JJ Eaton <58495321+jayleaton@users.noreply.github.com> Date: Sat, 21 Nov 2026 12:11:41 +0700 Subject: [PATCH] fix(computer): redact typed text before any clip, cap or slice Round 5 review found several places that cut a line before hiding the typed text, so a cut could keep a piece of it that no match then found. - Helper: clip() takes the run's secret and hides it before cutting, so every control description, tab title, page text, surface and popup passes through one redact-then-clip path; every question set goes out through ask(), which scrubs state and questions. - Server: redactThenClip/redactThenCap replace clip-then-hide in task ends, host events, refusals and approval bindings; readerBinding redacts every optional action field; clipped echoes whose cut ends in a prefix of the typed text count as echoes. - Quoted labels exempt short text only when {text} stands outside short label-only quotes, and word boundaries apply only at alphanumeric ends. Tests run a long, prefixed, mixed-case secret through the decider, snapshots, status, audit and errors, cover the quoted-label cases, and guard a decision raced against an in-flight type approval. Co-Authored-By: Claude Opus 5.5 --- .../src/computer/ComputerService.test.ts | 159 +++++++++++++++++- apps/server/src/computer/ComputerService.ts | 125 +++++++++----- .../computer-helper/src/controller.test.ts | 27 +++ packages/computer-helper/src/controller.ts | 79 +++++---- 4 files changed, 311 insertions(+), 79 deletions(-) diff --git a/apps/server/src/computer/ComputerService.test.ts b/apps/server/src/computer/ComputerService.test.ts index b2e6ed96f1f85..0ace821591085 100644 --- a/apps/server/src/computer/ComputerService.test.ts +++ b/apps/server/src/computer/ComputerService.test.ts @@ -237,6 +237,8 @@ const startHost = ( readonly confirmStops?: boolean; /** Runs before a control is answered, so a test can hold a control in flight. */ readonly beforeControlReply?: Effect.Effect; + /** The host refuses every task with this error. */ + readonly startError?: string; /** Told whether the server took each control's reply. */ readonly controlReplied?: Deferred.Deferred; } = {}, @@ -278,7 +280,9 @@ const startHost = ( owner, reply: { requestId: frame.requestId, - ok: true, + ...(frame.kind === "start" && options.startError !== undefined + ? { ok: false, error: options.startError } + : { ok: true }), ...(frame.kind === "control" ? { done: "Done." } : {}), ...(frame.kind === "look" ? { look: CALCULATOR } : {}), }, @@ -1882,3 +1886,156 @@ it.effect( ); }).pipe(Effect.provide(layerTest())), ); + +const SECRET = `Sk-Live-${"Ab".repeat(40)}`; +const PAD = "p".repeat(68); +/** A piece from the middle of the secret, which no cut of a leak can avoid. */ +const leaks = (value: unknown) => + JSON.stringify(value).toLowerCase().includes(SECRET.slice(10, 30).toLowerCase()); + +it.effect("a long, prefixed, mixed-case secret reaches no surface but its own type step", () => { + const sent: Array<{ readonly state: unknown; readonly questions: unknown }> = []; + return Effect.gen(function* () { + const service = yield* ComputerService.ComputerService; + const audit = yield* ComputerAudit.ComputerAudit; + const host = yield* startHost(); + const task = yield* startUse({ goal: "store {text} in the vault", text: SECRET }); + const reader = yield* service.subscribe({ revealTypedText: false }); + const approver = yield* service.subscribe({ revealTypedText: true }); + + yield* host.decide({ + taskId: task.taskId, + state: { + screen: { + app: "Vault", + title: `${PAD} ${SECRET}`, + visibleText: `${PAD} ${SECRET.toUpperCase()}`, + }, + }, + questions: { + [`type ${SECRET}`]: { + type: "choice", + instructions: `Put ${SECRET.toLowerCase()} in`, + criteria: { + c0: `Click the textbox “Token” containing “Bearer ${SECRET.slice(0, 50)}…”`, + [SECRET]: "Type it", + }, + }, + }, + }); + const say = (event: Parameters[0]["event"]) => + service.hostEvent({ hostId: host.hostId, owner: host.owner, taskId: task.taskId, event }); + yield* say({ type: "step", text: `${PAD} ${SECRET}` }); + yield* say({ type: "acted", kind: "type", mode: "background", how: `${PAD} ${SECRET}` }); + yield* host.authorize({ + taskId: task.taskId, + step: 1, + action: { kind: "scroll", direction: "down", ref: SECRET, role: SECRET, what: SECRET }, + }); + const tap = yield* service + .hostAuthorize({ + hostId: host.hostId, + owner: host.owner, + taskId: task.taskId, + step: 2, + observation: { + app: `${PAD} ${SECRET}`, + title: `${"t".repeat(380)} ${SECRET}`, + url: `https://x.test/${PAD}?k=${SECRET}`, + }, + action: click(`${PAD} ${SECRET}`), + }) + .pipe(Effect.forkScoped); + const approval = yield* waitForApproval(task.taskId); + const readerSnapshot = yield* Stream.runHead(reader); + assert.isFalse(leaks(readerSnapshot), "reader snapshot"); + yield* service.respondToApproval({ approvalId: approval.id, approve: true, decidedBy: "web" }); + yield* Fiber.join(tap); + yield* say({ + type: "finished", + status: "done", + reason: `${PAD} ${SECRET}`, + steps: [], + surface: `${PAD} ${SECRET}`, + }); + + assert.isFalse(leaks(sent), "decider"); + assert.isFalse(leaks(yield* service.status(null)), "status the MCP tools read"); + const approverTasks = yield* Stream.runHead(approver); + assert.isFalse( + leaks(approverTasks._tag === "Some" ? approverTasks.value.tasks : []), + "approver tasks", + ); + assert.isFalse(leaks(yield* audit.tail(200)), "audit"); + }).pipe(Effect.provide(layerTest({ decider: asking(sent) }))); +}); + +it.effect("a host's refusal never repeats the typed text", () => + Effect.gen(function* () { + const audit = yield* ComputerAudit.ComputerAudit; + yield* startHost({}, { startError: `could not type ${PAD} ${SECRET}` }); + const error = yield* startUse({ goal: "store {text}", text: SECRET }).pipe(Effect.flip); + assert.isFalse(leaks(error.detail), "error"); + const service = yield* ComputerService.ComputerService; + assert.isFalse(leaks(yield* service.status(null)), "status"); + assert.isFalse(leaks(yield* audit.tail(50)), "audit"); + }).pipe(Effect.provide(layerTest())), +); + +it.effect("a quoted label exempts short text only when {text} stands outside the quotes", () => + Effect.gen(function* () { + yield* startHost(); + for (const [goal, text] of [ + ['click "{text} OK" then send', "ok"], + [`click “OK, then type this into the box and press send”, then {text}`, "ok"], + ['press “OK" then {text}', "ok"], + ["say a. now", "a."], + ["dial 1 4 now", " 4"], + ["shout hi!1 loudly", "!1"], + ] as const) { + assert.strictEqual( + (yield* startUse({ goal, text }).pipe(Effect.flip)).reason, + "refused", + goal, + ); + } + const allowed = yield* startUse({ goal: "click “OK”, then type {text}", text: "ok" }); + assert.strictEqual(allowed.status, "running"); + }).pipe(Effect.provide(layerTest())), +); + +it.effect( + "a decision waits while a type step's tap is in flight, and redacts once it is granted", + () => { + const sent: Array<{ readonly state: unknown; readonly questions: unknown }> = []; + return Effect.gen(function* () { + const service = yield* ComputerService.ComputerService; + const host = yield* startHost(); + const task = yield* startUse({ goal: "put {text} in Reply", text: "ok" }); + const typing = yield* host + .authorize({ + taskId: task.taskId, + step: 1, + action: { kind: "type", ref: "r1", role: "textbox", label: "Reply", text: "ok" }, + }) + .pipe(Effect.forkScoped); + const approval = yield* waitForApproval(task.taskId); + const look = host.decide({ + taskId: task.taskId, + state: { screen: { app: "Messages", title: "Dana", visibleText: "Reply: ok" } }, + questions: { done: { type: "noul", instructions: "Done?" } }, + }); + assert.strictEqual((yield* look.pipe(Effect.flip)).reason, "refused"); + + yield* service.respondToApproval({ + approvalId: approval.id, + approve: true, + decidedBy: "web", + }); + assert.deepStrictEqual(yield* Fiber.join(typing), { ok: true, grant: "approved" }); + yield* look; + const state = sent.at(-1)?.state as { screen: { visibleText: string } }; + assert.strictEqual(state.screen.visibleText, ""); + }).pipe(Effect.provide(layerTest({ decider: asking(sent) }))); + }, +); diff --git a/apps/server/src/computer/ComputerService.ts b/apps/server/src/computer/ComputerService.ts index b92f603ca7ec0..f49a62cef6fe7 100644 --- a/apps/server/src/computer/ComputerService.ts +++ b/apps/server/src/computer/ComputerService.ts @@ -369,8 +369,16 @@ const escapeRegExp = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\ /** An echoed value that holds the text, or the start of it cut short with "…" as the helper clips values. */ function echoes(echoed: string, typed: string): boolean { if (holds(echoed, typed)) return true; - const cut = echoed.endsWith("…") ? echoed.slice(0, -1) : ""; - return cut !== "" && typed.toLowerCase().startsWith(cut.toLowerCase()); + if (!echoed.endsWith("…")) return false; + // A cut value can end partway into the text after other words ("Bearer sk-…"): any tail of it that starts + // the typed text is an echo. + const cut = echoed.slice(0, -1).toLowerCase(); + const start = typed.toLowerCase(); + const least = Math.min(MIN_HIDDEN, start.length); + for (let from = 0; from <= cut.length - least; from += 1) { + if (start.startsWith(cut.slice(from))) return true; + } + return false; } /** @@ -401,11 +409,16 @@ function hideEchoed(value: string, typed: string | null, alreadyTyped: boolean): /** How a goal refers to the task's `text` without writing it. */ const TEXT_REFERENCE = "{text}"; /** A quoted span in a goal: a control label as the agent copied it. */ -const QUOTED = /“[^”]*”|"[^"]*"/g; +const QUOTED = /“[^“”"]{1,40}”|"[^“”"]{1,40}"/g; /** Short text standing alone, not inside a longer word. */ -const standsAlone = (value: string, typed: string) => - new RegExp(`(?= MIN_HIDDEN) return null; - const outside = goal.includes(TEXT_REFERENCE) ? goal.replace(QUOTED, "") : goal; + const unquoted = goal.replace(QUOTED, ""); + const outside = unquoted.includes(TEXT_REFERENCE) ? unquoted : goal; return standsAlone(outside, typed) ? `Write ${TEXT_REFERENCE} where the goal needs the text to type, and quote a control label that matches it.` : null; @@ -435,6 +449,14 @@ function shownGoal(goal: string, typed: string | null): string { const utf8Bytes = (value: string) => new TextEncoder().encode(value).byteLength; +/** + * The typed text hidden, then the cut. Every cut of a string that can hold the text goes through one of these: + * cutting first can split the text, and redaction then misses the part that is left. + */ +const redactThenClip = (value: string, typed: string | null) => clip(hideTyped(value, typed)); +const redactThenCap = (value: string, typed: string | null, max: number) => + capBytes(hideTyped(value, typed), max); + /** At most `max` UTF-8 bytes, cut on a character. */ function capBytes(value: string, max: number): string { const encoded = new TextEncoder().encode(value); @@ -582,29 +604,25 @@ function readerBinding( typed: string | null, alreadyTyped: boolean, ): ComputerApprovalBinding { - const { text: _text, ...action } = binding.action; - const hidden = (field: "label" | "name" | "fieldLabel" | "why" | "url" | "path") => { - const value = action[field]; - if (value === undefined) return {}; - return { - [field]: field === "url" ? hideEchoed(value, typed, alreadyTyped) : hideTyped(value, typed), - }; - }; + const { text: _text, kind, ...rest } = binding.action; + // Every field the screen supplied, whatever it is called, so a new one cannot slip past. + const action = Object.fromEntries( + Object.entries(rest).map(([field, value]) => [ + field, + typeof value !== "string" + ? value + : field === "url" + ? hideEchoed(value, typed, alreadyTyped) + : hideTyped(value, typed), + ]), + ); return { ...binding, - app: hideTyped(binding.app, typed), - title: hideEchoed(binding.title, typed, alreadyTyped), - url: hideEchoed(binding.url, typed, alreadyTyped), + app: redactThenCap(binding.app, typed, MAX_BINDING_FIELD_BYTES), + title: capBytes(hideEchoed(binding.title, typed, alreadyTyped), MAX_BINDING_FIELD_BYTES), + url: capBytes(hideEchoed(binding.url, typed, alreadyTyped), MAX_BINDING_URL_BYTES), why: hideTyped(binding.why, typed), - action: { - ...action, - ...hidden("label"), - ...hidden("name"), - ...hidden("fieldLabel"), - ...hidden("why"), - ...hidden("url"), - ...hidden("path"), - }, + action: { kind, ...action }, }; } @@ -702,7 +720,7 @@ function describeAct(act: ComputerAct, typed: string | null): string { switch (act.action) { case "click": return act.label - ? `click “${clip(act.label)}”` + ? `click “${redactThenClip(act.label, typed)}”` : act.ref ? `click ${act.ref}` : `click at ${act.x ?? 0},${act.y ?? 0}`; @@ -713,11 +731,11 @@ function describeAct(act: ComputerAct, typed: string | null): string { case "scroll": return `scroll ${act.direction ?? "down"}`; case "menu": - return `open the menu of ${act.label ? `“${clip(act.label)}”` : (act.ref ?? "a control")}`; + return `open the menu of ${act.label ? `“${redactThenClip(act.label, typed)}”` : (act.ref ?? "a control")}`; case "open_url": - return `open ${clip(act.url ?? "a page")}`; + return `open ${redactThenClip(act.url ?? "a page", typed)}`; case "choose_file": - return `choose the file ${clip(act.path ?? "")}`; + return `choose the file ${redactThenClip(act.path ?? "", typed)}`; } } @@ -1061,8 +1079,11 @@ const make = Effect.gen(function* () { const finished: Task = { ...waitEnded(task, nowMs), status: input.status, - reason: clip(input.reason).slice(0, 600), - surface: input.surface == null ? task.surface : clip(input.surface), + reason: redactThenClip(input.reason, task.text).slice(0, 600), + surface: + input.surface == null + ? task.surface + : clip(hideEchoed(input.surface, task.text, task.typed)), endedAtMs: nowMs, approvalId: null, decision: null, @@ -1349,9 +1370,13 @@ const make = Effect.gen(function* () { }); const waitForTap = Effect.fn("ComputerService.waitForTap")( - function* (task: Task, binding: ComputerApprovalBinding) { + function* ( + task: Task, + binding: ComputerApprovalBinding, + full: ComputerApprovalBinding = binding, + ) { const approval = yield* approvals - .create({ binding, readerBinding: readerBinding(binding, task.text, task.typed) }) + .create({ binding, readerBinding: readerBinding(full, task.text, task.typed) }) .pipe( Effect.mapError((cause) => failure( @@ -1386,7 +1411,7 @@ const make = Effect.gen(function* () { } yield* record(APPROVAL_REQUEST_AUDIT_EVENT, { approvalId: approval.id, - ...auditBinding(binding, task), + ...auditBinding(full, task), }); const woken = yield* Deferred.await(wake).pipe( Effect.timeoutOrElse({ @@ -1570,7 +1595,10 @@ const make = Effect.gen(function* () { return yield* started.failure; } if (!started.success.ok) { - const reason = started.success.error ?? "The computer refused the task."; + const reason = hideTyped( + started.success.error ?? "The computer refused the task.", + task.text, + ); // The host refused it, so it never ran there. yield* finishTask({ taskId: task.id, status: "failed", reason, confirmed: true }); return yield* failure("refused", reason, { taskId: task.id }); @@ -1923,24 +1951,31 @@ const make = Effect.gen(function* () { if (decision.verdict === "refused") return yield* deny(task.id, decision.why); if (decision.verdict === "auto") return yield* grant(task, input, "allow", {}); const mode = yield* refreshApprovalsMode(task); - const binding: ComputerApprovalBinding = { + // Redaction reads the whole observation; only the approver's copy is cut to size here. + const full: ComputerApprovalBinding = { hostId: host.id, taskId: task.id, step: input.step, - app: capBytes(input.observation.app, MAX_BINDING_FIELD_BYTES), - title: capBytes(input.observation.title, MAX_BINDING_FIELD_BYTES), - url: capBytes(input.observation.url ?? "", MAX_BINDING_URL_BYTES), + app: input.observation.app, + title: input.observation.title, + url: input.observation.url ?? "", action: input.action, why: decision.why, goal: shownGoal(task.goal, task.text), requester: task.requester, }; + const binding: ComputerApprovalBinding = { + ...full, + app: capBytes(full.app, MAX_BINDING_FIELD_BYTES), + title: capBytes(full.title, MAX_BINDING_FIELD_BYTES), + url: capBytes(full.url, MAX_BINDING_URL_BYTES), + }; // The real pointer is the person's to lend, whatever the approvals setting. if (mode === "skip" && input.action.kind !== "pointer") { - yield* record(APPROVAL_SKIPPED_AUDIT_EVENT, auditBinding(binding, task)); + yield* record(APPROVAL_SKIPPED_AUDIT_EVENT, auditBinding(full, task)); return yield* grant(task, input, "approved", { approved: true, skipped: true }); } - const outcome = yield* waitForTap(task, binding); + const outcome = yield* waitForTap(task, binding, full); if (outcome._tag === "approved") { return yield* grant(task, input, "approved", { approved: true }); } @@ -2010,13 +2045,13 @@ const make = Effect.gen(function* () { taskId: task.id, kind: input.event.kind, mode: input.event.mode, - how: hideTyped(clip(input.event.how), task.text), + how: redactThenClip(input.event.how, task.text), }); return; } if (input.event.type === "step") { if (task.status !== "running") return; - const text = clip(input.event.text).slice(0, MAX_STEP_TEXT); + const text = redactThenClip(input.event.text, task.text).slice(0, MAX_STEP_TEXT); yield* mutate( (current) => [ @@ -2027,7 +2062,7 @@ const make = Effect.gen(function* () { })), ] as const, ); - yield* record("task.progress", { taskId: task.id, step: hideTyped(text, task.text) }); + yield* record("task.progress", { taskId: task.id, step: text }); return; } yield* finishTask({ diff --git a/packages/computer-helper/src/controller.test.ts b/packages/computer-helper/src/controller.test.ts index 7cd59c9f533d9..b62bc9df94561 100644 --- a/packages/computer-helper/src/controller.test.ts +++ b/packages/computer-helper/src/controller.test.ts @@ -274,6 +274,33 @@ describe("ComputerUse", () => { expect(JSON.stringify(asked.at(-1)!.state)).toContain("[typed text]"); }); + it("hides a long, prefixed typed text before any cut, in tab titles, labels, values and page text", async () => { + const secret = `Sk-Live-${"Ab".repeat(40)}`; + const pad = "p".repeat(20); + const parts = skool(); + const observe = parts.page.observe; + parts.page.observe = async () => { + const screen = await observe(); + return { + ...screen, + title: `${pad} ${secret}`, + text: `${pad} ${secret.toUpperCase()}`, + controls: screen.controls.map((each) => + each.value === undefined ? each : { ...each, value: `${pad} ${secret}` }, + ), + }; + }; + const tabs = parts.tabs; + tabs.list = async () => TABS.map((tab) => ({ ...tab, title: `${pad} ${tab.title} ${secret}` })); + const steps: string[] = []; + const { computer, asked } = computerWith(SKOOL_REPLIES(), parts, allowAll().authorize, { + onStep: (step) => void steps.push(step), + }); + const outcome = await computer.run({ ...TASK, text: secret }); + const piece = secret.slice(10, 30).toLowerCase(); + expect(JSON.stringify([asked, steps, outcome]).toLowerCase()).not.toContain(piece); + }); + it("stops before sending when the server denies the Enter that sends, and says why", async () => { const parts = skool(); const requests: AuthorizeRequest[] = []; diff --git a/packages/computer-helper/src/controller.ts b/packages/computer-helper/src/controller.ts index 555285aa6b911..936069f7211c2 100644 --- a/packages/computer-helper/src/controller.ts +++ b/packages/computer-helper/src/controller.ts @@ -150,16 +150,20 @@ export type RunOptions = { const STATE_NOTE = "`goal` is what the user wants done on screen."; const VISUAL_NOTE = "A screenshot, because the app would not show its window through accessibility: only its text is listed (clicking a text clicks where it is), not icons such as a play button."; -const clip = (text: string, max: number) => (text.length > max ? `${text.slice(0, max)}…` : text); -const host = (url: string) => { +/** At most `max` characters of a line the server reads, the typed text hidden first so no cut can split it. */ +const clip = (text: string, max: number, secret: string | null) => { + const shown = without(text, secret); + return shown.length > max ? `${shown.slice(0, max)}…` : shown; +}; +const host = (url: string, secret: string | null) => { try { return new URL(url).host; } catch { - return url.slice(0, 60); + return clip(url, 60, secret); } }; -const describeControl = (control: ScreenControl) => - `${control.role} “${clip(control.label || "(no label)", 90)}”${control.value ? ` containing “${clip(control.value, 60)}”` : ""}${control.focused ? " (focused)" : ""}${control.window ? ` in the ${control.window}` : ""}`; +const describeControl = (control: ScreenControl, secret: string | null) => + `${control.role} “${clip(control.label || "(no label)", 90, secret)}”${control.value ? ` containing “${clip(control.value, 60, secret)}”` : ""}${control.focused ? " (focused)" : ""}${control.window ? ` in the ${control.window}` : ""}`; const summary = (screen: Observation): ObservationSummary => ({ app: screen.app, title: screen.title, @@ -319,7 +323,7 @@ export class ComputerUse { const key = screenKey(screen); const avoid = avoidBy.get(key) ?? new Map(); avoidBy.set(key, avoid); - surface = `${screen.app}: ${clip(screen.title, 60)}`; + surface = `${screen.app}: ${clip(screen.title, 60, run.secret)}`; // Only going to an address: done once the page on screen is on that host, rather than leaving it to the decider. if (pureNavigation(task.goal) && navigationReached(task.goal, screen)) return end("done", ""); @@ -345,7 +349,7 @@ export class ComputerUse { backend, screen, { kind: "click", ref: direct.ref }, - `clicked the ${describeControl(direct)}`, + `clicked the ${describeControl(direct, run.secret)}`, ); clicked.add(match); if (quoted.every((name) => clicked.has(name)) && plainClickGoal(task)) @@ -440,7 +444,10 @@ export class ComputerUse { if (existing) { browser = existing.browser; await existing.press(); - this.record(run, `reused existing tab: “${clip(existing.tab.title, 60)}” in ${browser}`); + this.record( + run, + `reused existing tab: “${clip(existing.tab.title, 60, run.secret)}” in ${browser}`, + ); } else { browser = await this.deps.urls.open(url); this.record(run, `opened new tab: ${url} in ${browser}`); @@ -466,7 +473,7 @@ export class ComputerUse { null, ); const seen = await backend.observe(this.where(run)); - surface = `${seen.app}: ${clip(seen.title, 60)}`; + surface = `${seen.app}: ${clip(seen.title, 60, run.secret)}`; const screen = await this.reopenPopup(run, backend, seen, act); const chosen = actOn(screen, this.resolveRef(screen, act), task.text); if ("stop" in chosen) return end("stuck", chosen.stop); @@ -484,7 +491,7 @@ export class ComputerUse { if (after.popup) after = await settled(backend, this.where(run), this.clock, () => true, POPUP_SETTLE_MS); if (this.deps.popupSeen) this.deps.popupSeen.text = after.popup ?? ""; - if (after.popup) this.record(run, `popup open now: ${clip(after.popup, 400)}`); + if (after.popup) this.record(run, `popup open now: ${clip(after.popup, 400, run.secret)}`); } return end("done", ""); } catch (error) { @@ -524,7 +531,11 @@ export class ComputerUse { const asked = this.now(); let answers: Answers; try { - answers = await this.deps.decider.ask(state, questions); + // Every question goes out through here, so none can carry the typed text. + answers = await this.deps.decider.ask( + without(state, run.secret), + without(questions, run.secret), + ); } catch (error) { throw new Halt("stuck", `The server could not decide the next step: ${messageOf(error)}`); } finally { @@ -883,7 +894,7 @@ export class ComputerUse { ...Object.fromEntries( tabs.map((tab, index) => [ `t${index}`, - `${clip(tab.title || "(untitled)", 80)} (${host(tab.url)})`, + `${clip(tab.title || "(untitled)", 80, run.secret)} (${host(tab.url, run.secret)})`, ]), ), none: "None of these tabs, or it is not clear which.", @@ -921,11 +932,11 @@ export class ComputerUse { if (tab) { // Clicks the tab in the browser's tab strip with the cursor, like a person would. await this.surfaceChange(run, desktop, "tab", tab.title, () => browser.activate(tab)); - const switched = `${browser.browser}: ${clip(tab.title, 60)}`; + const switched = `${browser.browser}: ${clip(tab.title, 60, run.secret)}`; // Names that refer to the tab were what the tab was chosen by, so they are not looked for on screen afterwards. return { backend: desktop, - step: `clicked the “${clip(tab.title, 60)}” tab (${host(tab.url)}) in ${browser.browser}${named ? ` (it matched “${named.name}” by ${named.via})` : ""}`, + step: `clicked the “${clip(tab.title, 60, run.secret)}” tab (${host(tab.url, run.secret)}) in ${browser.browser}${named ? ` (it matched “${named.name}” by ${named.via})` : ""}`, clicked: [...(named ? [named.name] : []), ...tabNames(task.goal)], switchedTab: switched, }; @@ -944,9 +955,9 @@ export class ComputerUse { if (found) return { backend: desktop, - step: `switched to the “${clip(found, 60)}” tab with ${browser.browser}’s tab search`, + step: `switched to the “${clip(found, 60, run.secret)}” tab with ${browser.browser}’s tab search`, clicked: [wanted, ...tabNames(task.goal)], - switchedTab: `${browser.browser}: ${clip(found, 60)}`, + switchedTab: `${browser.browser}: ${clip(found, 60, run.secret)}`, }; if (tabName) return { @@ -1000,7 +1011,7 @@ export class ComputerUse { app: screen.app, title: screen.title, ...(screen.url ? { url: screen.url } : {}), - visibleText: clip(screen.text ?? "", 1_500), + visibleText: clip(screen.text ?? "", 1_500, run.secret), ...(screen.popup ? { popup: screen.popup } : {}), ...(screen.visual ? { readFrom: VISUAL_NOTE } : {}), }, @@ -1022,7 +1033,8 @@ export class ComputerUse { ...Object.fromEntries( controls .map( - (control, index) => [`c${index}`, `Click the ${describeControl(control)}`] as const, + (control, index) => + [`c${index}`, `Click the ${describeControl(control, run.secret)}`] as const, ) .filter(([key]) => { const control = controls[Number(key.slice(1))]; @@ -1066,7 +1078,10 @@ export class ComputerUse { instructions: `If typing now, which text box should the prepared text go into? ${STATE_NOTE}`, criteria: { ...Object.fromEntries( - textboxes.map(({ control, index }) => [`c${index}`, describeControl(control)]), + textboxes.map(({ control, index }) => [ + `c${index}`, + describeControl(control, run.secret), + ]), ), none: "None of these.", }, @@ -1074,11 +1089,7 @@ export class ComputerUse { } : {}), }; - const answers = await this.ask( - run, - { ...without({ ...state, goal: "" }, run.secret), goal: task.goal }, - without(questions, run.secret), - ); + const answers = await this.ask(run, { ...state, goal: task.goal }, questions); const next = this.confident(answers, "next"); if (!next || next === "stuck") return { stop: "stuck", reason: `It could not tell what to do next on ${screen.app}.` }; @@ -1101,7 +1112,7 @@ export class ComputerUse { if (!box) return { stop: "stuck", reason: "It could not tell which text box to type into." }; return { action: { kind: "type", ref: box.ref, text: task.text }, - step: `typed the text into the ${describeControl(box)}`, + step: `typed the text into the ${describeControl(box, run.secret)}`, }; } const control = /^c\d+$/.test(next) ? controls[Number(next.slice(1))] : undefined; @@ -1110,12 +1121,12 @@ export class ComputerUse { if (control.role === "textbox" && control.focused && canType && task.text) { return { action: { kind: "type", ref: control.ref, text: task.text }, - step: `typed the text into the ${describeControl(control)}`, + step: `typed the text into the ${describeControl(control, run.secret)}`, }; } return { action: { kind: "click", ref: control.ref }, - step: `clicked the ${describeControl(control)}`, + step: `clicked the ${describeControl(control, run.secret)}`, }; } } @@ -1161,10 +1172,12 @@ function actOn( return { screen, action: { kind: "pick", label: act.label }, - said: `picked “${clip(act.label, 60)}” from the menu`, + said: `picked “${clip(act.label, 60, text)}” from the menu`, }; if (act.label !== undefined && !byRef && !named) - return { stop: `No single control “${clip(act.label, 60)}” is on screen; look again.` }; + return { + stop: `No single control “${clip(act.label, 60, text)}” is on screen; look again.`, + }; const at = act.x !== undefined && act.y !== undefined ? { x: Math.round(act.x), y: Math.round(act.y) } @@ -1174,7 +1187,7 @@ function actOn( return { screen, action: { kind: "click", ref: target.ref }, - said: `clicked the ${describeControl(target)}`, + said: `clicked the ${describeControl(target, text)}`, }; if (!at) return { stop: "Name the control to click: its ref, its label, or x and y." }; const spot: ScreenControl = { @@ -1195,7 +1208,7 @@ function actOn( return { screen, action: { kind: "menu", ref: target.ref }, - said: `opened the menu of the ${describeControl(target)}`, + said: `opened the menu of the ${describeControl(target, text)}`, }; } case "type": { @@ -1205,7 +1218,7 @@ function actOn( return { screen, action: { kind: "type", ref: box.ref, text }, - said: `typed the text into the ${describeControl(box)}`, + said: `typed the text into the ${describeControl(box, text)}`, }; } case "key": { @@ -1234,7 +1247,7 @@ function actOn( return { screen, action: { kind: "choose_file", path: act.path }, - said: `chose the file “${clip(act.path, 120)}”`, + said: `chose the file “${clip(act.path, 120, text)}”`, }; case "open_url": return { stop: "Opening a web address is not a step on the screen." };