Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 74 additions & 0 deletions apps/desktop/src/computer/ComputerHostProcess.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ type SentFrame = {
readonly task?: { readonly goal: string; readonly text: string | null };
readonly limits?: { readonly maxSteps: number; readonly timeoutMs: number };
readonly taskId?: string;
readonly grant?: unknown;
};

function fakeChild() {
Expand Down Expand Up @@ -171,6 +172,79 @@ describe("the computer helper process", () => {
expect(process.task).toBeNull();
});

it("a stopped task waits for the helper's end, forwards it, and grants nothing meanwhile", async () => {
const child = fakeChild();
const process = startComputerHostProcess({
workerPath: "/app/computer.cjs",
fork: (() => child as unknown as ComputerHostChild) as never,
});
let grantLate: (grant: { ok: true; grant: "allow" }) => void = () => undefined;
const running = task({
ask: {
authorize: vi.fn(
() =>
new Promise<{ ok: true; grant: "allow" }>((resolve) => {
grantLate = resolve;
}),
),
decide: vi.fn().mockResolvedValue({}),
event: vi.fn().mockResolvedValue(undefined),
},
});
const started = process.start(running);
deliver(child, { id: "task-1", kind: "started", taskId: "task-1" });
await started;
const step = (id: string) => ({
id,
kind: "authorize",
taskId: "task-1",
request: {
step: 1,
observation: { app: "Safari", title: "Example" },
action: { kind: "scroll", direction: "down" },
},
});

deliver(child, step("grant_1"));
await vi.waitFor(() => expect(running.ask.authorize).toHaveBeenCalledTimes(1));
process.stop({ taskId: "task-1", reason: "You stopped it." });
expect(process.task).toEqual({ environmentId: "environment-a", taskId: "task-1" });

// The environment's grant arrives after the stop: the helper is refused all the same.
grantLate({ ok: true, grant: "allow" });
await vi.waitFor(() =>
expect(sent(child).find((frame) => frame.id === "grant_1")).toMatchObject({
kind: "grant",
grant: { ok: false, denied: "It was stopped." },
}),
);
// A step asked for while stopping is refused here, without asking the environment.
deliver(child, step("grant_2"));
await vi.waitFor(() =>
expect(sent(child).find((frame) => frame.id === "grant_2")).toMatchObject({
grant: { ok: false, denied: "It was stopped." },
}),
);
expect(running.ask.authorize).toHaveBeenCalledTimes(1);

deliver(child, {
id: "task-1",
kind: "finished",
taskId: "task-1",
outcome: { status: "stopped", reason: "You stopped it.", steps: [], surface: null },
});
await vi.waitFor(() =>
expect(running.ask.event).toHaveBeenCalledWith({
type: "finished",
status: "stopped",
reason: "You stopped it.",
steps: [],
surface: null,
}),
);
expect(process.task).toBeNull();
});

it("asks the environment for each step's grant and answers the helper with it", async () => {
const child = fakeChild();
const process = startComputerHostProcess({
Expand Down
45 changes: 40 additions & 5 deletions apps/desktop/src/computer/ComputerHostProcess.ts
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ export type ComputerHostStartResult =

export type ComputerHostProcess = {
readonly alive: boolean;
/** The task running on this Mac, if one is. */
/** The task running or stopping on this Mac, if one is. */
readonly task: { readonly environmentId: string; readonly taskId: string } | null;
readonly access: () => Promise<ComputerAccess>;
readonly start: (task: ComputerHostTask) => Promise<ComputerHostStartResult>;
Expand Down Expand Up @@ -278,7 +278,13 @@ type Pending = {
readonly deadline: ReturnType<typeof setTimeout>;
};

type ActiveTask = ComputerHostTask & { readonly deadline: ReturnType<typeof setTimeout> };
type ActiveTask = ComputerHostTask & {
readonly deadline: ReturnType<typeof setTimeout>;
/** Told to stop: held until the helper reports its end, and granted nothing meanwhile. */
readonly stopping: boolean;
};

const STOPPED = "It was stopped.";

/**
* The computer-use helper as this app runs it: one forked child, one task at a time, and a deadline on
Expand Down Expand Up @@ -392,8 +398,25 @@ export function startComputerHostProcess(options: ComputerHostProcessOptions): C
return settled;
};

const refuse = (
message: Extract<ChildMessage, { kind: "decide" | "authorize" }>,
denied: string,
) =>
write(
message.kind === "decide"
? { id: message.id, kind: "answers", answers: {} }
: { id: message.id, kind: "grant", grant: { ok: false, denied } },
);

/** A grant that comes back after its task was told to stop is refused, whatever the environment said. */
const late = (taskId: string) => task === null || task.taskId !== taskId || task.stopping;

const answer = (message: Extract<ChildMessage, { kind: "decide" | "authorize" }>) => {
const running = task === null || task.taskId !== message.taskId ? null : task;
if (running?.stopping === true) {
refuse(message, STOPPED);
return;
}
if (running === null) {
// The step outlived its task, so nothing waits for this answer: end the helper's step.
write(
Expand Down Expand Up @@ -425,7 +448,10 @@ export function startComputerHostProcess(options: ComputerHostProcessOptions): C
.then((grant) => ({ id: message.id, kind: "grant" as const, grant }));
void answered
.then(
(frame) => write(frame),
(frame) =>
frame.kind === "grant" && frame.grant.ok && late(message.taskId)
? refuse(message, STOPPED)
: write(frame),
(error: unknown) =>
write(
message.kind === "decide"
Expand Down Expand Up @@ -533,7 +559,7 @@ export function startComputerHostProcess(options: ComputerHostProcessOptions): C
}, taskTimeoutMs);
deadline.unref?.();
// Held from the request on, so a second start racing this one is busy, not sent to the helper.
task = { ...next, deadline };
task = { ...next, deadline, stopping: false };
const started = await request<Extract<ChildMessage, { kind: "started" }>>(
{
id: next.taskId,
Expand Down Expand Up @@ -594,7 +620,16 @@ export function startComputerHostProcess(options: ComputerHostProcessOptions): C
}),
),
stop: (input) => {
if (task?.taskId === input.taskId) clearTask();
// The task stays until the helper reports its end, which is what frees the computer for the server.
if (task?.taskId === input.taskId && !task.stopping) {
clearTimeout(task.deadline);
const deadline = setTimeout(() => {
log("computer_host.stop_deadline", { taskId: input.taskId });
kill("The computer helper did not stop.");
}, replyTimeoutMs);
deadline.unref?.();
task = { ...task, deadline, stopping: true };
}
write({
id: `stop_${input.taskId}`,
kind: "stop",
Expand Down
32 changes: 6 additions & 26 deletions apps/desktop/src/computer/DesktopComputerHost.ts
Original file line number Diff line number Diff line change
Expand Up @@ -220,31 +220,14 @@ const make = Effect.gen(function* () {
Electron.app.on("browser-window-focus", onWindowFocus);

/**
* Ends the task on this Mac first, so it works with the network down, then tells the environment:
* the helper's own end is dropped once its task is cleared.
* Ends the task on this Mac, without asking the environment, so it works offline. The environment
* hears the helper's own end, once the helper has let go of the computer.
*/
const stopLocally = () => {
const task = running;
if (task === null) return;
running = null;
releaseStopShortcut();
helper?.stop({ taskId: task.taskId, reason: STOPPED_HERE });
void ask<void>(
(callId) => ({
type: "event",
callId,
environmentId: task.environmentId,
taskId: task.taskId,
event: {
type: "finished",
status: "stopped",
reason: STOPPED_HERE,
steps: [],
surface: null,
},
}),
() => undefined,
);
};

const askAuthorize = (
Expand Down Expand Up @@ -292,9 +275,8 @@ const make = Effect.gen(function* () {
),
event: async (event: ComputerHostEvent) => {
if (event.type === "step" && running?.taskId !== owner.taskId) return;
if (event.type === "finished") {
// A task the server already stopped, or one whose stop came from this Mac, has no listener left.
if (running?.taskId !== owner.taskId) return;
// Always reported: for a task the environment stopped, it confirms the helper let go of the computer.
if (event.type === "finished" && running?.taskId === owner.taskId) {
running = null;
releaseStopShortcut();
}
Expand Down Expand Up @@ -413,10 +395,8 @@ const make = Effect.gen(function* () {
hosted.add(environmentId);
return null;
case "stop": {
if (running?.environmentId === environmentId) {
running = null;
releaseStopShortcut();
}
// `running` clears when the helper reports the task finished, which the environment waits for.
if (running?.environmentId === environmentId) releaseStopShortcut();
helper?.stop({ taskId: next.taskId, reason: next.reason });
return null;
}
Expand Down
49 changes: 35 additions & 14 deletions apps/server/src/computer/ComputerApprovals.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,10 @@ const makeApprovals = Effect.gen(function* () {
it.effect("an approval grants its own step once", () =>
Effect.gen(function* () {
const approvals = yield* makeApprovals;
const approval = yield* approvals.create(binding("on my way"));
const approval = yield* approvals.create({
binding: binding("on my way"),
readerBinding: binding("on my way"),
});
assert.strictEqual(approval.state, "pending");
assert.strictEqual(
yield* approvals.take({ approvalId: approval.id, binding: binding("on my way") }),
Expand Down Expand Up @@ -73,7 +76,7 @@ it.effect("binds the exact action: other text, step, task, window or host cannot
Effect.gen(function* () {
const approvals = yield* makeApprovals;
const own = binding("on my way");
const approval = yield* approvals.create(own);
const approval = yield* approvals.create({ binding: own, readerBinding: own });
yield* approvals.decide({ approvalId: approval.id, approve: true, decidedBy: "web" });

const others: ReadonlyArray<ComputerApprovalBinding> = [
Expand All @@ -100,7 +103,7 @@ it.effect("the digest does not collide when bytes move between fields", () =>
Effect.gen(function* () {
const approvals = yield* makeApprovals;
const own = binding("ab", "c");
const approval = yield* approvals.create(own);
const approval = yield* approvals.create({ binding: own, readerBinding: own });
yield* approvals.decide({ approvalId: approval.id, approve: true, decidedBy: "web" });

assert.strictEqual(
Expand All @@ -119,7 +122,7 @@ it.effect("neither a late tap nor a late take grants anything", () =>
const approvals = yield* makeApprovals;
const own = binding("x");

const late = yield* approvals.create(own);
const late = yield* approvals.create({ binding: own, readerBinding: own });
yield* TestClock.adjust("60 seconds");
assert.strictEqual(
yield* approvals.decide({ approvalId: late.id, approve: true, decidedBy: "web" }),
Expand All @@ -128,7 +131,7 @@ it.effect("neither a late tap nor a late take grants anything", () =>
assert.strictEqual(yield* approvals.take({ approvalId: late.id, binding: own }), "expired");

// Approved in time but taken too late: still expired.
const takenLate = yield* approvals.create(own);
const takenLate = yield* approvals.create({ binding: own, readerBinding: own });
yield* TestClock.adjust("10 seconds");
assert.strictEqual(
yield* approvals.decide({ approvalId: takenLate.id, approve: true, decidedBy: "web" }),
Expand All @@ -142,19 +145,19 @@ it.effect("neither a late tap nor a late take grants anything", () =>
}),
);

it.effect("deny and cancel end it", () =>
it.effect("deny and cancel end it, answered or not", () =>
Effect.gen(function* () {
const approvals = yield* makeApprovals;
const own = binding("x");

const denied = yield* approvals.create(own);
const denied = yield* approvals.create({ binding: own, readerBinding: own });
assert.strictEqual(
yield* approvals.decide({ approvalId: denied.id, approve: false, decidedBy: "web" }),
"ok",
);
assert.strictEqual(yield* approvals.take({ approvalId: denied.id, binding: own }), "denied");

const cancelled = yield* approvals.create(own);
const cancelled = yield* approvals.create({ binding: own, readerBinding: own });
yield* approvals.cancelTask(own.taskId);
assert.strictEqual(
yield* approvals.take({ approvalId: cancelled.id, binding: own }),
Expand All @@ -165,6 +168,15 @@ it.effect("deny and cancel end it", () =>
"not_pending",
);

// A tap already given but not yet used is revoked too: it cannot outlive its task.
const answered = yield* approvals.create({ binding: own, readerBinding: own });
yield* approvals.decide({ approvalId: answered.id, approve: true, decidedBy: "web" });
yield* approvals.cancelTask(own.taskId);
assert.strictEqual(
yield* approvals.take({ approvalId: answered.id, binding: own }),
"cancelled",
);

assert.strictEqual(
yield* approvals.decide({ approvalId: "appr_nope", approve: true, decidedBy: "web" }),
"unknown",
Expand All @@ -176,11 +188,14 @@ it.effect("deny and cancel end it", () =>
it.effect("lists pending approvals first, then finished ones, newest first", () =>
Effect.gen(function* () {
const approvals = yield* makeApprovals;
const first = yield* approvals.create(binding("first"));
const first = yield* approvals.create({
binding: binding("first"),
readerBinding: binding("first"),
});
yield* approvals.decide({ approvalId: first.id, approve: false, decidedBy: "web" });
yield* approvals.create(binding("second"));
yield* approvals.create({ binding: binding("second"), readerBinding: binding("second") });

const listed = yield* approvals.list;
const listed = yield* approvals.list({ revealTypedText: true });
assert.deepStrictEqual(
listed.map((approval) => approval.state),
["pending", "denied"],
Expand All @@ -192,9 +207,15 @@ it.effect("lists pending approvals first, then finished ones, newest first", ()
it.effect("refuses to hold more than 256 approvals", () =>
Effect.gen(function* () {
const approvals = yield* makeApprovals;
for (let index = 0; index < 256; index += 1) yield* approvals.create(binding(`text ${index}`));

const error = yield* approvals.create(binding("one too many")).pipe(Effect.flip);
for (let index = 0; index < 256; index += 1)
yield* approvals.create({
binding: binding(`text ${index}`),
readerBinding: binding(`text ${index}`),
});

const error = yield* approvals
.create({ binding: binding("one too many"), readerBinding: binding("one too many") })
.pipe(Effect.flip);
assert.strictEqual(error._tag, "ComputerApprovalLimitError");
assert.strictEqual(error.limit, 256);
assert.instanceOf(error, ComputerApprovalLimitError);
Expand Down
Loading
Loading