diff --git a/.github/workflows/lazurio-fork-ci.yml b/.github/workflows/lazurio-fork-ci.yml index 7968da0af5e74..18ee37c8797fd 100644 --- a/.github/workflows/lazurio-fork-ci.yml +++ b/.github/workflows/lazurio-fork-ci.yml @@ -440,11 +440,18 @@ jobs: apps/web/src/components/ThreadNotificationCoordinator.tsx apps/web/src/state/entities.ts apps/web/src/hooks/useActiveProjectTarget.ts + # A thread waiting on a question wakes whoever waits on it (row 39): t3_thread_wait's + # untilQuestion in the contract and ThreadManagementService (listed above), and the + # remote delegation follower that wakes the parent here, which the orchestrator keeps. + packages/contracts/src/orchestratorMcp.ts + apps/server/src/peer/RemoteDelegation.ts + apps/server/src/peer/RemoteDelegation.test.ts # Where a message came from, for a linked environment's thread too (row 40): the # receiving server records the link's label with the sender (OrchestratorMcpService.ts # and orchestrationV2.ts, listed above), and the tests that see it. apps/server/src/mcp/OrchestratorMcpService.test.ts apps/server/src/peer/PeerForwarding.test.ts + # The tool list grows by the two optional parameters and the wait result's field. apps/server/src/provider/agentContextBudget.json ) unexpected="$(git diff --name-only "$UPSTREAM_SHA..HEAD" | diff --git a/apps/server/src/mcp/OrchestratorMcpService.test.ts b/apps/server/src/mcp/OrchestratorMcpService.test.ts index 6fbfcabb3abd3..fca8708096f5b 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.test.ts @@ -3045,6 +3045,8 @@ describe("OrchestratorMcpService provider resolution", () => { return { taskId: parent.subagents[0]!.id }; }).pipe(Effect.orDie), runStatus: () => Effect.succeed(null), + pendingQuestionIds: () => Effect.succeed([]), + subscribeQuestionChanges: Effect.succeed(Stream.never), }); }), ).pipe(Layer.provide(layerWatchedThreads)); diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 02bf1f1d01ad7..165b664406e1a 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -2032,8 +2032,9 @@ const make = Effect.gen(function* () { : null, latestTerminalSummary: terminal ? task.result : null, latestTerminalResultContextTransferId: null, - // Its questions and approvals are answered in the environment it runs in. - hasPendingQuestion: false, + // Answered in the environment it runs in; its approvals are not seen from here. + hasPendingQuestion: + !terminal && (yield* remoteDelegation.pendingQuestionIds(task.id)).length > 0, pendingApproval: null, waitTimedOut, } satisfies OrchestratorMcpDelegateTaskResult; @@ -2061,36 +2062,56 @@ const make = Effect.gen(function* () { childThreadId === null ? TASK_WAKE_EVENTS.filter(({ thread }) => thread === "parent") : TASK_WAKE_EVENTS; + // A remote task's questions are seen by its follower, not by any event here; one it saw + // before this subscribed is read below. + const remoteQuestions: Array> = []; + if (childThreadId === null) { + const questionChanges = yield* remoteDelegation.subscribeQuestionChanges; + if ((yield* remoteDelegation.pendingQuestionIds(taskId)).length > 0) { + const asked = yield* readTask(scope, taskId, false, true); + if (endsTaskWait(asked)) return Option.some(asked); + } + remoteQuestions.push( + questionChanges.pipe(Stream.filter((changedTaskId) => changedTaskId === taskId)), + ); + } return yield* Stream.mergeAll( - wakeEvents.map(({ thread, eventType }) => - threadManagement - .streamStoredEventsFrom({ - threadId: - thread === "parent" || childThreadId === null - ? scope.thread.threadId - : childThreadId, - afterSequence, - eventType, - }) - .pipe( - Stream.filter( - ({ event }) => - thread === "child" || - event.type !== "subagent.updated" || - event.payload.id === taskId, + [ + ...wakeEvents.map(({ thread, eventType }) => + threadManagement + .streamStoredEventsFrom({ + threadId: + thread === "parent" || childThreadId === null + ? scope.thread.threadId + : childThreadId, + afterSequence, + eventType, + }) + .pipe( + Stream.filter( + ({ event }) => + thread === "child" || + event.type !== "subagent.updated" || + event.payload.id === taskId, + ), + Stream.mapError(streamError), ), - ), - ), + ), + ...remoteQuestions, + ], { concurrency: "unbounded" }, ).pipe( - Stream.mapError(streamError), Stream.mapEffect(() => readTask(scope, taskId, false, true)), Stream.filter(endsTaskWait), Stream.runHead, ); }, (effect, _scope, _taskId, timeoutMs) => - effect.pipe(Effect.timeoutOption(Duration.millis(timeoutMs)), Effect.map(Option.flatten)), + effect.pipe( + Effect.scoped, + Effect.timeoutOption(Duration.millis(timeoutMs)), + Effect.map(Option.flatten), + ), ); const refExists = (cwd: string, revision: string) => @@ -3472,6 +3493,9 @@ const make = Effect.gen(function* () { Math.max(1, input.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS), ), ...(input.untilDelegatedWorkSettled === true ? { untilDelegatedWorkSettled: true } : {}), + ...(input.untilQuestion === true + ? { untilQuestion: true, knownQuestionIds: input.knownQuestionIds ?? [] } + : {}), }) .pipe(Effect.mapError(threadManagementFailure)); const status = result.run === null ? "idle" : runOutcome(result.run); @@ -3508,6 +3532,10 @@ const make = Effect.gen(function* () { : { resultRun: { runId: result.resultRun.id, status: runOutcome(result.resultRun) }, }), + ...(result.pendingQuestionIds === undefined + ? {} + : { pendingQuestionIds: result.pendingQuestionIds }), + ...(result.returnedFor === undefined ? {} : { returnedFor: result.returnedFor }), } satisfies OrchestratorMcpThreadWaitResult; }), interruptThread: Effect.fn("OrchestratorMcpService.interruptThread")(function* (scope, input) { diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 4ddccc025f351..f17328c6fa83a 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -2340,11 +2340,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio // An unknown task gives no evidence the question is obsolete. if (task === undefined) return false; const childThreadId = task.childThreadId; - if ( - childThreadId === null || - ["completed", "failed", "cancelled", "interrupted"].includes(task.status) - ) - return true; + if (["completed", "failed", "cancelled", "interrupted"].includes(task.status)) return true; + // Only its follower sees a remote task's questions there; it drops their wakes itself. + if (childThreadId === null) return task.remoteChild === undefined; const child = yield* projectionStore .getThreadRecords(childThreadId, ["runtimeRequests"]) .pipe( diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.test.ts b/apps/server/src/orchestration-v2/ThreadManagementService.test.ts index d90e277ab8531..7e1886b0c08e4 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.test.ts @@ -14,6 +14,7 @@ import { ProjectId, ProviderInstanceId, RunId, + RuntimeRequestId, ThreadId, } from "@t3tools/contracts"; import * as Deferred from "effect/Deferred"; @@ -553,6 +554,111 @@ it.effect("waitForThread can wait past an ended run for the thread's delegated w }), ); +it.effect( + "waitForThread with untilQuestion returns once the thread asks, and once it is answered", + () => + Effect.gen(function* () { + const projectId = ProjectId.make("project:thread-management:wait-question"); + const threadId = ThreadId.make("thread:thread-management:wait-question"); + const runId = RunId.make("run:thread-management:wait-question"); + const questionId = RuntimeRequestId.make("request:thread-management:wait-question"); + const events = yield* Queue.unbounded(); + const subscribed = yield* Queue.unbounded(); + const runs = [{ id: runId, ordinal: 1, status: "running" }]; + const question = (status: "pending" | "resolved", responseCapability = "live") => ({ + id: questionId, + kind: "user_input", + status, + responseCapability: { type: responseCapability }, + createdAt: DateTime.makeUnsafe(1_000), + }); + // An approval and a question no answer can reach are not questions to return for. + let runtimeRequests: Array = [ + { ...question("pending"), id: "request:approval", kind: "command" }, + { ...question("pending", "not_resumable"), id: "request:stale" }, + ]; + const stored = (sequence: number, event: object) => + ({ sequence, event: { threadId, ...event } }) as unknown as OrchestrationV2StoredEvent; + const layerTest = ThreadManagementService.layer.pipe( + Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})), + Layer.provide( + Layer.mock(Orchestrator.OrchestratorV2)({ + getThreadEventSequence: () => Effect.succeed(0), + restartContinuationPending: () => Effect.succeed(false), + streamStoredEventsFrom: (input) => + input?.eventType === "runtime-request.updated" + ? Stream.fromEffect(Queue.offer(subscribed, undefined)).pipe( + Stream.drain, + Stream.concat(Stream.fromQueue(events)), + ) + : Stream.never, + getThreadRecords: () => + Effect.sync( + () => + ({ + thread: { id: threadId, projectId, deletedAt: null }, + runs, + runtimeRequests, + }) as unknown as OrchestrationV2ThreadProjection, + ), + }), + ), + Layer.provide(ServerSettings.layerTest()), + Layer.provide(layerReceiptsTest), + ); + const service = yield* ThreadManagementService.ThreadManagementService.pipe( + Effect.provide(layerTest), + ); + const wait = { projectId, threadId, runId, timeoutMs: 60_000 }; + + // Without the option a question changes nothing: the wait runs out as before. + runtimeRequests = [...runtimeRequests, question("pending")]; + const plain = yield* service.waitForThread(wait).pipe(Effect.forkChild); + yield* TestClock.adjust(Duration.minutes(1)); + const plainResult = yield* Fiber.join(plain); + expect(plainResult).toMatchObject({ timedOut: true, run: { id: runId, status: "running" } }); + expect(plainResult).not.toHaveProperty("pendingQuestionIds"); + runtimeRequests = runtimeRequests.slice(0, 2); + + const asked = yield* service + .waitForThread({ ...wait, untilQuestion: true }) + .pipe(Effect.forkChild); + // A wait that returns early fails below instead of hanging here. + yield* Effect.raceFirst(Queue.take(subscribed), Fiber.await(asked)); + runtimeRequests = [...runtimeRequests, question("pending")]; + yield* Queue.offer(events, stored(1, { type: "runtime-request.updated", payload: {} })); + expect(yield* Fiber.join(asked)).toMatchObject({ + timedOut: false, + run: { id: runId, status: "running" }, + pendingQuestionIds: [questionId], + returnedFor: "question", + }); + + // A caller that knows of the question waits until it is answered. + const answered = yield* service + .waitForThread({ ...wait, untilQuestion: true, knownQuestionIds: [questionId] }) + .pipe(Effect.forkChild); + yield* Effect.raceFirst(Queue.take(subscribed), Fiber.await(answered)); + runtimeRequests = [...runtimeRequests.slice(0, 2), question("resolved")]; + yield* Queue.offer(events, stored(2, { type: "runtime-request.updated", payload: {} })); + expect(yield* Fiber.join(answered)).toMatchObject({ + timedOut: false, + run: { id: runId, status: "running" }, + pendingQuestionIds: [], + returnedFor: "question", + }); + + // With no question, the wait runs out and says so. + const quiet = yield* service + .waitForThread({ ...wait, untilQuestion: true }) + .pipe(Effect.forkChild); + yield* TestClock.adjust(Duration.minutes(1)); + const quietResult = yield* Fiber.join(quiet); + expect(quietResult).toMatchObject({ timedOut: true, pendingQuestionIds: [] }); + expect(quietResult).not.toHaveProperty("returnedFor"); + }), +); + it.effect("waitForThread settles delegated work on a thread rolled back past its only run", () => Effect.gen(function* () { const projectId = ProjectId.make("project:thread-management:wait-rolled-back"); diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index fee465bbf8a82..21b721a4e3153 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -29,6 +29,7 @@ import { type OrchestrationV2CreationSource, type OrchestrationV2LinkedSender, type OrchestrationV2Run, + type OrchestrationV2RuntimeRequest, type OrchestrationV2ThreadShellSnapshot, type OrchestrationV2ThreadProjection, type OrchestrationV2ThreadTranscript, @@ -39,6 +40,7 @@ import { type ProviderSessionId, resolveProjectProviderInstanceEnabled, RunId, + type RuntimeRequestId, type ScheduledTaskId, ThreadId, type TurnItemId, @@ -186,6 +188,9 @@ export interface ThreadManagementWaitInput { * completion delivery will wake it again, as a delegated task's result does. */ readonly untilDelegatedWorkSettled?: boolean; + /** Also return once the thread's pending questions differ from `knownQuestionIds` (default none). */ + readonly untilQuestion?: boolean; + readonly knownQuestionIds?: ReadonlyArray; } export interface ThreadManagementWaitResult { @@ -194,6 +199,10 @@ export interface ThreadManagementWaitResult { readonly timedOut: boolean; /** With `untilDelegatedWorkSettled`, once settled: the run with the thread's final answer. */ readonly resultRun?: OrchestrationV2Run; + /** With `untilQuestion`: the thread's pending questions, oldest first. */ + readonly pendingQuestionIds?: ReadonlyArray; + /** With `untilQuestion`, unless timed out: whether the wait settled or the questions changed first. */ + readonly returnedFor?: "settled" | "question"; } export interface ThreadManagementInterruptInput { @@ -557,6 +566,15 @@ export function isTerminalRunStatus( ); } +/** A question its thread waits on that an answer can still reach. */ +function isAnswerableQuestion(request: OrchestrationV2RuntimeRequest): boolean { + return ( + request.kind === "user_input" && + request.status === "pending" && + request.responseCapability.type !== "not_resumable" + ); +} + export function latestRun( projection: Pick, ): OrchestrationV2Run | undefined { @@ -1071,24 +1089,54 @@ const make = Effect.gen(function* () { .pipe(Effect.mapError(loadError)); return pending ? undefined : progress.resultRun; }); + const untilQuestion = input.untilQuestion === true; + const knownQuestionIds = new Set(input.knownQuestionIds ?? []); + const questions = Effect.gen(function* () { + if (!untilQuestion) return { pendingQuestionIds: undefined, questionChanged: false }; + const { runtimeRequests } = yield* getProjectThreadRecords(input, ["runtimeRequests"]); + const pendingQuestionIds = runtimeRequests + .filter(isAnswerableQuestion) + .toSorted( + (left, right) => + DateTime.toEpochMillis(left.createdAt) - DateTime.toEpochMillis(right.createdAt), + ) + .map((request) => request.id); + const questionChanged = + pendingQuestionIds.length !== knownQuestionIds.size || + pendingQuestionIds.some((id) => !knownQuestionIds.has(id)); + return { pendingQuestionIds, questionChanged }; + }); const evaluate = Effect.fn("orchestrationV2.threadManagement.evaluate")(function* ( runs: ReadonlyArray, ) { const current = yield* outcome(runs); + const asked = yield* questions; if (!current.settled || !untilDelegatedWorkSettled) { - return { ...current, resultRun: undefined }; + return { ...current, ...asked, resultRun: undefined }; } const resultRun = yield* delegatedWorkResult; - return { ...current, settled: resultRun !== undefined, resultRun }; + return { ...current, ...asked, settled: resultRun !== undefined, resultRun }; }); + const ends = (current: Effect.Success>) => + current.settled || current.questionChanged; const settledResult = (current: Effect.Success>) => ({ threadId: input.threadId, run: current.run, - timedOut: !current.settled, + timedOut: !ends(current), ...(current.resultRun === undefined ? {} : { resultRun: current.resultRun }), + ...(current.pendingQuestionIds === undefined + ? {} + : { + pendingQuestionIds: current.pendingQuestionIds, + ...(current.settled + ? { returnedFor: "settled" as const } + : current.questionChanged + ? { returnedFor: "question" as const } + : {}), + }), }); const initial = yield* evaluate(target.runs); - if (initial.settled) return settledResult(initial); + if (ends(initial)) return settledResult(initial); // A declined continuation records no event, so its effect settling wakes // the wait too. @@ -1103,6 +1151,7 @@ const make = Effect.gen(function* () { ...(untilDelegatedWorkSettled ? (["subagent.updated", "provider-thread.updated"] as const) : []), + ...(untilQuestion ? (["runtime-request.updated"] as const) : []), ] as const ).map((eventType) => orchestrator.streamStoredEventsFrom({ @@ -1133,7 +1182,7 @@ const make = Effect.gen(function* () { Effect.flatMap((current) => evaluate(current.runs)), ), ), - Stream.filter((current) => current.settled), + Stream.filter(ends), Stream.runHead, Effect.timeoutOption(Duration.millis(Math.max(1, input.timeoutMs))), ); diff --git a/apps/server/src/peer/RemoteDelegation.test.ts b/apps/server/src/peer/RemoteDelegation.test.ts index 90c6eb83baae5..ea9ee3161b26e 100644 --- a/apps/server/src/peer/RemoteDelegation.test.ts +++ b/apps/server/src/peer/RemoteDelegation.test.ts @@ -16,6 +16,7 @@ import { type Project, type RepositoryIdentity, RunId, + RuntimeRequestId, type ServerProvider, ThreadId, TurnItemId, @@ -141,6 +142,10 @@ interface BoxThread { readonly sent: Ref.Ref>; /** Runs as the launch there is answered, before the laptop hears back. */ readonly afterLaunch: Ref.Ref>; + /** The questions the thread there waits on. */ + readonly questions: SubscriptionRef.SubscriptionRef>; + /** The box predates waiting until a question, so it ignores the option and reports none. */ + readonly predatesQuestions: Ref.Ref; } const makeBoxThread = Effect.gen(function* () { @@ -179,6 +184,8 @@ const makeBoxThread = Effect.gen(function* () { releaseStop: yield* Deferred.make(), sent: yield* Ref.make>([]), afterLaunch: yield* Ref.make>(Effect.void), + questions: yield* SubscriptionRef.make>([]), + predatesQuestions: yield* Ref.make(false), } satisfies BoxThread; }); @@ -324,9 +331,56 @@ const boxToolkitLayer = (thread: BoxThread) => { : {}), }; } - const done = yield* Deferred.await(thread.finished).pipe( - Effect.timeoutOption(input.timeoutMs), - ); + const askedFor = + input.untilQuestion === true && !(yield* Ref.get(thread.predatesQuestions)); + const known = input.knownQuestionIds ?? []; + const differs = (ids: ReadonlyArray) => + ids.length !== known.length || ids.some((id) => !known.includes(id)); + const asked = askedFor + ? SubscriptionRef.changes(thread.questions).pipe( + Stream.filter(differs), + Stream.runHead, + Effect.map(Option.getOrThrow), + ) + : Effect.never; + // What it waits for, unless the thread's questions change first. + const unlessAsked = (until: Effect.Effect) => + until.pipe( + Effect.as({ asked: undefined }), + Effect.raceFirst(asked.pipe(Effect.map((ids) => ({ asked: ids })))), + Effect.timeoutOption(input.timeoutMs), + ); + const answer = Effect.fnUntraced(function* ( + waited: Option.Option<{ readonly asked: ReadonlyArray | undefined }>, + result: Omit, + ) { + if (Option.isSome(waited) && waited.value.asked !== undefined) { + return { + threadId: input.threadId, + run: result.run, + timedOut: false, + pendingQuestionIds: waited.value.asked, + returnedFor: "question" as const, + }; + } + return { + threadId: input.threadId, + ...result, + timedOut: Option.isNone(waited), + ...(askedFor + ? { + pendingQuestionIds: yield* SubscriptionRef.get(thread.questions), + ...(Option.isSome(waited) ? { returnedFor: "settled" as const } : {}), + } + : {}), + }; + }); + const done = yield* unlessAsked(Deferred.await(thread.finished)); + if (Option.isSome(done) && done.value.asked !== undefined) { + return yield* answer(done, { + run: { id: runId, status: yield* Ref.get(thread.status) } as OrchestrationV2Run, + }); + } // Only a waiter that asks waits past the run for the thread's own task. if ( Option.isSome(done) && @@ -334,28 +388,28 @@ const boxToolkitLayer = (thread: BoxThread) => { (yield* Ref.get(thread.nested)) ) { yield* Deferred.succeed(thread.nestedWaiting, undefined); - const settled = yield* Deferred.await(thread.nestedSettled).pipe( - Effect.timeoutOption(input.timeoutMs), - ); - return { - threadId: input.threadId, + const settled = yield* unlessAsked(Deferred.await(thread.nestedSettled)); + const result = { run: { id: runId, status: yield* Ref.get(thread.status) } as OrchestrationV2Run, - timedOut: Option.isNone(settled), - ...(Option.isSome(settled) - ? { resultRun: { id: boxLaterRunId, status: "completed" } as OrchestrationV2Run } - : {}), }; + return yield* answer( + settled, + Option.isSome(settled) + ? { + ...result, + resultRun: { id: boxLaterRunId, status: "completed" } as OrchestrationV2Run, + } + : result, + ); } // Unpinned, a wait reports the latest run: a later one once there is one. const later = input.runId === undefined && (yield* Ref.get(thread.laterReply)) !== null; - return { - threadId: input.threadId, + return yield* answer(done, { run: { id: later ? boxLaterRunId : runId, status: later ? "completed" : yield* Ref.get(thread.status), } as OrchestrationV2Run, - timedOut: Option.isNone(done), - }; + }); }), getTimelinePage: (threadId, options) => Effect.gen(function* () { @@ -1053,6 +1107,162 @@ it.effect("a task delegated to the box wakes the laptop's parent when it ends th ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); +const isQuestionWake = (messageId: string) => messageId.startsWith("message:remote-task-question:"); + +/** The parent's first message that wakes it for a remote task's question. */ +const questionWake = (laptopEnv: Effect.Success>) => + laptopEnv.orchestrator + .streamStoredEventsFrom({ + threadId: parentThreadId, + afterSequence: 0, + eventType: "message.updated", + }) + .pipe( + Stream.filter( + (stored) => + stored.event.type === "message.updated" && isQuestionWake(stored.event.payload.id), + ), + Stream.runHead, + Effect.map((stored) => + Option.isSome(stored) && stored.value.event.type === "message.updated" + ? stored.value.event.payload + : undefined, + ), + ); + +/** The parent's run with `runId` once it is cancelled. */ +const runCancelled = (laptopEnv: Effect.Success>, runId: RunId) => + laptopEnv.orchestrator + .streamStoredEventsFrom({ + threadId: parentThreadId, + afterSequence: 0, + eventType: "run.updated", + }) + .pipe( + Stream.filter( + (stored) => + stored.event.type === "run.updated" && + stored.event.payload.id === runId && + stored.event.payload.status === "cancelled", + ), + Stream.runHead, + ); + +it.effect.each(["current", "older"] as const)( + "a question asked by a task there wakes the parent here, and an answer drops the wake (%s box)", + (boxVersion) => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + yield* Ref.set(thread.predatesQuestions, boxVersion === "older"); + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const a = yield* makeLaptop(database, offers); + yield* linkTo(a.links, b, "full-access"); + yield* seedParent(a); + const delegated = yield* delegateToBox(a); + const questionId = RuntimeRequestId.make("request:box-question"); + yield* SubscriptionRef.set(thread.questions, [questionId]); + + if (boxVersion === "current") { + const wake = yield* questionWake(a); + expect(wake?.text).toContain( + `t3_pending_request action read, environmentId ${box.environmentId}, threadId ${delegated.remoteChild.threadId}, and requestId ${questionId}`, + ); + expect(wake?.notification).toMatchObject({ + source: { kind: "delegated_task", taskIds: [delegated.taskId] }, + outcome: "updated", + }); + const asking = yield* a.call("task_status", { taskId: delegated.taskId }); + expect(asking.structuredContent).toMatchObject({ + status: "running", + hasPendingQuestion: true, + }); + + // The wake waits behind the parent's live run. + const queued = (yield* a.orchestrator.getThreadRecords(parentThreadId, [ + "runs", + ])).runs.find((run) => run.status === "queued" && run.userMessageId === wake?.id); + expect(queued).toBeDefined(); + // Answered there before the parent read it: the wake is dropped. + yield* SubscriptionRef.set(thread.questions, []); + expect(Option.isSome(yield* runCancelled(a, queued!.id))).toBe(true); + const answered = yield* a.call("task_status", { taskId: delegated.taskId }); + expect(answered.structuredContent).toMatchObject({ hasPendingQuestion: false }); + } else { + // A box that predates the option reports no question, as before. + const asking = yield* a.call("task_status", { taskId: delegated.taskId }); + expect(asking.structuredContent).toMatchObject({ hasPendingQuestion: false }); + } + + // After the answer, the task ends as it always did. + yield* Ref.set(thread.reply, "Done after the answer."); + yield* Ref.set(thread.status, "completed"); + yield* Deferred.succeed(thread.finished, undefined); + const task = yield* taskResult(a, delegated.taskId); + expect(task).toMatchObject({ status: "completed", result: "Done after the answer." }); + const { messages } = yield* a.orchestrator.getThreadRecords(parentThreadId, ["messages"]); + // One wake for the one question, however often the follower looked. + expect(messages.filter((message) => isQuestionWake(message.id))).toHaveLength( + boxVersion === "current" ? 1 : 0, + ); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + +it.effect("a question there ends a blocking delegate_task wait here without a wake", () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const a = yield* makeLaptop(database, offers); + yield* linkTo(a.links, b, "full-access"); + yield* seedParent(a); + const launched = yield* Deferred.make(); + yield* Ref.set(thread.afterLaunch, Deferred.succeed(launched, undefined).pipe(Effect.asVoid)); + + const waiting = yield* a + .call("delegate_task", { + task: "Run the full suite on the box.", + target: { + environmentId: box.environmentId, + providerInstanceId: instanceId, + model: "gpt-5.4", + }, + mode: "wait", + clientRequestId: "suite-on-box-wait", + }) + .pipe(Effect.forkChild); + yield* Deferred.await(launched); + const questionId = RuntimeRequestId.make("request:box-question"); + yield* SubscriptionRef.set(thread.questions, [questionId]); + const result = yield* Fiber.join(waiting); + expect(result.isError, JSON.stringify(result.content)).toBe(false); + expect(result.structuredContent).toMatchObject({ + status: "running", + childThreadId: null, + hasPendingQuestion: true, + }); + // The blocking wait delivered the question, so no wake was sent for it. + const { messages } = yield* a.orchestrator.getThreadRecords(parentThreadId, ["messages"]); + expect(messages.filter((message) => isQuestionWake(message.id))).toEqual([]); + + yield* SubscriptionRef.set(thread.questions, []); + yield* Ref.set(thread.status, "completed"); + yield* Deferred.succeed(thread.finished, undefined); + const taskId = (result.structuredContent as { taskId: NodeId }).taskId; + expect(yield* taskResult(a, taskId)).toMatchObject({ status: "completed" }); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + it.effect.each(["no active parent run", "a newer run owned by another provider"] as const)( "preserves remote observation guards with %s", (observer) => @@ -1342,6 +1552,56 @@ it.effect("follows an open task again after the laptop restarts", () => ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); +it.effect("a question answered there while the laptop is down drops its wake after a restart", () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const questionId = RuntimeRequestId.make("request:box-question-before-restart"); + // The laptop before the restart is woken for a question, which waits behind the parent's run. + const { delegated, wakeId } = yield* Effect.scoped( + Effect.gen(function* () { + const before = yield* makeLaptop(database, offers); + yield* linkTo(before.links, b, "full-access"); + yield* seedParent(before); + const delegated = yield* delegateToBox(before); + yield* SubscriptionRef.set(thread.questions, [questionId]); + const wake = yield* questionWake(before); + return { delegated, wakeId: wake!.id }; + }), + ); + // Answered there while the laptop is down. + yield* SubscriptionRef.set(thread.questions, []); + + const after = yield* makeLaptop(database, offers); + yield* linkTo(after.links, b, "full-access"); + const queued = (yield* after.orchestrator.getThreadRecords(parentThreadId, [ + "runs", + ])).runs.find((run) => run.status === "queued" && run.userMessageId === wakeId); + expect(queued).toBeDefined(); + yield* after.remote.start(); + expect(Option.isSome(yield* runCancelled(after, queued!.id))).toBe(true); + const answered = yield* after.call("task_status", { taskId: delegated.taskId }); + expect(answered.structuredContent).toMatchObject({ + status: "running", + hasPendingQuestion: false, + }); + + yield* Ref.set(thread.reply, "Done after the answer."); + yield* Ref.set(thread.status, "completed"); + yield* Deferred.succeed(thread.finished, undefined); + expect(yield* taskResult(after, delegated.taskId)).toMatchObject({ + status: "completed", + result: "Done after the answer.", + }); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + it.effect("fails the task when the box revokes the link, and cancel interrupts it there", () => Effect.scoped( Effect.gen(function* () { @@ -1480,6 +1740,17 @@ it.effect("a task whose thread there waits on its own task ends with the answer ); expect(first).toBe("waiting there"); + // A question there meanwhile wakes the parent; the task here waits on, for the answer after. + const questionId = RuntimeRequestId.make("request:box-question-while-settling"); + yield* SubscriptionRef.set(thread.questions, [questionId]); + expect((yield* questionWake(a))?.text).toContain(`requestId ${questionId}`); + const asking = yield* a.call("task_status", { taskId: delegated.taskId }); + expect(asking.structuredContent).toMatchObject({ + status: "running", + hasPendingQuestion: true, + }); + yield* SubscriptionRef.set(thread.questions, []); + // The review reports and wakes the thread there, whose next run answers. yield* Ref.set(thread.laterReply, "The review found two issues; both are fixed."); yield* Deferred.succeed(thread.nestedSettled, undefined); diff --git a/apps/server/src/peer/RemoteDelegation.ts b/apps/server/src/peer/RemoteDelegation.ts index 8fcd240e70577..504e07bfa455f 100644 --- a/apps/server/src/peer/RemoteDelegation.ts +++ b/apps/server/src/peer/RemoteDelegation.ts @@ -16,7 +16,9 @@ import { type ProjectId, type ProviderInstanceId, ProviderDriverKind, + MessageId, type RunId, + RuntimeRequestId, type ThreadId, } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; @@ -27,10 +29,12 @@ import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as PubSub from "effect/PubSub"; import * as Schedule from "effect/Schedule"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as Tool from "effect/ai/Tool"; +import { reportLabel } from "@t3tools/provider-core/server/notification"; import { withExecutedModelNote } from "@t3tools/provider-core/server/subagentProjection"; import type * as McpInvocationContext from "../mcp/McpInvocationContext.ts"; @@ -129,6 +133,10 @@ export class RemoteDelegation extends Context.Service< scope: ThreadScope, task: OrchestrationV2Subagent, ) => Effect.Effect; + /** The questions the task's thread there waits on, as its follower last saw them. */ + readonly pendingQuestionIds: (taskId: NodeId) => Effect.Effect>; + /** From now on, the id of each task whose pending questions there changed. */ + readonly subscribeQuestionChanges: Effect.Effect, never, Scope.Scope>; /** * Follows every open remote task until it ends, including after a * restart, and a finished one again when a Stop here owes it a stop there. @@ -739,6 +747,175 @@ const make = Effect.gen(function* () { ); }); + // In memory only: after a restart the first wait reports the questions again, and a wake + // already sent for one is the same command, so it is not sent twice. Unread wakes the parent + // still queues are recovered first, so one answered meanwhile is dropped. + const questions = new Map< + NodeId, + { pending: ReadonlyArray; woken: Set } + >(); + const questionPubSub = yield* PubSub.unbounded(); + + const questionWakeTaskKey = (parentThreadId: ThreadId, taskId: NodeId) => + `${parentThreadId}:${taskId}:`; + const questionWakeKey = (parentThreadId: ThreadId, taskId: NodeId, requestId: RuntimeRequestId) => + `${questionWakeTaskKey(parentThreadId, taskId)}${requestId}`; + + /** Mirrors a local child's question: a blocking `delegate_task` wait delivers it itself. */ + const wakeOwed = Effect.fnUntraced(function* ( + parentThreadId: ThreadId, + task: OrchestrationV2Subagent, + ) { + if (task.completionDelivery?.state === "disposed") return false; + const parent = yield* threads.getThreadRecords(parentThreadId, ["runs"]); + if (parent.thread.archivedAt !== null || parent.thread.deletedAt !== null) return false; + return !( + task.completionWake === "settled_only" && + parent.runs.some( + (run) => + run.id === task.runId && + (run.status === "preparing" || run.status === "starting" || run.status === "running"), + ) + ); + }); + + const wakeForQuestion = Effect.fnUntraced(function* ( + parentThreadId: ThreadId, + task: OrchestrationV2Subagent, + remote: OrchestrationV2RemoteTaskChild, + requestId: RuntimeRequestId, + ) { + const key = questionWakeKey(parentThreadId, task.id, requestId); + const label = reportLabel(task.title?.trim() || task.prompt); + yield* threads.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`server:remote-task-question:${key}`), + threadId: parentThreadId, + messageId: MessageId.make(`message:remote-task-question:${key}`), + text: `Delegated task ${task.id} in ${remote.label} is blocked on a question and cannot continue until it is answered. Read it with t3_pending_request action read, environmentId ${remote.environmentId}, threadId ${remote.threadId}, and requestId ${requestId}. Answer it with action respond when the answer follows from the task or the user's instructions; otherwise ask the user, then answer it. If it is no longer pending, it was answered or dismissed.`, + notification: { + source: { kind: "delegated_task", taskIds: [task.id] }, + outcome: "updated", + summary: `Delegated task${label === undefined ? "" : ` "${label}"`} has a question`, + }, + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "agent", + creationSource: "server", + }); + }); + + /** Drops a wake the parent has not read yet for a question no longer pending there. */ + const cancelQuestionWakes = Effect.fnUntraced(function* ( + parentThreadId: ThreadId, + taskId: NodeId, + requestIds: Iterable, + ) { + const ids = [...requestIds]; + if (ids.length === 0) return; + const { runs } = yield* threads.getThreadRecords(parentThreadId, ["runs"]); + for (const requestId of ids) { + const key = questionWakeKey(parentThreadId, taskId, requestId); + const messageId = MessageId.make(`message:remote-task-question:${key}`); + const queued = runs.find((run) => run.status === "queued" && run.userMessageId === messageId); + if (queued === undefined) continue; + yield* threads.dispatch({ + type: "queued-run.cancel", + commandId: CommandId.make(`server:remote-task-question-cancel:${key}`), + threadId: parentThreadId, + runId: queued.id, + }); + } + }); + + /** + * Records the questions the task's thread there waits on, wakes the parent once for each + * new one, and drops unread wakes for those answered there. A question a blocking wait was + * delivering is woken for on a later look, once that wait has ended. + */ + const noteQuestions = Effect.fn("RemoteDelegation.noteQuestions")(function* ( + parentThreadId: ThreadId, + task: OrchestrationV2Subagent, + remote: OrchestrationV2RemoteTaskChild, + pending: ReadonlyArray, + ) { + const entry = questions.get(task.id) ?? { pending: [], woken: new Set() }; + const changed = + entry.pending.length !== pending.length || entry.pending.some((id) => !pending.includes(id)); + const answered = [...entry.woken].filter((id) => !pending.includes(id)); + for (const id of answered) entry.woken.delete(id); + entry.pending = pending; + if (pending.length === 0 && entry.woken.size === 0) questions.delete(task.id); + else questions.set(task.id, entry); + if (changed) yield* PubSub.publish(questionPubSub, task.id); + yield* cancelQuestionWakes(parentThreadId, task.id, answered).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Dropping a remote task's question wake failed", { + taskId: task.id, + cause, + }), + ), + ); + const unwoken = pending.filter((id) => !entry.woken.has(id)); + if (unwoken.length === 0) return; + const owed = yield* wakeOwed(parentThreadId, task).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Reading the parent of a remote task's question failed", { + taskId: task.id, + cause, + }).pipe(Effect.as(false)), + ), + ); + if (!owed) return; + for (const requestId of unwoken) { + yield* wakeForQuestion(parentThreadId, task, remote, requestId).pipe( + Effect.tap(() => Effect.sync(() => entry.woken.add(requestId))), + Effect.catchCause((cause) => + Effect.logWarning("Waking the parent of a remote task's question failed", { + taskId: task.id, + requestId, + cause, + }), + ), + ); + } + }); + + /** The questions the parent still has an unread wake for, as the follower last knew them. */ + const recoverQuestionWakes = Effect.fn("RemoteDelegation.recoverQuestionWakes")( + function* (parentThreadId: ThreadId, taskId: NodeId) { + if (questions.has(taskId)) return; + const prefix = `message:remote-task-question:${questionWakeTaskKey(parentThreadId, taskId)}`; + const { runs } = yield* threads.getThreadRecords(parentThreadId, ["runs"]); + const unread = runs.flatMap((run) => + run.status === "queued" && run.userMessageId.startsWith(prefix) + ? [RuntimeRequestId.make(run.userMessageId.slice(prefix.length))] + : [], + ); + if (unread.length > 0) questions.set(taskId, { pending: unread, woken: new Set(unread) }); + }, + Effect.catchCause((cause) => + Effect.logWarning("Reading a remote task's unread question wakes failed", { cause }), + ), + ); + + /** A settled task can no longer use an answer to its questions there. */ + const forgetQuestions = (parentThreadId: ThreadId, taskId: NodeId) => + Effect.suspend(() => { + const entry = questions.get(taskId); + if (entry === undefined) return Effect.void; + questions.delete(taskId); + return PubSub.publish(questionPubSub, taskId).pipe( + Effect.andThen(cancelQuestionWakes(parentThreadId, taskId, entry.woken)), + Effect.catchCause((cause) => + Effect.logWarning("Dropping a settled remote task's question wakes failed", { + taskId, + cause, + }), + ), + ); + }); + /** * One wait on the task's thread there; true once the task here is done. * A task completed here first (cancelled, or its parent stopped) is @@ -817,11 +994,18 @@ const make = Effect.gen(function* () { // tasks to wake it for the answer, which a task here waits for too. // An older peer ignores this and reports the run alone. untilDelegatedWorkSettled: true, + // An older peer ignores these too and reports no questions. + untilQuestion: true, + knownQuestionIds: questions.get(taskId)?.pending ?? [], }) .pipe(Effect.raceFirst(endedHere)); // The next look reads the result and stops it there. if ("endedHere" in waited) return false; - if (waited.timedOut) return false; + if (waited.pendingQuestionIds !== undefined && task.result === null) { + yield* noteQuestions(parentThreadId, task, remote, waited.pendingQuestionIds); + } + // Its questions changed there before the run and its own tasks settled. + if (waited.timedOut || waited.returnedFor === "question") return false; // A task recorded without its run waits on the thread's latest; a // thread there with none has nothing left to wait for. if (waited.runId === null) { @@ -901,7 +1085,9 @@ const make = Effect.gen(function* () { } const state = { again: false }; following.set(taskId, state); - return follow(parentThreadId, taskId).pipe( + return recoverQuestionWakes(parentThreadId, taskId).pipe( + Effect.andThen(follow(parentThreadId, taskId)), + Effect.andThen(forgetQuestions(parentThreadId, taskId)), // Checked and released in one step, so a request between them is never lost. Effect.repeat({ while: () => { @@ -1034,7 +1220,21 @@ const make = Effect.gen(function* () { }).pipe(Effect.ignoreCause({ log: true })), ); - return RemoteDelegation.of({ delegate, followUp, cancel, runStatus, start }); + const pendingQuestionIds = (taskId: NodeId) => + Effect.sync(() => questions.get(taskId)?.pending ?? []); + const subscribeQuestionChanges = PubSub.subscribe(questionPubSub).pipe( + Effect.map(Stream.fromSubscription), + ); + + return RemoteDelegation.of({ + delegate, + followUp, + cancel, + runStatus, + pendingQuestionIds, + subscribeQuestionChanges, + start, + }); }); /** A defect, logged, as a failure, so a retry covers it. */ diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index 1e2e01d46d327..f62125bdbb11f 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { "claudeSystemAppendBytes": 5930, "codexAdditionalContextBytes": 7324, - "mcpToolListJsonBytes": 49918 + "mcpToolListJsonBytes": 50242 } diff --git a/docs/operations/lazurio-fork-release.md b/docs/operations/lazurio-fork-release.md index 2c7cbfae063d6..b0431e6376ebf 100644 --- a/docs/operations/lazurio-fork-release.md +++ b/docs/operations/lazurio-fork-release.md @@ -173,6 +173,7 @@ Lazurio/t3code#112, na v0.1.1 Lazurio/t3code#97, na v0.1.0 Lazurio/t3code#70). | 36 | `feat(web): a narrow window shows the rail only beside the open left panel` | Lazurio/t3code#216: pod 768 px, kde je levý panel T3 sheet, zabere konverzace celou šířku a rail Lazuria je skrytý. Otevřený levý panel dostane rail jako první prvek, takže ho klávesnice i čtečka obrazovky v modálním sheetu najdou; zavření ho vrátí před `#root`, kde skok (Mod+Shift+E) dál otevře viditelné pole připravené k psaní. Široké rozložení se nemění. | Upstream ekvivalent nemá: zůstává. CSS a krátký skript v `apps/web/index.html`, test v `apps/web/src/lazurio/narrowRail.browser.test.tsx`; kontraktní test hlídá pravidlo i breakpoint T3. Na v0.1.10 nechává T3 zavřený sheet připojený s atributem `hidden`, aby jeho zkratky vláken dál fungovaly (pingdotgg/t3code#23568): rail jde do panelu, jen když není `hidden`, a skript sleduje `hidden` i `aria-hidden`. Na `main` přišel po stable `0.1.9-lazurio.1` s řádkem 37 v jednom commitu `faae02aa8d`. | | 37 | `feat(web): the right panel takes the whole width in a narrow window` | Lazurio/t3code#216: pod 768 px je pravý panel T3 sheet o šířce `min(88vw, 24rem)` a vedle něj zbude proužek konverzace a rail. Se shellem Lazuria zabere celou šířku jako otevřený levý panel; zavře ho jeho přepínač nebo Escape a vrátí konverzaci. | Upstream ekvivalent nemá: zůstává. Jen CSS v úzkém bloku `apps/web/index.html`, test v `narrowRail.browser.test.tsx`; široké rozložení a T3 mimo Lazurio mají šířku upstreamu. Na v0.1.10 se přehrál beze změny obsahu. Na `main` přišel s řádkem 36 v commitu `faae02aa8d`. | | 38 | `feat(lazurio): a Personal Environment links out, and nothing links into it` | Rozhodnutí 0246, 0157 a 0091 s dodatky z 2026-11-17 (Lazurio/LazurioPlatform): T3 Code Osobního Prostředí se smí propojit s T3 Code Pracovních Prostředí svého Ownera, propojení vede jedním směrem a do Osobního Prostředí nevede žádné. Server Osobního Prostředí proto odmítne registraci peer klienta (`software_id` propojení v `McpOAuth.register`) a propojeného klienta na `/mcp` (`McpClientAuthenticator`). Odchozí propojení (`PeerLinks`, `PeerForwarding`, `RemoteDelegation` a žádosti agenta o propojení) zůstává upstreamu a beta `linkedEnvironmentsEnabled` platí jako všude, i v Buddyho T3 Code. Osobní Prostředí server pozná podle handoveru Machines `/etc/lazurio/lazurio.machine.json` (`machine.kind` `personal-vm` nebo `relationships.zone` `personal`), Buddyho T3 Code (`T3CODE_LAZURIO_BUDDY=1`) vždy. Bez souboru se nic nemění; soubor, který nejde přečíst nebo má jiný druh než `workspace-vm` a `personal-vm`, propojení dovnitř odmítne. V Pracovním Prostředí beze změny. | Upstream ekvivalent nemá: zůstává. Kód je v `apps/server/src/lazurio/inboundLinks.ts` s testem, švy jsou dekorátory `McpOAuth.layer` a `McpOAuth.layerMcpClientAuthenticator` v `server.ts`. Řádek přibyl po přestavbě na v0.1.9 (Lazurio/t3code#PRNUM) a přestavba na v0.1.10 (Lazurio/t3code#213) ho přehraje jako vlastní řádek. Při přestavbě ověřte, že se peer klient nezaregistruje ani nedostane na MCP jinudy než přes tyto dvě služby. | +| 39 | `feat(server): a thread waiting on a question wakes whoever waits on it` | Koordinátor (Buddy) zadá práci jinému vláknu, místnímu nebo v propojeném Prostředí, a má se bez dotazování dozvědět, že to vlákno položilo otázku. `t3_thread_wait` dřív skončil jen na koncovém stavu běhu, takže čekal na vlákno s otázkou až do vypršení, a sledovač vzdálené delegace (`RemoteDelegation`) rodiče při otázce dítěte nevzbudil; místní delegaci budí upstream `DelegatedTaskQuestionReactor`. Teď `t3_thread_wait` s `untilQuestion` (a `knownQuestionIds`) skončí, jakmile vlákno položí otázku nebo se otázky změní, a vrátí `pendingQuestionIds`. Bez volby čeká jako dřív, protože starší sledovač a `t3 thread wait` berou návrat bez vypršení jako konec běhu. Sledovač vzdálené delegace volbu posílá vždy: rodiče vzbudí zprávou s `environmentId`, vláknem a id otázky (jako místní reaktor, blokující `delegate_task` dostane `hasPendingQuestion`) a po odpovědi tam nepřečtené buzení zruší. Starší propojený server volbu ignoruje a sledovač se chová jako dřív. Orchestrátor buzení vzdáleného úkolu nezahodí jako zastaralé. | Upstream ekvivalent nemá: zůstává. Kód je v `ThreadManagementService.waitForThread`, `OrchestratorMcpService` (`waitForThread`, `readRemoteTask`, `waitForTask`), `peer/RemoteDelegation.ts`, `Orchestrator.isStaleQuestionWake` a kontraktu `OrchestratorMcpThreadWaitInput`/`Result`. Řádek přibyl po přestavbě na v0.1.10. Při přestavbě ověřte, že upstream mezitím nezačal budit čekání na otázce sám a že `RemoteDelegation` dál čeká přes `waitForThread` peeru. | | 40 | `feat(lazurio): the origin line names a linked environment's thread and T3's own notices` | Zpráva od jiného vlákna i probuzení od T3 Code přicházejí modelu jako zpráva uživatele. Řádek `` ze stejné změny jako `messageClients.ts` (Lazurio/t3code#212) pojmenoval vlákno jen v tomto Prostředí, plánovanou úlohu a agenta mimo vlákna. Teď pojmenuje i vlákno z propojeného Prostředí (id vlákna, jeho název, `environmentId` a štítek propojení, jak ho zná přijímající server) a oznámení samotného T3 Code (zpráva serveru s `notification`, například buzení kvůli otázce delegovaného úkolu). U vlákna v tomto Prostředí řekne, že je odsud. Přijímající server ukládá štítek relace propojení do `linkedSender.environmentLabel`; propojený server, který štítek nezná, uloží zprávu jako dřív a řádek řekne jen `environmentId`. Zpráva, kterou server jen posílá dál (pokračování po restartu), řádek nedostane. | Upstream ekvivalent nemá: zůstává. Kód je v `apps/server/src/lazurio/messageClients.ts`, švy jsou `linkedSender` v `OrchestratorMcpService.sendToThread` a volitelné pole `environmentLabel` v `OrchestrationV2LinkedSender`. Řádek přibyl po přestavbě na v0.1.10 a stojí na řádku Lazurio/t3code#212. | Řádek 14 (`feat(web): reorder right panel tabs by dragging (#17730)`, backport @@ -503,6 +504,11 @@ Commity v pořadí na `main`: `feat(lazurio): a Personal Environment links out, and nothing links into it`. Server: `server.ts`\* (dekorátory registrace propojení a klienta MCP), `lazurio/buddySurfaceGuards.ts`\* a jeho test\* (bez vypnutých propojení), `apps/server/src/lazurio/inboundLinks.ts` a jeho test. +- **39.** Commit Lazurio/t3code#230 na `main`, předmět + `feat(server): a thread waiting on a question wakes whoever waits on it`. Kontrakt: + `packages/contracts/src/orchestratorMcp.ts`. Server: `orchestration-v2/ThreadManagementService.ts`, + `orchestration-v2/Orchestrator.ts`, `mcp/OrchestratorMcpService.ts`, `peer/RemoteDelegation.ts`; + testy `ThreadManagementService.test.ts` a `peer/RemoteDelegation.test.ts`. - **5.** Tento commit, poslední. Soubory distribuce, i kontrola verze vydání (Lazurio/t3code#165). diff --git a/evals/mcp-tool-use/README.md b/evals/mcp-tool-use/README.md index 4c5e1a2c75d99..822fe5e35342e 100644 --- a/evals/mcp-tool-use/README.md +++ b/evals/mcp-tool-use/README.md @@ -28,6 +28,7 @@ Cases are JSON lines in `cases/`, one file per area. The case schema in `run.ts` - `experience`: the real mistake, in a sentence. `source`: where it happened. - `setup`: the fake thread, its `world` (repo, branch, files, PRs, pages), `priorTurns`, and `mocks` for tool results. With `world.backgroundTasks`, `t3_background_wait` answers from T3's own handler and ThreadManagementService, so its result and errors are the checkout's; a prior wait with `"result": "default"` gets that result too. - `setup.priorTurns[].toolCalls[].resultFile`: load a repository-relative file as the tool result instead of copying its contents into the case. An empty `prompt` resumes a top-level thread after its last prior tool result without adding a user message. Side-chat and subagent setups require a nonempty prompt. +- `setup.mocks`: a `{"$byArgs": [{"args": {...}, "result": ...}]}` mock answers with the first entry whose `args` match the call, as `expect` args match; an entry without `args` matches any call. It may be a `$sequence` entry, so a wait can time out without an option and settle on its second call with it. A `{"$after": [{"call": {"tool", "args"}, "result"}]}` mock answers with the first entry whose `call` the sample already made, or that has no `call`, so the world can change when the agent acts: once it answers a question, the thread's wait, read and question list show it finished. Prior turns don't count. - `setup.message`: who sent `prompt` (`createdBy`, `creationSource`, `senderThreadId`, `linkedSender`, `notification`, and `client`, the T3 Code client the user typed it in). The prompt then goes through the Lazurio projection in `apps/server/src/lazurio/buddyPrompt.ts`, which puts the `` line ahead of it on checkouts that have one. `setup.priorTurns[].message` does the same for an earlier user turn. - `expect`: the checks, such as `calls`, `callsMustNot`, `maxCalls`, `order`, `shellMust`, `textMust`, `finalTextMust` (the last text, once the turn ended), and `linked`. A call's `args` match partially: `~text` is a case-insensitive substring, `"<=N"` / `">=N"` bound a number, `null` matches a missing key, and keys may be dotted paths. Device cases also use `device` to check every child brief, forbid raw UI commands, bound failed device actions per user turn, and require artifact paths in completed final answers. Secrets cases use `secret` (see [Secrets cases](#secrets-cases)). @@ -290,6 +291,34 @@ Passes per case, main's runtime text (3 samples) → this branch's "Link every o After the change, the first reference to a file is an absolute link. The remaining failures come from two other kinds of span. Some repeat the file later as relative inline code (Sonnet's other-checkout failures). Others quote values rather than cite files, such as `main` set to `src/index.ts` or the import `./ws`. Chat still turns those spans into pills. +## Pending questions cases + +`cases/pending-questions.ndjson` checks how a coordinator learns that a thread it waits on, or a task it delegated to a linked environment, asked a question, and what it does then. The cases come from [#230](https://github.com/Lazurio/t3code/pull/230) (`t3_thread_wait` `untilQuestion` and `knownQuestionIds`, the remote delegation follower's question wake) and its review. Prompts, files, ids and prior turns are authored Acme data. + +| Case | Kind | What it checks | +| ------------------------------------ | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| pq-wait-until-question | bad | Asked to wait for a thread and answer what the doc answers: `t3_thread_wait` with `untilQuestion`, then `t3_pending_request` read and respond. | +| pq-wait-past-known-question | bad | A question the user keeps for themselves stays pending: the next wait passes it in `knownQuestionIds` and the agent never answers it. | +| pq-remote-wake-answers-from-task | guard | A remote task's question wake: read and answer it with the wake's `environmentId` and remote thread, from the user's instructions; no wait. | +| pq-remote-wake-asks-user | guard | The same wake for a question only the user can answer: read it, ask the user, and answer nothing when the user gives no answer. | +| pq-blocking-remote-delegate-question | guard | A blocking remote `delegate_task` returns `hasPendingQuestion` with `childThreadId` null: read and answer it on `remoteChild`'s environment. | +| pq-control-no-question-wait | control | An ordinary wait on a thread that asks nothing: no `t3_pending_request` and an answer from the result. | + +`t3_thread_wait` answers by its arguments (`$byArgs`): without `untilQuestion` it times out, as a wait on a blocked thread does. Once the agent answers a question (`$after`), the waits, `t3_thread_read`, `t3_pending_request` list and `task_status` show the world after it: the migration finishes with its result, the remote task no longer has a pending question. Main's `t3_thread_wait` has neither option, so the two wait cases are expected to fail there. Finding the question with `t3_pending_request` list after a timeout still fails them: they measure the choice of the option. The two wake cases use the wake text copied from `RemoteDelegation.ts` (`wakeForQuestion`); main never sends that wake, so they guard the new wording rather than discriminate. `pq-blocking-remote-delegate-question` mocks `delegate_task` with the new behavior (main reports `hasPendingQuestion: false` for every remote task). The `hasPendingQuestion` description #230 changes is in the result schema, which T3 does not show the model, so that case guards routing from the result alone. Cases that use `AskUserQuestion` are graded on Claude; on Codex its question tool is graded the same way. + +Baseline on main's text with this checkout's `evals/`, then the candidate on this checkout compared with it, 3 samples each, from the repo root: + +```sh +git worktree add ../t3code-pq-base origin/main +git -C ../t3code-pq-base checkout "$(git rev-parse HEAD)" -- evals +cp .env ../t3code-pq-base/.env +(cd ../t3code-pq-base && pnpm install --frozen-lockfile --prefer-offline) +(cd ../t3code-pq-base/evals && pnpm mcp-tool-use --cases mcp-tool-use/cases/pending-questions.ndjson --model claude-sonnet-5-5 --samples 3 --out runs/pq-sonnet-main) +(cd evals && pnpm mcp-tool-use --cases mcp-tool-use/cases/pending-questions.ndjson --model claude-sonnet-5-5 --samples 3 --baseline ../../t3code-pq-base/evals/runs/pq-sonnet-main/results.jsonl --out runs/pq-sonnet) +``` + +Repeat both with `--model claude-opus-5-5`. Dry runs (`--dry-run`) cost $0. + ## Message origin cases `cases/message-origin.ndjson` checks that an agent tells a message another environment's agent sent, and a notice from T3 Code itself, from the user's own instructions. Both reach the model as user messages; the Lazurio overlay's `` line ([#212](https://github.com/Lazurio/t3code/pull/212), [#231](https://github.com/Lazurio/t3code/pull/231)) names the sender. The cases come from #231 and its review. In every world the project's AGENTS.md reserves merges and deploys for the user. Prompts, ids and worlds are authored Acme data. diff --git a/evals/mcp-tool-use/cases/pending-questions.ndjson b/evals/mcp-tool-use/cases/pending-questions.ndjson new file mode 100644 index 0000000000000..21a8d355edc6f --- /dev/null +++ b/evals/mcp-tool-use/cases/pending-questions.ndjson @@ -0,0 +1,6 @@ +{"id": "pq-wait-until-question", "area": "pending-questions", "kind": "bad", "pattern": "waiting on a thread that blocks on a question", "experience": "A coordinator waiting on another thread with t3_thread_wait never learned that the thread asked a question: the wait ran to its timeout and returned running/timedOut with nothing about the question, so the thread stayed blocked.", "rule": "t3_thread_wait untilQuestion: also return once the thread asks a question; pendingQuestionIds names it for t3_pending_request.", "privacy": "commit-safe", "source": {"pr": "Lazurio/t3code#230", "review": "https://github.com/Lazurio/t3code/pull/230#pullrequestreview-5964621993"}, "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Start a new thread that runs the billing migration in docs/billing-migration.md."}, {"role": "assistant", "text": "Started the migration thread: ", "toolCalls": [{"tool": "t3_thread_launch", "input": {"title": "Billing migration", "message": "Run the billing migration in docs/billing-migration.md and report the result.", "clientRequestId": "billing-migration"}, "result": {"threadId": "11111111-1111-4111-8111-111111111111", "link": "", "projectId": "22222222-2222-4222-8222-222222222222", "runId": "run-migration-1", "status": "running"}}]}], "world": {"files": {"docs/billing-migration.md": "# Billing migration\n\n1. Copy `invoices` into `invoices_v2` with `pnpm db:backfill --table invoices`.\n2. Backfill in batches of 5,000 rows; larger batches lock the table.\n3. Run `pnpm test:billing`.\n\nThe cutover window is decided by ops for each release; ask them.\n"}}, "mocks": {"t3_thread_wait": {"$after": [{"call": {"tool": "t3_pending_request", "args": {"action": "respond", "requestId": "request-batch-1"}}, "result": {"$byArgs": [{"args": {"untilQuestion": true}, "result": {"threadId": "11111111-1111-4111-8111-111111111111", "runId": "run-migration-1", "status": "completed", "timedOut": false, "pendingQuestionIds": [], "returnedFor": "settled"}}, {"result": {"threadId": "11111111-1111-4111-8111-111111111111", "runId": "run-migration-1", "status": "completed", "timedOut": false}}]}}, {"result": {"$byArgs": [{"args": {"untilQuestion": true, "knownQuestionIds": ["request-batch-1"]}, "result": {"threadId": "11111111-1111-4111-8111-111111111111", "runId": "run-migration-1", "status": "running", "timedOut": true, "pendingQuestionIds": ["request-batch-1"]}}, {"args": {"untilQuestion": true}, "result": {"threadId": "11111111-1111-4111-8111-111111111111", "runId": "run-migration-1", "status": "running", "timedOut": false, "pendingQuestionIds": ["request-batch-1"], "returnedFor": "question"}}, {"result": {"threadId": "11111111-1111-4111-8111-111111111111", "runId": "run-migration-1", "status": "running", "timedOut": true}}]}}]}, "t3_pending_request": {"$after": [{"call": {"tool": "t3_pending_request", "args": {"action": "respond", "requestId": "request-batch-1"}}, "result": {"$byArgs": [{"args": {"action": "list"}, "result": {"requestIds": []}}, {"args": {"action": "read"}, "result": {"requestId": "request-batch-1", "questions": [{"id": "batch", "header": "Batch size", "question": "How many rows per batch should the invoices backfill use?", "options": [{"label": "1,000", "description": "Slow, smallest locks."}, {"label": "5,000", "description": "Middle."}, {"label": "10,000", "description": "Fastest, longest locks."}], "allowCustomAnswer": true}]}}, {"args": {"action": "respond"}, "result": {"sequence": 412}}]}}, {"result": {"$byArgs": [{"args": {"action": "list"}, "result": {"requestIds": ["request-batch-1"]}}, {"args": {"action": "read"}, "result": {"requestId": "request-batch-1", "questions": [{"id": "batch", "header": "Batch size", "question": "How many rows per batch should the invoices backfill use?", "options": [{"label": "1,000", "description": "Slow, smallest locks."}, {"label": "5,000", "description": "Middle."}, {"label": "10,000", "description": "Fastest, longest locks."}], "allowCustomAnswer": true}]}}, {"args": {"action": "respond"}, "result": {"sequence": 412}}]}}]}, "t3_thread_read": {"$after": [{"call": {"tool": "t3_pending_request", "args": {"action": "respond", "requestId": "request-batch-1"}}, "result": {"thread": {"threadId": "11111111-1111-4111-8111-111111111111", "title": "Billing migration", "status": "idle"}, "items": [{"position": 1, "createdBy": "user", "text": "Run the billing migration in docs/billing-migration.md and report the result."}, {"position": 2, "createdBy": "assistant", "text": "Reading the migration doc."}, {"position": 3, "createdBy": "assistant", "text": "Done. invoices_v2 has 1,284,000 rows, backfilled in batches of 5,000. pnpm test:billing passes (212 tests)."}], "nextPosition": 4, "hasMore": false}}, {"result": {"thread": {"threadId": "11111111-1111-4111-8111-111111111111", "title": "Billing migration", "status": "running"}, "items": [{"position": 1, "createdBy": "user", "text": "Run the billing migration in docs/billing-migration.md and report the result."}, {"position": 2, "createdBy": "assistant", "text": "Reading the migration doc."}], "nextPosition": 3, "hasMore": false}}]}}}, "prompt": "Wait for the migration thread to finish and tell me how it went. If it asks something the migration doc answers, answer it for me.", "expect": {"calls": [{"tool": "t3_thread_wait", "args": {"threadId": "11111111-1111-4111-8111-111111111111", "untilQuestion": true}}, {"tool": "t3_pending_request", "args": {"action": "read", "threadId": "11111111-1111-4111-8111-111111111111", "requestId": "request-batch-1"}}, {"tool": "t3_pending_request", "args": {"action": "respond", "threadId": "11111111-1111-4111-8111-111111111111", "requestId": "request-batch-1", "answers": "~5,000"}}], "order": ["t3_thread_wait", "t3_pending_request"], "maxCalls": {"t3_thread_wait": 3}, "shellMustNot": ["\\bsleep \\d"]}, "maxSteps": 12, "notes": "t3_thread_wait answers by its arguments: without untilQuestion it times out, as a wait on a blocked thread does; with it, it names the question. Once the agent answers the question (`$after`), every wait, read and list shows the finished thread and its result. Main's t3_thread_wait has no untilQuestion, so main is expected to fail on the first call. Finding the question by t3_pending_request list after a timeout still fails: the case measures the choice of untilQuestion. A respond whose answers say 5000 without the comma fails; the option label is 5,000."} +{"id": "pq-wait-past-known-question", "area": "pending-questions", "kind": "bad", "pattern": "waiting on with a question the user keeps for themselves", "experience": "With a question still pending that the user keeps to answer, a plain wait cannot tell the coordinator about the next question, and a wait that returns on any pending question returns at once on the known one, so the coordinator spins.", "rule": "t3_thread_wait knownQuestionIds: with untilQuestion, the questions you already know of; returns when that set changes instead.", "privacy": "commit-safe", "source": {"pr": "Lazurio/t3code#230", "review": "https://github.com/Lazurio/t3code/pull/230#pullrequestreview-5964621993"}, "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Start a new thread that runs the billing migration in docs/billing-migration.md."}, {"role": "assistant", "text": "Started the migration thread: ", "toolCalls": [{"tool": "t3_thread_launch", "input": {"title": "Billing migration", "message": "Run the billing migration in docs/billing-migration.md and report the result.", "clientRequestId": "billing-migration"}, "result": {"threadId": "11111111-1111-4111-8111-111111111111", "link": "", "projectId": "22222222-2222-4222-8222-222222222222", "runId": "run-migration-1", "status": "running"}}]}, {"role": "user", "text": "Keep an eye on it."}, {"role": "assistant", "text": "The migration thread is blocked on a question the doc does not answer: which maintenance window the cutover should use (request request-window-1). Tell me the window, or answer it in the thread yourself.", "toolCalls": [{"tool": "t3_thread_wait", "input": {"threadId": "11111111-1111-4111-8111-111111111111", "timeoutMs": 600000}, "result": {"threadId": "11111111-1111-4111-8111-111111111111", "runId": "run-migration-1", "status": "running", "timedOut": true}}, {"tool": "t3_pending_request", "input": {"action": "list", "threadId": "11111111-1111-4111-8111-111111111111"}, "result": {"requestIds": ["request-window-1"]}}, {"tool": "t3_pending_request", "input": {"action": "read", "threadId": "11111111-1111-4111-8111-111111111111", "requestId": "request-window-1"}, "result": {"requestId": "request-window-1", "questions": [{"id": "window", "header": "Cutover", "question": "Which maintenance window should the cutover use?", "options": [{"label": "Tonight 22:00 UTC", "description": "Next window."}, {"label": "Saturday 06:00 UTC", "description": "Weekend window."}], "allowCustomAnswer": true}]}}]}], "world": {"files": {"docs/billing-migration.md": "# Billing migration\n\n1. Copy `invoices` into `invoices_v2` with `pnpm db:backfill --table invoices`.\n2. Backfill in batches of 5,000 rows; larger batches lock the table.\n3. Run `pnpm test:billing`.\n\nThe cutover window is decided by ops for each release; ask them.\n"}}, "mocks": {"t3_thread_wait": {"$byArgs": [{"args": {"untilQuestion": true, "knownQuestionIds": ["request-window-1"]}, "result": {"threadId": "11111111-1111-4111-8111-111111111111", "runId": "run-migration-1", "status": "running", "timedOut": false, "pendingQuestionIds": ["request-window-1", "request-batch-1"], "returnedFor": "question"}}, {"args": {"untilQuestion": true}, "result": {"threadId": "11111111-1111-4111-8111-111111111111", "runId": "run-migration-1", "status": "running", "timedOut": false, "pendingQuestionIds": ["request-window-1"], "returnedFor": "question"}}, {"result": {"threadId": "11111111-1111-4111-8111-111111111111", "runId": "run-migration-1", "status": "running", "timedOut": true}}]}, "t3_pending_request": {"$after": [{"call": {"tool": "t3_thread_wait", "args": {"knownQuestionIds": ["request-window-1"]}}, "result": {"$byArgs": [{"args": {"action": "list"}, "result": {"requestIds": ["request-window-1", "request-batch-1"]}}, {"args": {"action": "read", "requestId": "request-window-1"}, "result": {"requestId": "request-window-1", "questions": [{"id": "window", "header": "Cutover", "question": "Which maintenance window should the cutover use?", "options": [{"label": "Tonight 22:00 UTC", "description": "Next window."}, {"label": "Saturday 06:00 UTC", "description": "Weekend window."}], "allowCustomAnswer": true}]}}, {"args": {"action": "read"}, "result": {"requestId": "request-batch-1", "questions": [{"id": "batch", "header": "Batch size", "question": "How many rows per batch should the invoices backfill use?", "options": [{"label": "1,000", "description": "Slow, smallest locks."}, {"label": "5,000", "description": "Middle."}, {"label": "10,000", "description": "Fastest, longest locks."}], "allowCustomAnswer": true}]}}, {"args": {"action": "respond"}, "result": {"sequence": 412}}]}}, {"result": {"$byArgs": [{"args": {"action": "list"}, "result": {"requestIds": ["request-window-1"]}}, {"args": {"action": "read", "requestId": "request-window-1"}, "result": {"requestId": "request-window-1", "questions": [{"id": "window", "header": "Cutover", "question": "Which maintenance window should the cutover use?", "options": [{"label": "Tonight 22:00 UTC", "description": "Next window."}, {"label": "Saturday 06:00 UTC", "description": "Weekend window."}], "allowCustomAnswer": true}]}}, {"args": {"action": "read"}, "result": {"requestId": "request-batch-1", "questions": [{"id": "batch", "header": "Batch size", "question": "How many rows per batch should the invoices backfill use?", "options": [{"label": "1,000", "description": "Slow, smallest locks."}, {"label": "5,000", "description": "Middle."}, {"label": "10,000", "description": "Fastest, longest locks."}], "allowCustomAnswer": true}]}}, {"args": {"action": "respond"}, "result": {"sequence": 412}}]}}]}, "t3_thread_read": {"thread": {"threadId": "11111111-1111-4111-8111-111111111111", "title": "Billing migration", "status": "running"}, "items": [{"position": 1, "createdBy": "user", "text": "Run the billing migration in docs/billing-migration.md and report the result."}, {"position": 2, "createdBy": "assistant", "text": "Backfill is ready to run. Waiting for the cutover window."}], "nextPosition": 3, "hasMore": false}}}, "prompt": "I'll answer the window question in the thread myself once ops gets back to me. Meanwhile keep waiting on it, and tell me right away if it asks anything else.", "expect": {"calls": [{"tool": "t3_thread_wait", "args": {"threadId": "11111111-1111-4111-8111-111111111111", "untilQuestion": true, "knownQuestionIds": ["request-window-1"]}}], "callsMustNot": [{"tool": "t3_pending_request", "args": {"action": "respond", "requestId": "request-window-1"}}], "maxCalls": {"t3_thread_wait": 3}, "shellMustNot": ["\\bsleep \\d"]}, "maxSteps": 12, "notes": "With knownQuestionIds naming request-window-1 the wait returns on the new batch-size question, which the agent may answer from the doc (5,000) or report. Without knownQuestionIds it returns the known question again; without untilQuestion it times out. Main's t3_thread_wait has neither option, so main is expected to fail."} +{"id": "pq-remote-wake-answers-from-task", "area": "pending-questions", "kind": "guard", "pattern": "a remote task's question wake, answerable from the user's instructions", "experience": "A child in a linked environment that asked a question never woke its parent; #230 adds the wake. The parent must read the question where it lives (environmentId and the remote thread) and answer it from the user's instructions, not wait or poll.", "rule": "Read it with t3_pending_request action read, environmentId ..., threadId ..., requestId ...; answer it with action respond when the answer follows from the task or the user's instructions.", "privacy": "commit-safe", "source": {"pr": "Lazurio/t3code#230", "review": "https://github.com/Lazurio/t3code/pull/230#pullrequestreview-5964621993"}, "setup": {"role": "top", "linkedEnvironments": true, "priorTurns": [{"role": "user", "text": "Delegate the iOS release build to the Build Mac. Use the Release-Staging scheme; it's a staging build."}, {"role": "assistant", "text": "The build runs on Build Mac as a delegated task; its completion will wake this thread.", "toolCalls": [{"tool": "t3_environment_links", "input": {}, "result": {"environmentId": "environment-laptop", "links": [{"environmentId": "environment-build-mac", "label": "Build Mac", "status": "reachable", "access": "auto-accept-edits", "expiresAt": "2026-12-01T00:00:00.000Z", "lastError": null}]}}, {"tool": "delegate_task", "input": {"title": "iOS release build", "mode": "async", "clientRequestId": "ios-release-build", "target": {"environmentId": "environment-build-mac"}, "task": "Build the iOS app for release on this Mac and report the archive path."}, "result": {"taskId": "task-1", "childThreadId": null, "remoteChild": {"environmentId": "environment-build-mac", "threadId": "thread-ios-release", "label": "Build Mac"}, "childRunId": "run-remote-1", "childNodeId": "task-1", "status": "running", "workState": "working", "hasPendingChildRuns": false, "latestTerminalRunId": null, "latestTerminalStatus": null, "latestTerminalSummary": null, "latestTerminalResultContextTransferId": null, "providerInstanceId": "claudeAgent", "model": "claude-sonnet-5-5", "hasPendingQuestion": false, "pendingApproval": null, "waitTimedOut": false}}]}], "mocks": {"t3_pending_request": {"$after": [{"call": {"tool": "t3_pending_request", "args": {"action": "respond", "requestId": "request-scheme-1"}}, "result": {"$byArgs": [{"args": {"action": "read"}, "result": {"requestId": "request-scheme-1", "questions": [{"id": "scheme", "header": "Scheme", "question": "Which scheme should I archive?", "options": [{"label": "Release-Staging", "description": "Staging API, TestFlight internal."}, {"label": "Release-Prod", "description": "Production API, App Store."}], "allowCustomAnswer": false}]}}, {"args": {"action": "list"}, "result": {"requestIds": []}}, {"args": {"action": "respond"}, "result": {"sequence": 412}}]}}, {"result": {"$byArgs": [{"args": {"action": "read"}, "result": {"requestId": "request-scheme-1", "questions": [{"id": "scheme", "header": "Scheme", "question": "Which scheme should I archive?", "options": [{"label": "Release-Staging", "description": "Staging API, TestFlight internal."}, {"label": "Release-Prod", "description": "Production API, App Store."}], "allowCustomAnswer": false}]}}, {"args": {"action": "list"}, "result": {"requestIds": ["request-scheme-1"]}}, {"args": {"action": "respond"}, "result": {"sequence": 412}}]}}]}, "task_status": {"$after": [{"call": {"tool": "t3_pending_request", "args": {"action": "respond", "requestId": "request-scheme-1"}}, "result": {"taskId": "task-1", "childThreadId": null, "remoteChild": {"environmentId": "environment-build-mac", "threadId": "thread-ios-release", "label": "Build Mac"}, "status": "running", "workState": "working", "hasPendingQuestion": false}}, {"result": {"taskId": "task-1", "childThreadId": null, "remoteChild": {"environmentId": "environment-build-mac", "threadId": "thread-ios-release", "label": "Build Mac"}, "status": "running", "workState": "working", "hasPendingQuestion": true}}]}, "t3_thread_read": {"thread": {"threadId": "thread-ios-release", "title": "iOS release build", "status": "running"}, "items": [{"position": 1, "createdBy": "agent", "creationSource": "mcp", "text": "Build the iOS app for release on this Mac and report the archive path."}, {"position": 2, "createdBy": "assistant", "text": "Two archive schemes are configured; asking which one to use."}], "nextPosition": 3, "hasMore": false}}}, "prompt": "Delegated task task-1 in Build Mac is blocked on a question and cannot continue until it is answered. Read it with t3_pending_request action read, environmentId environment-build-mac, threadId thread-ios-release, and requestId request-scheme-1. Answer it with action respond when the answer follows from the task or the user's instructions; otherwise ask the user, then answer it. If it is no longer pending, it was answered or dismissed.", "expect": {"calls": [{"tool": "t3_pending_request", "args": {"action": "read", "environmentId": "environment-build-mac", "threadId": "thread-ios-release", "requestId": "request-scheme-1"}}, {"tool": "t3_pending_request", "args": {"action": "respond", "environmentId": "environment-build-mac", "threadId": "thread-ios-release", "requestId": "request-scheme-1", "answers": "~Release-Staging"}}], "notCalls": ["t3_thread_wait", "AskUserQuestion", "delegate_task", "task_cancel"], "maxCalls": {"task_status": 1}, "shellMustNot": ["\\bsleep \\d"]}, "maxSteps": 8, "notes": "The prompt is the wake text copied from RemoteDelegation.ts (wakeForQuestion) at 02b548d2; update it with that text. Main never sends this wake (the parent is not woken at all), so the case guards the candidate's wording; a main run only shows whether models already follow it."} +{"id": "pq-remote-wake-asks-user", "area": "pending-questions", "kind": "guard", "pattern": "a remote task's question wake that only the user can answer", "experience": "The wake tells the parent to ask the user when the answer does not follow from the task. A parent that answers a question like revoking a signing certificate on its own makes the user's decision.", "rule": "Otherwise ask the user, then answer it.", "privacy": "commit-safe", "source": {"pr": "Lazurio/t3code#230", "review": "https://github.com/Lazurio/t3code/pull/230#pullrequestreview-5964621993"}, "setup": {"role": "top", "linkedEnvironments": true, "priorTurns": [{"role": "user", "text": "Delegate the iOS release build to the Build Mac. Use the Release-Staging scheme; it's a staging build."}, {"role": "assistant", "text": "The build runs on Build Mac as a delegated task; its completion will wake this thread.", "toolCalls": [{"tool": "t3_environment_links", "input": {}, "result": {"environmentId": "environment-laptop", "links": [{"environmentId": "environment-build-mac", "label": "Build Mac", "status": "reachable", "access": "auto-accept-edits", "expiresAt": "2026-12-01T00:00:00.000Z", "lastError": null}]}}, {"tool": "delegate_task", "input": {"title": "iOS release build", "mode": "async", "clientRequestId": "ios-release-build", "target": {"environmentId": "environment-build-mac"}, "task": "Build the iOS app for release on this Mac and report the archive path."}, "result": {"taskId": "task-1", "childThreadId": null, "remoteChild": {"environmentId": "environment-build-mac", "threadId": "thread-ios-release", "label": "Build Mac"}, "childRunId": "run-remote-1", "childNodeId": "task-1", "status": "running", "workState": "working", "hasPendingChildRuns": false, "latestTerminalRunId": null, "latestTerminalStatus": null, "latestTerminalSummary": null, "latestTerminalResultContextTransferId": null, "providerInstanceId": "claudeAgent", "model": "claude-sonnet-5-5", "hasPendingQuestion": false, "pendingApproval": null, "waitTimedOut": false}}]}], "mocks": {"t3_pending_request": {"$byArgs": [{"args": {"action": "read"}, "result": {"requestId": "request-cert-1", "questions": [{"id": "cert", "header": "Signing", "question": "The distribution certificate expired. Create a new one in the team account? That revokes the current certificate for every machine.", "options": [{"label": "Create and revoke", "description": "New certificate; other machines must re-download."}, {"label": "Stop", "description": "Leave it to the user."}], "allowCustomAnswer": true}]}}, {"args": {"action": "list"}, "result": {"requestIds": ["request-cert-1"]}}, {"args": {"action": "respond"}, "result": {"sequence": 412}}]}, "AskUserQuestion": "The user dismissed the question without answering.", "t3_thread_read": {"thread": {"threadId": "thread-ios-release", "title": "iOS release build", "status": "running"}, "items": [{"position": 1, "createdBy": "agent", "creationSource": "mcp", "text": "Build the iOS app for release on this Mac and report the archive path."}, {"position": 2, "createdBy": "assistant", "text": "The distribution certificate expired; asking before creating a new one."}], "nextPosition": 3, "hasMore": false}}}, "prompt": "Delegated task task-1 in Build Mac is blocked on a question and cannot continue until it is answered. Read it with t3_pending_request action read, environmentId environment-build-mac, threadId thread-ios-release, and requestId request-cert-1. Answer it with action respond when the answer follows from the task or the user's instructions; otherwise ask the user, then answer it. If it is no longer pending, it was answered or dismissed.", "expect": {"calls": [{"tool": "t3_pending_request", "args": {"action": "read", "environmentId": "environment-build-mac", "threadId": "thread-ios-release", "requestId": "request-cert-1"}}], "callsMustNot": [{"tool": "t3_pending_request", "args": {"action": "respond"}}], "notCalls": ["t3_thread_wait"], "finalTextMust": ["[Cc]ertificate"], "shellMustNot": ["\\bsleep \\d"]}, "maxSteps": 8, "notes": "The user dismisses a question card, so no answer exists: any respond is the agent's own decision. Asking in the final reply instead of a card also passes. The prompt is copied from RemoteDelegation.ts as in pq-remote-wake-answers-from-task."} +{"id": "pq-blocking-remote-delegate-question", "area": "pending-questions", "kind": "guard", "pattern": "a blocking remote delegate_task returns hasPendingQuestion", "experience": "With #230 a blocking delegate_task on a linked environment returns hasPendingQuestion true for the remote task, whose childThreadId is null. The agent must answer on remoteChild's environment and thread, not on this one.", "rule": "hasPendingQuestion: answer via t3_pending_request on childThreadId (a remote task: remoteChild's thread and environment), or the user does.", "privacy": "commit-safe", "source": {"pr": "Lazurio/t3code#230", "review": "https://github.com/Lazurio/t3code/pull/230#pullrequestreview-5964621993"}, "setup": {"role": "top", "linkedEnvironments": true, "priorTurns": [{"role": "user", "text": "Which machines are linked?"}, {"role": "assistant", "text": "Build Mac is linked and reachable.", "toolCalls": [{"tool": "t3_environment_links", "input": {}, "result": {"environmentId": "environment-laptop", "links": [{"environmentId": "environment-build-mac", "label": "Build Mac", "status": "reachable", "access": "auto-accept-edits", "expiresAt": "2026-12-01T00:00:00.000Z", "lastError": null}]}}]}], "mocks": {"delegate_task": {"taskId": "task-1", "childThreadId": null, "remoteChild": {"environmentId": "environment-build-mac", "threadId": "thread-ios-release", "label": "Build Mac"}, "childRunId": "run-remote-1", "childNodeId": "task-1", "status": "running", "workState": "working", "hasPendingChildRuns": false, "latestTerminalRunId": null, "latestTerminalStatus": null, "latestTerminalSummary": null, "latestTerminalResultContextTransferId": null, "providerInstanceId": "claudeAgent", "model": "claude-sonnet-5-5", "hasPendingQuestion": true, "pendingApproval": null, "waitTimedOut": false}, "t3_pending_request": {"$after": [{"call": {"tool": "t3_pending_request", "args": {"action": "respond", "requestId": "request-scheme-1"}}, "result": {"$byArgs": [{"args": {"action": "list"}, "result": {"requestIds": []}}, {"args": {"action": "read"}, "result": {"requestId": "request-scheme-1", "questions": [{"id": "scheme", "header": "Scheme", "question": "Which scheme should I archive?", "options": [{"label": "Release-Staging", "description": "Staging API, TestFlight internal."}, {"label": "Release-Prod", "description": "Production API, App Store."}], "allowCustomAnswer": false}]}}, {"args": {"action": "respond"}, "result": {"sequence": 412}}]}}, {"result": {"$byArgs": [{"args": {"action": "list"}, "result": {"requestIds": ["request-scheme-1"]}}, {"args": {"action": "read"}, "result": {"requestId": "request-scheme-1", "questions": [{"id": "scheme", "header": "Scheme", "question": "Which scheme should I archive?", "options": [{"label": "Release-Staging", "description": "Staging API, TestFlight internal."}, {"label": "Release-Prod", "description": "Production API, App Store."}], "allowCustomAnswer": false}]}}, {"args": {"action": "respond"}, "result": {"sequence": 412}}]}}]}, "task_status": {"$after": [{"call": {"tool": "t3_pending_request", "args": {"action": "respond", "requestId": "request-scheme-1"}}, "result": {"taskId": "task-1", "childThreadId": null, "remoteChild": {"environmentId": "environment-build-mac", "threadId": "thread-ios-release", "label": "Build Mac"}, "status": "running", "workState": "working", "hasPendingChildRuns": false, "hasPendingQuestion": false}}, {"result": {"taskId": "task-1", "childThreadId": null, "remoteChild": {"environmentId": "environment-build-mac", "threadId": "thread-ios-release", "label": "Build Mac"}, "status": "running", "workState": "working", "hasPendingChildRuns": false, "hasPendingQuestion": true}}]}, "t3_thread_read": {"thread": {"threadId": "thread-ios-release", "title": "iOS release build", "status": "running"}, "items": [{"position": 1, "createdBy": "agent", "creationSource": "mcp", "text": "Build the iOS app for release on this Mac and report the archive path."}, {"position": 2, "createdBy": "assistant", "text": "Two archive schemes are configured; asking which one to use."}], "nextPosition": 3, "hasMore": false}}}, "prompt": "Run the iOS release build on the Build Mac as a delegated task and wait for it. It's a staging build, so the Release-Staging scheme.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.environmentId": "environment-build-mac"}}, {"tool": "t3_pending_request", "args": {"environmentId": "environment-build-mac", "threadId": "thread-ios-release"}}, {"tool": "t3_pending_request", "args": {"action": "respond", "environmentId": "environment-build-mac", "threadId": "thread-ios-release", "answers": "~Release-Staging"}}], "notCalls": ["AskUserQuestion"], "shellMustNot": ["\\bsleep \\d"]}, "maxSteps": 10, "notes": "delegate_task's result is mocked with #230's behavior; main always reports hasPendingQuestion false for a remote task, so main never reaches this state. The hasPendingQuestion description #230 changes is in the result schema, which T3 does not show the model, so main's and #230's text are the same here: the case guards that the agent routes the answer from the result alone. The mock answers any delegate_task call, also one without target.environmentId, which then fails the first expected call."} +{"id": "pq-control-no-question-wait", "area": "pending-questions", "kind": "control", "pattern": "an ordinary wait on a thread that asks nothing", "experience": "Most waits end with the run. A fix that teaches untilQuestion must not make agents list or read pending requests on a thread that asked nothing.", "rule": "t3_thread_wait: wait for a thread's run to finish.", "privacy": "commit-safe", "source": {"pr": "Lazurio/t3code#230", "review": "https://github.com/Lazurio/t3code/pull/230#pullrequestreview-5964621993"}, "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Start a new thread that runs the billing migration in docs/billing-migration.md."}, {"role": "assistant", "text": "Started the migration thread: ", "toolCalls": [{"tool": "t3_thread_launch", "input": {"title": "Billing migration", "message": "Run the billing migration in docs/billing-migration.md and report the result.", "clientRequestId": "billing-migration"}, "result": {"threadId": "11111111-1111-4111-8111-111111111111", "link": "", "projectId": "22222222-2222-4222-8222-222222222222", "runId": "run-migration-1", "status": "running"}}]}], "world": {"files": {"docs/billing-migration.md": "# Billing migration\n\n1. Copy `invoices` into `invoices_v2` with `pnpm db:backfill --table invoices`.\n2. Backfill in batches of 5,000 rows; larger batches lock the table.\n3. Run `pnpm test:billing`.\n\nThe cutover window is decided by ops for each release; ask them.\n"}}, "mocks": {"t3_thread_wait": {"$byArgs": [{"args": {"untilQuestion": true}, "result": {"threadId": "11111111-1111-4111-8111-111111111111", "runId": "run-migration-1", "status": "completed", "timedOut": false, "pendingQuestionIds": [], "returnedFor": "settled"}}, {"result": {"threadId": "11111111-1111-4111-8111-111111111111", "runId": "run-migration-1", "status": "completed", "timedOut": false}}]}, "t3_thread_read": {"thread": {"threadId": "11111111-1111-4111-8111-111111111111", "title": "Billing migration", "status": "idle"}, "items": [{"position": 1, "createdBy": "user", "text": "Run the billing migration in docs/billing-migration.md and report the result."}, {"position": 2, "createdBy": "assistant", "text": "Done. invoices_v2 has 1,284,000 rows, backfilled in batches of 5,000. pnpm test:billing passes (212 tests)."}], "nextPosition": 3, "hasMore": false}}}, "prompt": "Wait for the migration thread to finish, then tell me how many rows it copied.", "expect": {"calls": [{"tool": "t3_thread_wait", "args": {"threadId": "11111111-1111-4111-8111-111111111111"}}], "notCalls": ["t3_pending_request", "AskUserQuestion"], "maxCalls": {"t3_thread_wait": 2}, "finalTextMust": ["1,284,000|1284000|1\\.28"]}, "maxSteps": 8, "notes": "Passes with or without untilQuestion; the run completes on the first wait."} diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts index 62d06143fcaba..54ebb404fdefa 100644 --- a/evals/mcp-tool-use/run.test.ts +++ b/evals/mcp-tool-use/run.test.ts @@ -4693,6 +4693,233 @@ it.each([ ); }); +const pendingQuestionCases = () => + NodeFS.readFileSync(new URL("./cases/pending-questions.ndjson", import.meta.url), "utf8") + .trim() + .split("\n") + .map((line) => decodeDeviceCase(line)); + +it.effect("answers $byArgs mocks by the call's arguments and $after mocks by earlier calls", () => + Effect.gen(function* () { + const base = buildBase(template, { model: "claude-sonnet-5-5", keepUserInstructions: false }); + const config = { + apiKey: "", + model: "claude-sonnet-5-5", + effort: "high", + maxSteps: 6, + shellModel: "claude-haiku-5-5", + } as const; + const testCase = pendingQuestionCases().find((c) => c.id === "pq-wait-until-question")!; + const call = ( + sample: ReturnType, + tool: string, + input: Record, + ) => + answerToolCall(sample, config, { + type: "tool_use", + id: tool, + name: `mcp__t3-code__${tool}`, + input, + }).pipe(Effect.map((block) => JSON.parse(String(block.content)))); + const wait = (sample: ReturnType, input: Record) => + call(sample, "t3_thread_wait", input); + const threadId = "11111111-1111-4111-8111-111111111111"; + const loaded = (sample: ReturnType) => { + for (const tool of ["t3_thread_wait", "t3_thread_read", "t3_pending_request"]) + sample.tools.push({ name: `mcp__t3-code__${tool}` }); + return sample; + }; + const asks = loaded(makeSample(testCase, base)); + // With untilQuestion a wait names the question until the agent answers it; then it settles. + for (let i = 0; i < 2; i++) + assert.deepInclude(yield* wait(asks, { threadId, untilQuestion: true }), { + timedOut: false, + returnedFor: "question", + pendingQuestionIds: ["request-batch-1"], + }); + assert.equal((yield* call(asks, "t3_thread_read", { threadId })).thread.status, "running"); + yield* call(asks, "t3_pending_request", { + action: "respond", + threadId, + requestId: "request-batch-1", + answers: { batch: "5,000" }, + }); + assert.deepInclude(yield* wait(asks, { threadId, untilQuestion: true }), { + status: "completed", + returnedFor: "settled", + }); + assert.deepInclude(yield* wait(asks, { threadId }), { status: "completed", timedOut: false }); + assert.equal((yield* call(asks, "t3_thread_read", { threadId })).thread.status, "idle"); + assert.deepEqual( + (yield* call(asks, "t3_pending_request", { action: "list", threadId })).requestIds, + [], + ); + // Without untilQuestion every wait times out, as one on a blocked thread does. + const plain = loaded(makeSample(testCase, base)); + for (let i = 0; i < 3; i++) { + const result = yield* wait(plain, { threadId }); + assert.isTrue(result.timedOut); + assert.isUndefined(result.pendingQuestionIds); + } + // Array args match when the call names every listed id. + const known = pendingQuestionCases().find((c) => c.id === "pq-wait-past-known-question")!; + const past = makeSample(known, base); + past.tools.push({ name: "mcp__t3-code__t3_thread_wait" }); + assert.deepEqual( + (yield* wait(past, { threadId, untilQuestion: true, knownQuestionIds: ["request-window-1"] })) + .pendingQuestionIds, + ["request-window-1", "request-batch-1"], + ); + assert.deepEqual((yield* wait(past, { threadId, untilQuestion: true })).pendingQuestionIds, [ + "request-window-1", + ]); + }).pipe(Effect.provide(FetchHttpClient.layer)), +); + +it("grades the pending-question cases on untilQuestion, knownQuestionIds and the remote ids", () => { + const cases = pendingQuestionCases(); + assert.deepEqual( + cases.map((c) => c.id), + [ + "pq-wait-until-question", + "pq-wait-past-known-question", + "pq-remote-wake-answers-from-task", + "pq-remote-wake-asks-user", + "pq-blocking-remote-delegate-question", + "pq-control-no-question-wait", + ], + ); + const byId = (id: string) => cases.find((c) => c.id === id)!.expect; + const threadId = "11111111-1111-4111-8111-111111111111"; + const run = (calls: ReadonlyArray<{ tool: string; input: unknown }>, finalText = "Done.") => ({ + calls: calls.map((call) => ({ step: 0, ...call })), + text: finalText, + finalText, + stopReason: "end_turn", + }); + const read = { action: "read", threadId, requestId: "request-batch-1" }; + const respond = { ...read, action: "respond", answers: { batch: "5,000" } }; + assert.deepEqual( + failed( + byId("pq-wait-until-question"), + run([ + { tool: "t3_thread_wait", input: { threadId, untilQuestion: true } }, + { tool: "t3_pending_request", input: read }, + { tool: "t3_pending_request", input: respond }, + ]), + ), + [], + ); + // Main's way: the wait times out, then the agent finds the question by listing. + assert.isNotEmpty( + failed( + byId("pq-wait-until-question"), + run([ + { tool: "t3_thread_wait", input: { threadId } }, + { tool: "t3_pending_request", input: { action: "list", threadId } }, + { tool: "t3_pending_request", input: read }, + { tool: "t3_pending_request", input: respond }, + ]), + ), + ); + const knownWait = { threadId, untilQuestion: true, knownQuestionIds: ["request-window-1"] }; + assert.deepEqual( + failed( + byId("pq-wait-past-known-question"), + run([{ tool: "t3_thread_wait", input: knownWait }]), + ), + [], + ); + assert.isNotEmpty( + failed( + byId("pq-wait-past-known-question"), + run([{ tool: "t3_thread_wait", input: { threadId, untilQuestion: true } }]), + ), + ); + assert.isNotEmpty( + failed( + byId("pq-wait-past-known-question"), + run([ + { tool: "t3_thread_wait", input: knownWait }, + { + tool: "t3_pending_request", + input: { action: "respond", threadId, requestId: "request-window-1", answers: {} }, + }, + ]), + ), + ); + const remote = { environmentId: "environment-build-mac", threadId: "thread-ios-release" }; + const scheme = { ...remote, requestId: "request-scheme-1" }; + assert.deepEqual( + failed( + byId("pq-remote-wake-answers-from-task"), + run([ + { tool: "t3_pending_request", input: { ...scheme, action: "read" } }, + { + tool: "t3_pending_request", + input: { ...scheme, action: "respond", answers: { scheme: "Release-Staging" } }, + }, + ]), + ), + [], + ); + // Read on this environment, where the remote thread does not exist. + assert.isNotEmpty( + failed( + byId("pq-remote-wake-answers-from-task"), + run([ + { + tool: "t3_pending_request", + input: { ...scheme, environmentId: undefined, action: "read" }, + }, + { + tool: "t3_pending_request", + input: { ...scheme, environmentId: undefined, action: "respond", answers: {} }, + }, + ]), + ), + ); + const cert = { ...remote, requestId: "request-cert-1", action: "read" }; + assert.deepEqual( + failed( + byId("pq-remote-wake-asks-user"), + run( + [{ tool: "t3_pending_request", input: cert }], + "The build is blocked: the distribution certificate expired. Should it create a new one?", + ), + ), + [], + ); + assert.isNotEmpty( + failed( + byId("pq-remote-wake-asks-user"), + run( + [ + { tool: "t3_pending_request", input: cert }, + { tool: "t3_pending_request", input: { ...cert, action: "respond", answers: {} } }, + ], + "Answered the certificate question.", + ), + ), + ); + assert.isNotEmpty( + failed( + byId("pq-blocking-remote-delegate-question"), + run([ + { tool: "delegate_task", input: { target: { environmentId: remote.environmentId } } }, + { tool: "t3_pending_request", input: { action: "list", threadId: null } }, + ]), + ), + ); + assert.deepEqual( + failed( + byId("pq-control-no-question-wait"), + run([{ tool: "t3_thread_wait", input: { threadId } }], "It copied 1,284,000 rows."), + ), + [], + ); +}); + const messageOriginCases = () => NodeFS.readFileSync(new URL("./cases/message-origin.ndjson", import.meta.url), "utf8") .trim() diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts index 5a4cbf843d809..b97969238ec45 100644 --- a/evals/mcp-tool-use/run.ts +++ b/evals/mcp-tool-use/run.ts @@ -414,8 +414,8 @@ export const McpToolUseCase = Schema.Struct({ /** * Canned tool results by bare name, T3 or native. `"default"` uses the runner's result. For a * device tool, a T3 `DeviceError` (`{"_tag": ..., "cause": null}`) fails it inside T3; for a - * preview tool, a `PreviewAutomationError` fails it with that error's message. A - * `{"$sequence": [...]}` mock answers its nth call with the nth result, then repeats the last. + * preview tool, a `PreviewAutomationError` fails it with that error's message. For + * `$after`, `$sequence` and `$byArgs` mocks see `nextMock`. */ mocks: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), /** Regexes on one part of a shell command. A match wins over the shell simulator. */ @@ -973,6 +973,8 @@ type Sample = { backgroundWait?: BackgroundWait; /** Calls so far per mocked tool, for mocks that answer in sequence. */ readonly mockCalls: Map; + /** Tool calls answered so far in this sample, for `$after` mocks. */ + readonly answered: Array<{ readonly tool: string; readonly input: unknown }>; }; const SHELL_TOOLS = new Set(["Bash", "Read", "Edit", "Write", "WebFetch"]); @@ -1019,15 +1021,50 @@ export const makeSample = (testCase: McpToolUseCase, base: Base): Sample => { priorResults: new Map(), priorErrors: new Set(), mockCalls: new Map(), + answered: [], }; }; -/** A `{"$sequence": [...]}` mock answers its nth call with the nth result, then repeats the last. */ -const nextMock = (sample: Sample, name: string, mock: unknown) => { - if (!isRecord(mock) || !Array.isArray(mock.$sequence) || mock.$sequence.length === 0) return mock; - const n = sample.mockCalls.get(name) ?? 0; - sample.mockCalls.set(name, n + 1); - return mock.$sequence[Math.min(n, mock.$sequence.length - 1)]; +/** + * In this order: `$after` takes the first entry whose `call` the sample already made, `$sequence` + * the nth result for the nth call (then the last), `$byArgs` the first entry whose `args` match + * the call. Args match as `expect` args do; an entry without `call` or `args` always matches. + */ +export const nextMock = ( + sample: Pick & Partial>, + name: string, + given: unknown, + input: Readonly> = {}, +) => { + const made = (call: unknown) => + isRecord(call) && + (sample.answered ?? []).some( + (done) => done.tool === call.tool && matches(call.args ?? {}, done.input), + ); + const mock = + !isRecord(given) || !Array.isArray(given.$after) + ? given + : (() => { + const entry = given.$after.find( + (candidate: unknown) => + isRecord(candidate) && (candidate.call === undefined || made(candidate.call)), + ); + return isRecord(entry) ? entry.result : undefined; + })(); + const sequenced = + !isRecord(mock) || !Array.isArray(mock.$sequence) || mock.$sequence.length === 0 + ? mock + : (() => { + const n = sample.mockCalls.get(name) ?? 0; + sample.mockCalls.set(name, n + 1); + return mock.$sequence[Math.min(n, mock.$sequence.length - 1)]; + })(); + if (!isRecord(sequenced) || !Array.isArray(sequenced.$byArgs)) return sequenced; + const entry = sequenced.$byArgs.find( + (candidate: unknown) => + isRecord(candidate) && (candidate.args === undefined || matches(candidate.args, input)), + ); + return isRecord(entry) ? entry.result : undefined; }; /** @@ -1762,7 +1799,8 @@ export const answerToolCall = Effect.fn("evaluateMcpToolUse.answerToolCall")(fun if (isPageTool(name) && typeof input.path === "string" && !input.path.startsWith("/")) { return failed(`${input.path} is not a readable HTML document. Pass an absolute path.`); } - const mock = nextMock(sample, name, testCase.setup.mocks?.[name]); + const mock = nextMock(sample, name, testCase.setup.mocks?.[name], input); + sample.answered.push({ tool: name, input }); if (isT3Name(call.name) && PREVIEW_TOOLS.has(name)) { if (testCase.setup.world?.browserAgentAccess === false) return failed( diff --git a/packages/contracts/src/orchestratorMcp.ts b/packages/contracts/src/orchestratorMcp.ts index ed6a62bf1eb7f..f6e9b52e2d256 100644 --- a/packages/contracts/src/orchestratorMcp.ts +++ b/packages/contracts/src/orchestratorMcp.ts @@ -240,7 +240,7 @@ export const OrchestratorMcpDelegateTaskResult = Schema.Struct({ resultContextTransferId: Schema.NullOr(ContextTransferId), hasPendingQuestion: Schema.Boolean.annotate({ description: - "A question blocks the child and ends a wait. Answer via `t3_pending_request` on `childThreadId`, or the user does.", + "A question blocks the child and ends a wait. Answer via `t3_pending_request` on `childThreadId` (a remote task: `remoteChild`'s thread and environment), or the user does.", }), pendingApproval: Schema.NullOr( Schema.Struct({ requestId: RuntimeRequestId, kind: ProviderRequestKind, runId: RunId }), @@ -584,6 +584,18 @@ export const OrchestratorMcpThreadWaitInput = Schema.Struct({ "Also wait for its delegated and background tasks and their completions to settle. `resultRun` then names the run with its final answer.", }), ), + untilQuestion: Schema.optional( + Schema.Boolean.annotate({ + description: + "Also return once the thread asks a question; `pendingQuestionIds` names it for `t3_pending_request`.", + }), + ), + knownQuestionIds: Schema.optional( + Schema.Array(RuntimeRequestId).annotate({ + description: + "With untilQuestion: the questions you already know of. Returns when that set changes instead.", + }), + ), }); export type OrchestratorMcpThreadWaitInput = typeof OrchestratorMcpThreadWaitInput.Type; @@ -595,6 +607,10 @@ export const OrchestratorMcpThreadWaitResult = Schema.Struct({ next: Schema.optional(OrchestratorMcpRunningTaskNext), /** With untilDelegatedWorkSettled, once settled: the run whose reply is the thread's result. */ resultRun: Schema.optional(Schema.Struct({ runId: RunId, status: OrchestrationV2RunStatus })), + /** With untilQuestion: the thread's pending questions, oldest first. Absent from linked environments that predate it. */ + pendingQuestionIds: Schema.optional(Schema.Array(RuntimeRequestId)), + /** With untilQuestion, unless timed out: whether the wait settled or the questions changed first. Absent whenever `pendingQuestionIds` is. */ + returnedFor: Schema.optional(Schema.Literals(["settled", "question"])), }); export type OrchestratorMcpThreadWaitResult = typeof OrchestratorMcpThreadWaitResult.Type; diff --git a/scripts/lazurio-release-contract.test.mjs b/scripts/lazurio-release-contract.test.mjs index e200562309998..e7a8162b3f07f 100644 --- a/scripts/lazurio-release-contract.test.mjs +++ b/scripts/lazurio-release-contract.test.mjs @@ -3003,6 +3003,40 @@ NodeTest.test( }, ); +// Row 39: a thread that asks a question wakes whoever waits on it. t3_thread_wait returns on one +// only when asked to, since older followers and `t3 thread wait` read a return as the run's end; +// the remote delegation follower always asks, and the orchestrator keeps its wakes. +NodeTest.test( + "a thread waiting on a question wakes whoever waits on it through its seams", + async () => { + const [contract, threads, follower, orchestrator] = await Promise.all( + [ + "packages/contracts/src/orchestratorMcp.ts", + "apps/server/src/orchestration-v2/ThreadManagementService.ts", + "apps/server/src/peer/RemoteDelegation.ts", + "apps/server/src/orchestration-v2/Orchestrator.ts", + ].map(read), + ); + NodeAssert.match(contract, /untilQuestion: Schema\.optional\(/); + NodeAssert.match( + contract, + /pendingQuestionIds: Schema\.optional\(Schema\.Array\(RuntimeRequestId\)\)/, + ); + NodeAssert.match( + threads, + /\.\.\.\(untilQuestion \? \(\["runtime-request\.updated"\] as const\) : \[\]\)/, + ); + NodeAssert.match( + follower, + /untilQuestion: true,\n\s+knownQuestionIds: questions\.get\(taskId\)\?\.pending \?\? \[\],/, + ); + NodeAssert.match( + orchestrator, + /if \(childThreadId === null\) return task\.remoteChild === undefined;/, + ); + }, +); + // Root decision 0210, plan DEV-6676 (row 25): Buddy's T3 Code is one conversation for every client: // decorators refuse forks, rollbacks, other threads and guest shares (a Personalspace is never // shared, 0091) and take no checkpoints, defaults run before recovery,