diff --git a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts index 4946377c9450..d9f9c4352fe0 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts @@ -1638,6 +1638,250 @@ describe("AcpAdapterV2", () => { assert.deepEqual(promptTexts.slice(3), ["continue"], "a turn resumes only once"); }).pipe(Effect.provide(layerTest)), ); + it.effect( + "re-checks prompt outcome before deferred finalize and discards stale in-flight checks", + () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const selfInvocation = yield* resolveSelfInvocation(); + const mockAgentPath = yield* bundledMockAgentPath; + const instanceId = ProviderInstanceId.make("acp-test-deferred-outcome"); + const threadId = ThreadId.make("thread-acp-deferred-outcome"); + const quota = { + class: "usage_limit", + message: "Quota reached after background work.", + code: "QUOTA_EXHAUSTED", + retryable: true, + resetAt: "2030-01-01T00:00:00.000Z", + } as const; + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + type PromptOutcome = { + readonly failure: typeof quota; + readonly threadDisposition: "reusable" | "broken"; + }; + let sessionUpdateHandler: Parameters[0] | undefined; + const subagentStatuses = new Map([ + ["tool-call-generic-1", "running"], + ]); + const promptRuntimes: Array = []; + const checkStopReasons: Array = []; + let nextOutcome: () => Effect.Effect = () => Effect.undefined; + const promptSettled = yield* Queue.unbounded(); + const deferredScheduled = yield* Queue.unbounded(); + let nextDeferredFinalized: Deferred.Deferred | undefined; + const adapter = yield* makeAcpAdapterV2({ + instanceId, + selfInvocation, + testHooks: { + afterPromptSettledWithBackgroundWork: () => + Queue.offer(promptSettled, undefined).pipe(Effect.asVoid), + onDeferredFinalizeScheduled: () => + Queue.offer(deferredScheduled, undefined).pipe(Effect.asVoid), + afterDeferredFinalize: () => { + const milestone = nextDeferredFinalized; + nextDeferredFinalized = undefined; + return milestone === undefined + ? Effect.void + : Deferred.succeed(milestone, undefined).pipe(Effect.asVoid); + }, + }, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + extractSubagentUpdate: (toolCall) => { + const status = subagentStatuses.get(toolCall.toolCallId); + if (status === undefined) return undefined; + return { + nativeTaskId: `task-${toolCall.toolCallId}`, + prompt: toolCall.toolCallId, + title: toolCall.toolCallId, + model: null, + status, + childSessionId: null, + result: status === "completed" ? "DONE" : null, + }; + }, + watchPromptOutcome: () => + Effect.succeed((result: EffectAcpSchema.PromptResponse) => + Effect.sync(() => { + checkStopReasons.push(result.stopReason); + }).pipe(Effect.andThen(Effect.suspend(() => nextOutcome()))), + ), + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, + wrapRuntime: (runtime, runtimeOrdinal) => ({ + ...runtime, + handleSessionUpdate: (handler) => + Effect.sync(() => { + sessionUpdateHandler = handler; + }).pipe(Effect.andThen(runtime.handleSessionUpdate(handler))), + prompt: (payload, promptOptions) => + Effect.sync(() => { + promptRuntimes.push(runtimeOrdinal); + }).pipe(Effect.andThen(runtime.prompt(payload, promptOptions))), + }), + }), + }, + }); + const runtimePolicy = ProviderAdapter.ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-deferred-outcome"), + modelSelection, + runtimePolicy, + }); + const terminals = + yield* Queue.unbounded< + Extract + >(); + yield* runtime.events.pipe( + Stream.filter((event) => event.type === "turn.terminal"), + Stream.runForEach((event) => Queue.offer(terminals, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const startTurn = Effect.fnUntraced(function* (ordinal: number) { + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + ordinal, + }), + ); + }); + const emitSubagent = ( + toolCallId: string, + status: "running" | "completed", + sessionUpdate: "tool_call" | "tool_call_update" = "tool_call_update", + ) => { + subagentStatuses.set(toolCallId, status); + return sessionUpdateHandler!({ + sessionId: "mock-session-1", + update: { + sessionUpdate, + toolCallId, + title: toolCallId, + kind: "other", + status: status === "running" ? "in_progress" : "completed", + ...(status === "completed" ? { rawOutput: { content: "DONE" } } : {}), + }, + }); + }; + + yield* startTurn(1); + yield* Queue.take(promptSettled); + assert.deepEqual(checkStopReasons, ["end_turn"]); + assert.isDefined(sessionUpdateHandler); + + nextOutcome = () => Effect.succeed({ failure: quota, threadDisposition: "broken" }); + yield* emitSubagent("tool-call-generic-1", "completed"); + yield* Queue.take(deferredScheduled); + yield* TestClock.adjust("3000 millis"); + + const firstTerminal = yield* Queue.take(terminals); + assert.equal(firstTerminal.status, "failed"); + assert.deepEqual(firstTerminal.failure, quota); + assert.equal(firstTerminal.threadDisposition, "broken"); + assert.deepEqual(checkStopReasons, ["end_turn", "end_turn"]); + + nextOutcome = () => Effect.undefined; + subagentStatuses.set("tool-call-generic-1", "running"); + yield* startTurn(2); + yield* Queue.take(promptSettled); + assert.deepEqual(promptRuntimes, [1, 2], "hidden deferred failure must restart runtime"); + + const workRaceStarted = yield* Deferred.make(); + const releaseWorkRace = yield* Deferred.make(); + nextOutcome = () => + Deferred.succeed(workRaceStarted, undefined).pipe( + Effect.andThen(Deferred.await(releaseWorkRace)), + Effect.as({ failure: quota, threadDisposition: "reusable" }), + ); + yield* emitSubagent("tool-call-generic-1", "completed"); + yield* Queue.take(deferredScheduled); + yield* TestClock.adjust("3000 millis"); + yield* Deferred.await(workRaceStarted); + + yield* emitSubagent("tool-call-generic-2", "running", "tool_call"); + const workRaceSettled = yield* Deferred.make(); + nextDeferredFinalized = workRaceSettled; + nextOutcome = () => Effect.undefined; + yield* Deferred.succeed(releaseWorkRace, undefined); + // Settle the exact blocked attempt while the second subagent still holds the turn open. + yield* Deferred.await(workRaceSettled); + assert.deepEqual(yield* Queue.clear(terminals), []); + + yield* emitSubagent("tool-call-generic-2", "completed"); + yield* Queue.take(deferredScheduled); + yield* TestClock.adjust("3000 millis"); + const secondTerminal = yield* Queue.take(terminals); + assert.equal(secondTerminal.status, "completed"); + + subagentStatuses.set("tool-call-generic-1", "running"); + yield* startTurn(3); + yield* Queue.take(promptSettled); + assert.deepEqual( + promptRuntimes, + [1, 2, 2], + "discarded deferred failure must not restart runtime", + ); + + const interruptRaceStarted = yield* Deferred.make(); + const releaseInterruptRace = yield* Deferred.make(); + nextOutcome = () => + Deferred.succeed(interruptRaceStarted, undefined).pipe( + Effect.andThen(Deferred.await(releaseInterruptRace)), + Effect.as({ failure: quota, threadDisposition: "reusable" }), + ); + yield* emitSubagent("tool-call-generic-1", "completed"); + yield* Queue.take(deferredScheduled); + yield* TestClock.adjust("3000 millis"); + yield* Deferred.await(interruptRaceStarted); + + yield* runtime.interruptTurn({ + providerThread, + providerTurnId: idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: acpScopedNativeId(instanceId, "mock-session-1:turn:3"), + }), + }); + const thirdTerminal = yield* Queue.take(terminals); + assert.equal(thirdTerminal.status, "interrupted"); + + const interruptRaceSettled = yield* Deferred.make(); + nextDeferredFinalized = interruptRaceSettled; + nextOutcome = () => Effect.undefined; + yield* Deferred.succeed(releaseInterruptRace, undefined); + yield* Deferred.await(interruptRaceSettled); + assert.deepEqual(yield* Queue.clear(terminals), []); + + subagentStatuses.set("tool-call-generic-1", "completed"); + yield* startTurn(4); + const fourthTerminal = yield* Queue.take(terminals); + assert.equal(fourthTerminal.status, "completed"); + assert.deepEqual( + promptRuntimes, + [1, 2, 2, 2], + "interrupted deferred check must not restart runtime", + ); + }).pipe(Effect.provide(layerTest)), + ); it.live("retains returned usage when Stop finalizes before the prompt callback", () => Effect.gen(function* () { const instanceId = ProviderInstanceId.make("acp-test-prompt-usage-stop"); diff --git a/packages/provider-acp/src/server/adapter.ts b/packages/provider-acp/src/server/adapter.ts index cf7bb6c0f92f..29a4b563619d 100644 --- a/packages/provider-acp/src/server/adapter.ts +++ b/packages/provider-acp/src/server/adapter.ts @@ -787,6 +787,7 @@ export interface AcpAdapterV2Options { }; readonly testHooks?: { readonly onDeferredFinalizeScheduled?: (debounce: Duration.Input) => Effect.Effect; + readonly afterDeferredFinalize?: () => Effect.Effect; /** `cancelAfterPromptStarts`: Stop is holding `session/cancel` until the prompt starts. */ readonly onCancelHeld?: () => Effect.Effect; readonly afterPromptWireSettled?: () => Effect.Effect; @@ -1726,6 +1727,9 @@ interface ActiveAcpTurn { /** Completed when an update after dispatch matches `cancelAfterPromptStarts.isStart`. */ readonly promptStarted: Deferred.Deferred; promptUsage: EffectAcpSchema.Usage | undefined; + checkPromptOutcome: + | Effect.Success>> + | undefined; backgroundFinalizeGeneration: number; /** `replyEndsWithUsageUpdate`: output arrived and its closing `usage_update` has not. */ replyOpen: boolean; @@ -8733,6 +8737,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( if (hasDeferredBackgroundWork(context)) return; context.backgroundFinalizeGeneration += 1; const generation = context.backgroundFinalizeGeneration; + const runtimeGeneration = yield* Ref.get(runtimeCallbackGeneration); const replyOpen = context.replyOpen; const debounce = replyOpen ? ACP_OPEN_REPLY_QUIET_LIMIT : ACP_DEFERRED_FINALIZE_DEBOUNCE; yield* Effect.gen(function* () { @@ -8754,7 +8759,47 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( return; } const status = context.promptSettledStatus ?? "completed"; - yield* finalizeTurn(context, status); + const checkPromptOutcome = + status === "completed" ? context.checkPromptOutcome : undefined; + if (checkPromptOutcome === undefined) { + yield* finalizeTurn(context, status); + yield* options.testHooks?.afterDeferredFinalize?.() ?? Effect.void; + return; + } + if ( + (yield* Ref.get(runtimeTeardownState))._tag !== "Idle" || + (yield* Ref.get(runtimeCallbackGeneration)) !== runtimeGeneration + ) { + return; + } + // A completed status implies end_turn. Deferred finalization reports hidden failures + // directly rather than re-running followPrompt's resumeWith retry loop. + const hiddenFailure = yield* Effect.suspend(() => + checkPromptOutcome({ stopReason: "end_turn" }), + ).pipe( + Effect.catchDefect(() => + Effect.logWarning("Could not check deferred ACP prompt outcome.").pipe( + Effect.as(undefined), + ), + ), + ); + yield* runRuntimeCallbackAtGeneration( + runtimeGeneration, + Effect.gen(function* () { + if (!stillCurrent()) return; + if (hiddenFailure !== undefined) { + yield* Ref.set(runtimeRestartRequired, true); + yield* terminalizeOpenForegroundTools(context, "failed"); + yield* finalizeTurn(context, "failed", hiddenFailure.failure, { + childTurnsContinue: hasSurvivingBackgroundWork(context), + threadDisposition: hiddenFailure.threadDisposition, + }); + return; + } + yield* finalizeTurn(context, status); + }), + ); + yield* options.testHooks?.afterDeferredFinalize?.() ?? Effect.void; }).pipe(Effect.forkIn(sessionScope), Effect.asVoid); yield* options.testHooks?.onDeferredFinalizeScheduled?.(debounce) ?? Effect.void; }); @@ -9445,6 +9490,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( promptSentAt: null, promptStarted, promptUsage: undefined, + checkPromptOutcome: undefined, backgroundFinalizeGeneration: 0, replyOpen: false, nativePromptId: undefined, @@ -9601,6 +9647,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( flavor.watchPromptOutcome === undefined ? undefined : yield* flavor.watchPromptOutcome(requestedSessionId); + context.checkPromptOutcome = checkPromptOutcome; const promptGeneration = yield* Ref.get(runtimeCallbackGeneration); const promptOptions = { dispatched: context.promptDispatched, @@ -9702,6 +9749,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( context.promptSettled = false; context.promptSettledStatus = null; context.backgroundFinalizeGeneration += 1; + context.checkPromptOutcome = checkPromptOutcome; context.promptSentAt = yield* Clock.currentTimeMillis; context.lastProgressAt = context.promptSentAt; const watchdog = context.inactivityWatchdog;