diff --git a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts index d71b0f7b5e2b8..de1a613b5dbf2 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts @@ -9273,6 +9273,7 @@ describe("AcpAdapterV2", () => { const userPromptStarted = yield* Deferred.make(); const releaseUserPrompt = yield* Deferred.make(); const bufferedAssistantText = "BUFFERED_WAKE_QUEUED_AFTER_USER"; + const lostReplyModel = "composer-2[fast=true]"; let promptOrdinal = 0; type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; let sessionUpdateHandler: Parameters[0] | undefined; @@ -9304,6 +9305,20 @@ describe("AcpAdapterV2", () => { } return result; }), + // The agent applies this model, but its reply never arrives. + setConfigOption: (id, value, options) => + value === lostReplyModel + ? runtime.setConfigOption(id, value, options).pipe( + Effect.andThen( + Effect.fail( + new EffectAcpErrors.AcpTransportError({ + detail: "Injected lost reply", + cause: "test", + }), + ), + ), + ) + : runtime.setConfigOption(id, value, options), }), }), }, @@ -9458,6 +9473,76 @@ describe("AcpAdapterV2", () => { assert.equal(continuationTerminalStatus, "completed"); assert.isTrue(bufferedTextSeen, "continuation must drain the wake buffer after the user run"); assert.isFalse(yield* hasPendingBackgroundWork); + + // A model write whose outcome is unknown leaves the producing selection unknown, and the + // output buffered under it still drains with the next wake. + const lostReply = yield* Effect.exit( + runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + ordinal: 4, + modelSelection: { instanceId, model: lostReplyModel }, + }), + ), + ); + assert.isTrue(Exit.isFailure(lostReply)); + const wakeText = (text: string) => + sessionUpdateHandler!({ + sessionId: "mock-session-1", + update: { sessionUpdate: "agent_message_chunk", content: { type: "text", text } }, + }); + yield* wakeText("UNKNOWN_SELECTION_WAKE"); + assert.isTrue(yield* runtime.hasBufferedOutputForThread!(providerThread)); + assert.isNull(yield* runtime.bufferedExecutionSelection!(providerThread)); + const composer = { instanceId, model: "composer-2" } as const; + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + ordinal: 5, + modelSelection: composer, + }), + ); + yield* Stream.fromQueue(events).pipe( + Stream.takeUntil((event) => event.type === "turn.terminal"), + Stream.runDrain, + ); + yield* wakeText("KNOWN_SELECTION_WAKE"); + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + ordinal: 6, + modelSelection: composer, + messageCreatedBy: "agent", + messageCreationSource: "provider", + messageText: "Background task completed.", + }), + ); + yield* TestClock.adjust("3 seconds"); + const drained = Array.from( + yield* Stream.fromQueue(events).pipe( + Stream.takeUntil((event) => event.type === "turn.terminal"), + Stream.runCollect, + ), + ).flatMap((event) => + event.type === "message.updated" && event.message.role === "assistant" + ? [event.message.text] + : [], + ); + assert.include(drained.join(""), "UNKNOWN_SELECTION_WAKE"); + assert.include(drained.join(""), "KNOWN_SELECTION_WAKE"); + assert.isFalse(yield* runtime.hasBufferedOutputForThread!(providerThread)); }).pipe(Effect.provide(layerTest)), ); diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 4bb712aa9ffca..9f1339cbcb4e2 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -3557,6 +3557,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { readonly settings?: ClaudeSettings; readonly modelCatalog?: Effect.Effect; readonly freshQueueOnReopen?: boolean; + readonly reopen?: Effect.Effect; readonly forkSession?: ClaudeAdapterV2.ClaudeAgentSdkQueryRunnerShape["forkSession"]; readonly allocateSessionId?: Effect.Effect; readonly offer?: (message: SDKUserMessage) => Effect.Effect; @@ -3643,6 +3644,9 @@ describe("ClaudeAdapterV2 background wake turns", () => { Effect.succeed(options?.nativeSessionId ?? WAKE_NATIVE_SESSION), open: (input) => Effect.gen(function* () { + if (options?.reopen !== undefined && processQueues.length > 0) { + return yield* options.reopen; + } openedOptions = input.options; const processMessages = options?.freshQueueOnReopen === true && processQueues.length > 0 @@ -3833,6 +3837,29 @@ describe("ClaudeAdapterV2 background wake turns", () => { }); const makeWakeHarness = makeWakeHarnessWithOptions(); + // An idle process's exit emits nothing by itself. One still running a background command + // drops that command from the roster in its exit cleanup, which is the event awaited here. + const exitIdleProcess = Effect.fnUntraced(function* ( + harness: Effect.Success, + ) { + const taskId = "task-running-at-exit"; + yield* harness.offerAndWait( + claudeSdkFrame({ + ...wakeTaskStarted, + task_id: taskId, + tool_use_id: "toolu_01RunningAtExit", + uuid: "00000000-0000-4000-8000-000000000109", + }), + ); + const rosterListsTask = () => + harness.events + .findLast((event) => event.type === "provider_thread.updated") + ?.providerThread.pendingBackgroundTasks?.some((task) => task.taskId === taskId) === true; + yield* harness.until(rosterListsTask); + yield* Queue.shutdown(harness.sdkMessages); + yield* harness.until(() => !rosterListsTask()); + }); + describe("ClaudeAdapterV2 session title sync", () => { type TitleSyncHarness = Effect.Success>; const startTurn = Effect.fnUntraced(function* ( @@ -7544,6 +7571,25 @@ describe("ClaudeAdapterV2 background wake turns", () => { { effortLevel: "high", fastMode: null, ultracode: null }, ]); assert.lengthOf(harness.offeredMessages, 2); + + // A wake after the steered turn is the steered model's output. + yield* harness.offerAndWait( + makeResultFrame({ uuid: "00000000-0000-4000-8000-000000000c01", result: "Sonnet." }), + ); + yield* Queue.take(harness.terminalReceipts); + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "user", + message: { role: "user", content: "A background task finished." }, + parent_tool_use_id: null, + uuid: "00000000-0000-4000-8000-000000000c02", + session_id: WAKE_NATIVE_SESSION, + }), + ); + assert.deepEqual(yield* harness.runtime.bufferedExecutionSelection!(harness.providerThread), { + ...opus, + model: "claude-sonnet-4-6", + }); }).pipe( Effect.provide( Layer.mergeAll( @@ -7649,10 +7695,29 @@ describe("ClaudeAdapterV2 background wake turns", () => { yield* harness.offerAndWait(turnOneResult); yield* Queue.take(harness.terminalReceipts); + assert.deepEqual( + yield* harness.runtime.liveExecutionSelection!(harness.providerThread), + opus, + ); + const failed = yield* harness.runtime .startTurn(turn(2, { ...opus, model: "claude-sonnet-4-6" })) .pipe(Effect.result); assert.equal(failed._tag, "Failure"); + assert.isNull(yield* harness.runtime.liveExecutionSelection!(harness.providerThread)); + + // Output the process buffers now came from an unknown selection, not from Opus. + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "user", + message: { role: "user", content: "A background task finished." }, + parent_tool_use_id: null, + uuid: "00000000-0000-4000-8000-000000000c03", + session_id: WAKE_NATIVE_SESSION, + }), + ); + assert.isTrue(yield* harness.runtime.hasBufferedOutputForThread!(harness.providerThread)); + assert.isNull(yield* harness.runtime.bufferedExecutionSelection!(harness.providerThread)); // The process may now run Sonnet, so returning to Opus must send it. yield* harness.runtime.startTurn(turn(3, opus)); @@ -15393,7 +15458,14 @@ describe("ClaudeAdapterV2 background wake turns", () => { it.effect("buffers wake output and requests a single continuation run", () => Effect.gen(function* () { - const harness = yield* makeWakeHarness; + const harness = yield* makeWakeHarnessWithOptions({ + reopen: Effect.fail( + new ClaudeAdapterV2.ClaudeAgentSdkQueryRunnerError({ + method: "open", + cause: "rejected open", + }), + ), + }); const now = yield* DateTime.now; yield* harness.runtime.startTurn( @@ -15424,10 +15496,50 @@ describe("ClaudeAdapterV2 background wake turns", () => { assert.equal(harness.continuationRequests[0]?.driver, ClaudeAdapterV2.CLAUDE_PROVIDER); assert.equal(harness.continuationRequests[0]?.detail, WAKE_SUMMARY); + // The wake starts a background subagent that is still running at the root's result. + yield* harness.offerAndWait( + makeSubagentTaskStartedFrame({ + taskId: "a2f6c1d09b3e4f587", + toolUseId: "toolu_01ExitedWakeSubagentLaunch", + uuid: "00000000-0000-4000-8000-000000000108", + }), + ); yield* harness.offerAndWait(wakeResult); assert.lengthOf(harness.continuationRequests, 1); assert.lengthOf(harness.terminalEvents(), 1); assert.isTrue(yield* harness.hasPendingBackgroundWork); + + // The process exits with the whole wake buffered, so delivering it needs no new process. + yield* exitIdleProcess(harness); + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-wake-2"), + text: "Background task completed.", + attachments: [], + providerTurnOrdinal: 2, + messageCreatedBy: "agent", + messageCreationSource: "provider", + }), + ); + yield* harness.awaitTerminal(2); + assert.equal(harness.terminalEvents()[1]?.status, "completed"); + assert.isTrue( + harness.events.some( + (event) => event.type === "message.updated" && event.message.text === WAKE_ASSISTANT_TEXT, + ), + ); + assert.isFalse(yield* harness.runtime.hasBufferedOutputForThread!(harness.providerThread)); + // Nothing is left to end the subagent the replayed wake started, so the replay does. + assert.deepEqual( + harness.events.flatMap((event) => + event.type === "subagent.updated" ? [event.subagent.status] : [], + ), + ["running", "failed"], + ); + assert.isFalse(yield* harness.hasPendingBackgroundWork); }).pipe( Effect.provide( Layer.mergeAll( @@ -16419,7 +16531,14 @@ describe("ClaudeAdapterV2 background wake turns", () => { "carries a rejected wake rate limit into the continuation failure (event=%s)", (standaloneEvent) => Effect.gen(function* () { - const harness = yield* makeWakeHarness; + const harness = yield* makeWakeHarnessWithOptions({ + reopen: Effect.fail( + new ClaudeAdapterV2.ClaudeAgentSdkQueryRunnerError({ + method: "open", + cause: "rejected open", + }), + ), + }); const now = yield* DateTime.now; const resetsAt = 1790298600; @@ -16494,6 +16613,8 @@ describe("ClaudeAdapterV2 background wake turns", () => { }), ); + // The process exits behind that lifecycle result; the wake is still whole. + yield* exitIdleProcess(harness); yield* harness.runtime.startTurn( makeClaudeTestTurnInput({ threadId: harness.threadId, @@ -17048,12 +17169,29 @@ describe("ClaudeAdapterV2 background wake turns", () => { assert.deepInclude(harness.liveSelectionChanges, { model: "claude-sonnet-4-6" }); assert.lengthOf(harness.continuationRequests, 0); assert.isFalse(yield* harness.hasPendingBackgroundWork); + assert.isFalse(yield* harness.runtime.hasBufferedOutputForThread!(harness.providerThread)); // A root user frame the CLI produced on its own still marks a wake. yield* harness.offerAndWait( idleUserFrame("00000000-0000-4000-8000-000000000b02", "A background task finished.", false), ); assert.isTrue(yield* harness.hasPendingBackgroundWork); + // The wake belongs to the turn that produced it, so no other selection may replace + // its process before it is delivered. + assert.deepEqual( + yield* harness.runtime.bufferedExecutionSelection!(harness.providerThread), + turn(2, "claude-sonnet-4-6").modelSelection, + ); + const replaced = yield* harness.runtime + .startTurn({ ...turn(3, ""), modelSelection: CLAUDE_TEST_RESTART_MODEL_SELECTION }) + .pipe(Effect.result); + assert.equal(replaced._tag, "Failure"); + if (replaced._tag === "Failure") { + assert.instanceOf( + replaced.failure.cause, + ProviderAdapter.ProviderAdapterBufferedOutputError, + ); + } }).pipe( Effect.provide( Layer.mergeAll( diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index bced035fed011..65e9d44b0e892 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -5044,6 +5044,8 @@ interface ClaudeLiveQueryContext { selection: CompiledClaudeModelSelection; readonly closed: Deferred.Deferred; lastTurn: ActiveClaudeTurnContext | null; + // The last turn whose prompt this process took; its selection produced any later output. + acknowledgedTurn: ProviderAdapter.ProviderAdapterV2TurnInput | null; // "early" echoes on the first frame; "acknowledged" confirms uuid acceptance, but resume turns can still precede the prompt. promptEchoMode: "unknown" | "acknowledged" | "early" | "result_only"; // The CLI may fall back from the requested mode on its first init frame. @@ -5899,10 +5901,20 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ); // A gateway's relative reset counts from the result's arrival, not its replay. const bufferedResultArrivals = new WeakMap(); + // `producer` is the process and acknowledged turn the first buffered frame came from. const wakeBuffers = yield* Ref.make( new Map< string, - { readonly messages: ReadonlyArray; readonly detail: string | null } + { + readonly messages: ReadonlyArray; + readonly detail: string | null; + readonly producer: { + readonly query: ClaudeLiveQueryContext; + readonly input: ProviderAdapter.ProviderAdapterV2TurnInput; + // False once a frame arrives after a failed live selection update. + readonly selectionKnown: boolean; + } | null; + } >(), ); // Wake results lack task ids and can follow queued prompts; retain reports for one user turn. @@ -9835,12 +9847,22 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ) { monitorEventWakeStarts.add(message); } + const live = yield* Ref.get(queryContext); yield* Ref.update(wakeBuffers, (current) => { const existing = current.get(wakeInput.nativeThreadId); + const producer = + existing?.producer ?? + (live?.acknowledgedTurn == null + ? null + : { query: live, input: live.acknowledgedTurn, selectionKnown: true }); const updated = new Map(current); updated.set(wakeInput.nativeThreadId, { messages: [...(existing?.messages ?? []), message], detail: notificationSummary ?? existing?.detail ?? null, + producer: + producer !== null && producer.query === live && live.selection.queryIdentity === "" + ? { ...producer, selectionKnown: false } + : producer, }); return updated; }); @@ -10023,7 +10045,11 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const updated = new Map(current); if (messages.length === 0) updated.delete(nativeThreadId); else - updated.set(nativeThreadId, { messages, detail: claudeWakeBufferDetail(messages) }); + updated.set(nativeThreadId, { + messages, + detail: claudeWakeBufferDetail(messages), + producer: current.get(nativeThreadId)?.producer ?? null, + }); return [true, updated] as const; }); if (!replaced) return false; @@ -10317,8 +10343,10 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( readonly query: ClaudeAgentSdkQuerySession; readonly message: SDKMessage; readonly replayed?: boolean; + /** The exited process whose buffered wake is being replayed. */ + readonly replayContext?: ClaudeLiveQueryContext; }) { - const liveQuery = yield* Ref.get(queryContext); + const liveQuery = input.replayContext ?? (yield* Ref.get(queryContext)); if (liveQuery?.query !== input.query) { return; } @@ -11820,15 +11848,16 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const handleRoutedSdkMessage = Effect.fnUntraced(function* (input: { readonly query: ClaudeAgentSdkQuerySession; readonly message: SDKMessage; + readonly replayContext?: ClaudeLiveQueryContext; }) { if (input.message.type !== "system" || input.message.subtype !== "task_started") { for (const message of yield* takeReleasableSubagentFrames(input.message)) { - yield* handleSdkMessageFrame({ query: input.query, message, replayed: true }); + yield* handleSdkMessageFrame({ ...input, message, replayed: true }); } } yield* handleSdkMessageFrame(input); for (const message of yield* takeReleasableSubagentFrames(input.message)) { - yield* handleSdkMessageFrame({ query: input.query, message, replayed: true }); + yield* handleSdkMessageFrame({ ...input, message, replayed: true }); } }); @@ -11858,10 +11887,11 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const handleSdkMessage = Effect.fnUntraced(function* (input: { readonly query: ClaudeAgentSdkQuerySession; readonly message: SDKMessage; + readonly replayContext?: ClaudeLiveQueryContext; }) { const message = input.message; const context = yield* Ref.get(activeTurn); - const liveQuery = yield* Ref.get(queryContext); + const liveQuery = input.replayContext ?? (yield* Ref.get(queryContext)); // The API answering again is recovery; a long text or tool input block emits nothing // else. Judged here, in arrival order: held and buffered frames are handled later. if ( @@ -12494,6 +12524,56 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }), ); + // The wake's process is still this thread's (or exited on its own), so its turn's + // selection produced the output; a replaced process leaves that unknown. + const ownedBufferedWake = Effect.fnUntraced(function* ( + providerThread: OrchestrationV2ProviderThread, + ) { + const nativeThreadId = providerThread.nativeThreadRef?.nativeId; + if (nativeThreadId == null) return null; + const buffered = (yield* Ref.get(wakeBuffers)).get(nativeThreadId); + const producer = buffered?.producer; + const live = yield* Ref.get(queryContext); + if ( + buffered === undefined || + producer == null || + producer.query.stopping || + producer.query.nativeThreadId !== nativeThreadId || + producer.input.providerThread.id !== providerThread.id || + (live !== null && live !== producer.query) || + !(yield* holdsBufferedWake(nativeThreadId, buffered.messages)) + ) { + return null; + } + return producer; + }); + + // A process that exited on its own after buffering a whole wake left nothing to + // resume: the frames are replayed against it, with no new process to start. + const exitedWakeProducer = Effect.fnUntraced(function* ( + providerThread: OrchestrationV2ProviderThread, + nativeThreadId: string, + ) { + if ((yield* Ref.get(queryContext)) !== null) return null; + const producer = yield* ownedBufferedWake(providerThread); + const messages = (yield* Ref.get(wakeBuffers)).get(nativeThreadId)?.messages ?? []; + const rootResult = messages.findLastIndex( + (entry) => entry.type === "result" && !isClaudeEmptyTaskNotificationSuccess(entry), + ); + // Only session-task lifecycle turns that ended may follow the root's result. + return producer !== null && + rootResult !== -1 && + messages.at(-1)?.type === "result" && + messages + .slice(rootResult + 1) + .every( + (entry) => + isClaudeTurnStartMessage(entry) || isClaudeEmptyTaskNotificationSuccess(entry), + ) + ? producer.query + : null; + }); + const liveProcessRunsBackgroundWork = Effect.fnUntraced(function* ( live: ClaudeLiveQueryContext, options?: { readonly forGoalCheck?: boolean }, @@ -12533,6 +12613,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const applySelectionLive = Effect.fnUntraced(function* ( live: ClaudeLiveQueryContext, selection: CompiledClaudeModelSelection, + modelSelection: ModelSelection, ) { // Preserve launcher-selected models unless the model changed or a failed update left it unknown. yield* ( @@ -12551,6 +12632,10 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ), ); live.selection = selection; + // The process runs this selection from here on, so it produces any later wake. + if (live.acknowledgedTurn !== null) { + live.acknowledgedTurn = { ...live.acknowledgedTurn, modelSelection }; + } }); const openQuery = Effect.fnUntraced(function* ( @@ -12620,7 +12705,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( (existing.selection.apiModelId === compiledSelection.apiModelId || existing.openedPermissionMode === queryPolicy.permissionMode) ) { - yield* applySelectionLive(existing, compiledSelection); + yield* applySelectionLive(existing, compiledSelection, turnInput.modelSelection); } if ( !turnInput.resume && @@ -12650,6 +12735,21 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( return existing; } + // Replacing the process would attribute its undelivered wake to the new selection. + // Resume and wake turns deliver that wake themselves. + if ( + existing !== null && + !existing.stopping && + !turnInput.resume && + !isClaudeProviderContinuationTurn(turnInput) && + existing.acknowledgedTurn !== null && + (yield* ownedBufferedWake(existing.acknowledgedTurn.providerThread)) !== null + ) { + return yield* new ProviderAdapter.ProviderAdapterBufferedOutputError({ + driver: CLAUDE_PROVIDER, + providerThreadId: existing.acknowledgedTurn.providerThread.id, + }); + } // Replacing this thread's process kills its background work, even with a new native identity. if ( existing !== null && @@ -12802,6 +12902,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( selection: compiledSelection, closed, lastTurn: null, + acknowledgedTurn: null, promptEchoMode: "unknown", openedPermissionMode: queryOptions.permissionMode, receivedInit: false, @@ -13050,8 +13151,20 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), Effect.provideService(Path.Path, path), ); - const querySession = yield* openQuery(turnInput, nativeThreadId, queryPolicy); - yield* applySessionTitle(querySession.query, nativeThreadId, turnInput.appThread.title); + const exitedProducer = + promptUuid === null && !turnInput.resume + ? yield* exitedWakeProducer(turnInput.providerThread, nativeThreadId) + : null; + const querySession = + exitedProducer ?? (yield* openQuery(turnInput, nativeThreadId, queryPolicy)); + if (exitedProducer === null) { + yield* applySessionTitle( + querySession.query, + nativeThreadId, + turnInput.appThread.title, + ); + } + const replayContext = exitedProducer === null ? {} : { replayContext: exitedProducer }; const autoCompactWindow = yield* resolveAutoCompactWindow( querySession.settingsCwd, turnInput.modelSelection, @@ -13123,7 +13236,12 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( // Set before claiming: a query that exits after the claim finalizes this turn; one that exits before it fails the start. yield* Ref.set(activeTurn, context); const attached = yield* Ref.modify(queryContext, (current) => { - if (current?.query !== querySession.query || querySession.stopping) { + if ( + (exitedProducer === null + ? current?.query !== querySession.query + : current !== null) || + querySession.stopping + ) { return [false, current]; } querySession.lastTurn = context; @@ -13207,6 +13325,11 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ).pipe(Effect.tap(() => Effect.sync(() => pdfReservation.commit()))), ), ), + Effect.tap(() => + Effect.sync(() => { + querySession.acknowledgedTurn = turnInput; + }), + ), ); if (userMessage !== null && !turnInput.resume) { yield* attachTurn; @@ -13282,7 +13405,11 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( continue; } if (entry.type !== "result") { - yield* handleSdkMessage({ query: querySession.query, message: entry }); + yield* handleSdkMessage({ + query: querySession.query, + message: entry, + ...replayContext, + }); } yield* Ref.update(wakeBuffers, (current) => { const buffered = current.get(nativeThreadId); @@ -13299,8 +13426,22 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( return next; }); } + if (exitedProducer !== null) { + // Its exit cleanup ran before these frames started or reopened subagents. + yield* endSubagentsOfClosedQuery({ + closedQuery: exitedProducer, + context, + subagents: yield* Ref.get(sessionSubagentsByTaskId), + buffered: (yield* Ref.get(wakeBuffers)).get(nativeThreadId)?.messages, + interrupted: false, + }); + } if (lastResult !== undefined && !emptyTurnBeforeNextTurn) { - yield* handleSdkMessage({ query: querySession.query, message: lastResult }); + yield* handleSdkMessage({ + query: querySession.query, + message: lastResult, + ...replayContext, + }); if (leavesLaterWake) yield* offerBufferedWake(nativeThreadId); return; } @@ -13591,8 +13732,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( Effect.provideService(Path.Path, path), ); yield* backgroundForegroundSubagents(existing, backgroundDeadline); - if (liveSteerSelection !== undefined) { - yield* applySelectionLive(existing, liveSteerSelection); + if (liveSteerSelection !== undefined && turnInput.modelSelection !== undefined) { + yield* applySelectionLive(existing, liveSteerSelection, turnInput.modelSelection); } // The turn can end while the moves or the selection change are answered. // A steer offered then would start a new turn the orchestrator never @@ -13793,6 +13934,41 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } return false; }), + continuationDrainsOutput: true, + liveExecutionSelection: Effect.fn("ClaudeAdapterV2.liveExecutionSelection")( + function* (providerThread) { + const live = yield* Ref.get(queryContext); + const acknowledged = live?.acknowledgedTurn; + if ( + live == null || + acknowledged == null || + live.stopping || + acknowledged.providerThread.id !== providerThread.id || + live.nativeThreadId !== providerThread.nativeThreadRef?.nativeId + ) { + return null; + } + // A failed live update blanks the identity, so the process's selection is unknown. + const requested = compileClaudeModelSelection( + acknowledged.modelSelection, + yield* routedCatalogFor(live), + ); + return live.selection.queryIdentity === requested.queryIdentity + ? acknowledged.modelSelection + : null; + }, + ), + bufferedExecutionSelection: Effect.fn("ClaudeAdapterV2.bufferedExecutionSelection")( + function* (providerThread) { + const producer = yield* ownedBufferedWake(providerThread); + return producer?.selectionKnown === true ? producer.input.modelSelection : null; + }, + ), + hasBufferedOutputForThread: Effect.fn("ClaudeAdapterV2.hasBufferedOutputForThread")( + function* (providerThread) { + return (yield* ownedBufferedWake(providerThread)) !== null; + }, + ), // Untraced: the run's event stream calls this once per provider event. hasPendingBackgroundWorkForThread: Effect.fnUntraced(function* (providerThread, options) { const nativeThreadId = providerThread.nativeThreadRef?.nativeId; @@ -14028,7 +14204,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( return; } // A failed update clears the live identity, so the next turn applies the selection again. - yield* applySelectionLive(live, selection).pipe( + yield* applySelectionLive(live, selection, modelSelection).pipe( Effect.catch((cause) => Effect.logWarning("orchestration-v2.claude-live-selection-failed", { cause }), ), diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 4947f4da8a440..7e541e79d8e14 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -7845,6 +7845,15 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }; const runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime = { + preservesSelectionOnSameSelectionFailure: true, + reappliesFullSelection: (selection) => + Effect.succeed( + getModelSelectionStringOptionValue(selection, "reasoningEffort") !== undefined && + getCodexServiceTierOptionValue(selection) !== undefined && + (selection.options ?? []).every((option) => + ["reasoningEffort", "serviceTier", "fastMode"].includes(option.id), + ), + ), instanceId: adapterOptions.instanceId, driver: CODEX_PROVIDER, providerSessionId: input.providerSessionId, diff --git a/apps/server/src/orchestration-v2/EffectOutbox.ts b/apps/server/src/orchestration-v2/EffectOutbox.ts index 69fd60684bbcb..bfab945d10ad7 100644 --- a/apps/server/src/orchestration-v2/EffectOutbox.ts +++ b/apps/server/src/orchestration-v2/EffectOutbox.ts @@ -43,6 +43,7 @@ export const OrchestrationEffectRequestV2 = Schema.Union([ providerSessionId: ProviderSessionId, detail: Schema.optional(Schema.String), revokeMcpCredential: Schema.optional(Schema.Boolean), + preserveBufferedOutput: Schema.optional(Schema.Boolean), }), Schema.Struct({ type: Schema.Literal("provider-session.apply-model-selection"), diff --git a/apps/server/src/orchestration-v2/EffectWorker.ts b/apps/server/src/orchestration-v2/EffectWorker.ts index c6aa09efd5e69..d557b172b3e08 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.ts @@ -189,6 +189,9 @@ export const layerExecutor: Layer.Layer< .detach({ providerSessionId: effect.request.providerSessionId, threadId: effect.threadId, + ...(effect.request.preserveBufferedOutput === undefined + ? {} + : { preserveBufferedOutput: effect.request.preserveBufferedOutput }), ...(effect.request.detail === undefined ? {} : { detail: effect.request.detail }), ...(effect.request.revokeMcpCredential === undefined ? {} diff --git a/apps/server/src/orchestration-v2/OpenCode2OrchestratorV2.integration.test.ts b/apps/server/src/orchestration-v2/OpenCode2OrchestratorV2.integration.test.ts index 8681ce4333cb4..ac9f45557cb64 100644 --- a/apps/server/src/orchestration-v2/OpenCode2OrchestratorV2.integration.test.ts +++ b/apps/server/src/orchestration-v2/OpenCode2OrchestratorV2.integration.test.ts @@ -21,6 +21,11 @@ import { OpenCode2OrchestratorReplayHarness, } from "./Adapters/OpenCode2AdapterV2.testkit.ts"; import { OPENCODE_PROVIDER } from "@t3tools/provider-opencode/testing"; +import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; +import { ORCHESTRATOR_REPLAY_FIXTURES } from "./testkit/fixtures/index.ts"; +import { materializeFixtureInput } from "./testkit/fixtures/shared.ts"; +import { materializeReplayTranscriptRuntimeInstructions } from "./testkit/ReplayRuntimeInstructions.ts"; import { provideDeterministicTestRuntime } from "./testkit/DeterministicRuntime.ts"; import type { OrchestratorV2ScenarioStep } from "./testkit/OrchestratorScenario.ts"; import { runOrchestratorV2ProviderReplayScenario } from "./testkit/ProviderReplayHarness.ts"; @@ -294,6 +299,42 @@ const runScenario = (input: { return projection; }); +const FIXTURE_REVERT = "opencode2_revert"; +/** Held until both messages were refused: the last frame after the follow-up. */ +const WAKE_HELD = "wake-held"; +/** Another OpenCode instance the thread could move to. */ +const OTHER = ProviderInstanceId.make("opencode-other"); +const labelOf = (entry: ProviderReplayEntry) => + entry.type === "runtime_exit" ? undefined : entry.label; +/** A recorded session event again, later in the stream, with its own id and sequence. */ +const again = (entry: ProviderReplayEntry, seq: number, label?: string): ProviderReplayEntry => { + if (entry.type !== "emit_inbound") return entry; + const frame = entry.frame as { readonly event: Record }; + const durable = frame.event["durable"] as Record; + return { + ...entry, + ...(label === undefined ? {} : { label }), + frame: { + ...frame, + event: { ...frame.event, id: `evt_wake_${seq}`, durable: { ...durable, seq } }, + }, + }; +}; +/** An error's message and its causes' messages, for asserting on the refusal reason. */ +const causeText = (error: unknown): string => { + const parts: Array = []; + let current: unknown = error; + for (let depth = 0; depth < 6 && current !== undefined && current !== null; depth++) { + if (typeof current === "string") { + parts.push(current); + break; + } + if (current instanceof Error) parts.push(current.message); + current = (current as { readonly cause?: unknown }).cause; + } + return parts.join(" | "); +}; + describe("OpenCode 2 through the orchestrator", () => { it.effect.each(["message", "thread settings"] as const)( "switches the session's model before the next prompt when changed from the %s", @@ -942,4 +983,201 @@ describe("OpenCode 2 through the orchestrator", () => { assert.deepEqual(shell?.pendingBackgroundTasks ?? [], []); }).pipe(Effect.scoped, Effect.provide(Layer.merge(NodeServices.layer, IdAllocator.layer))), ); + + // A rollback re-attaches a binding the session manager has no selection record + // for, and OpenCode then holds a follow-up for a continuation. Changing the + // selection or moving the thread must be refused before either is committed. + it.effect("refuses a selection change while a rollback-attached binding holds a follow-up", () => + Effect.gen(function* () { + const fixture = ORCHESTRATOR_REPLAY_FIXTURES.find( + (candidate) => candidate.name === FIXTURE_REVERT, + ); + const provider = fixture?.providers[0]; + if (fixture === undefined || provider === undefined) + return yield* Effect.die("The opencode2_revert fixture has no OpenCode 2 run."); + const recorded = yield* readProviderReplayTranscript(provider.transcriptFile).pipe( + Effect.provide(NodeServices.layer), + ); + const at = (label: string) => { + const index = recorded.entries.findIndex((entry) => labelOf(entry) === label); + assert.isAtLeast(index, 0, label); + return index; + }; + const entry = (label: string) => recorded.entries[at(label)]!; + const turns = recorded.entries.slice(0, at("session.execution.succeeded.2") + 1); + const revert = recorded.entries.slice( + at("message.list.users.1"), + at("message.list.history.response.2") + 1, + ); + const sessionId = "ses_f1484bef1ffeGfSJ1e2FXLCaus"; + const created = entry("session.create.response"); + const sessionData = + created.type === "emit_inbound" + ? (created.frame as { readonly data: { readonly data: unknown } }).data.data + : null; + const entries: ReadonlyArray = [ + ...turns, + // Settling detached the thread, and the runtime unloaded its session. + // The rollback loads it again on its own (no load the session manager + // sees) before rolling it back. + // It first asks the server whether anything still runs there. + out("session.active"), + reply("session.active", { data: {} }), + out("session.get", { sessionID: sessionId }), + reply("session.get", { data: sessionData }), + out("permission.list", { sessionID: sessionId }), + reply("permission.list", { data: [] }), + out("session.form.list", { sessionID: sessionId }), + reply("session.form.list", { data: [] }), + // It gives the session T3's rules for the thread's mode again. + out("session.update", ""), + reply("session.update", null), + ...revert, + // OpenCode then runs a follow-up on its own, with no turn of T3's. + again(entry("session.execution.started"), 40), + again(entry("session.text.ended"), 41), + again(entry("session.execution.succeeded"), 42), + again(entry("session.usage.updated"), 43, WAKE_HELD), + { type: "runtime_exit", status: "success" }, + ]; + const fixtureInput = { + ...fixture.buildInput(), + steps: fixture.buildInput().steps.filter((step) => step.type === "message"), + }; + const workspace = yield* checkpointWorkspace(FIXTURE_REVERT); + const transcript = yield* OpenCode2OrchestratorReplayHarness.decodeTranscript( + materializeReplayTranscriptRuntimeInstructions( + { ...recorded, scenario: `${FIXTURE_REVERT}-wake-admission`, entries }, + { driver: provider.driver, model: provider.modelSelection.model }, + ), + ); + const materialized = yield* materializeFixtureInput({ + scenario: FIXTURE_REVERT, + fixtureInput, + driver: provider.driver, + modelSelection: provider.modelSelection, + }).pipe(Effect.provide(IdAllocator.layer), provideDeterministicTestRuntime); + const threadId = materialized.projectionThreadIds[0]!; + const message = (key: string, modelSelection: ModelSelection): OrchestrationV2Command => ({ + type: "message.dispatch", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(`command:wake-admission:${key}`), + threadId, + messageId: MessageId.make(`message:wake-admission:${key}`), + text: `Reply with exactly: ${key}`, + attachments: [], + modelSelection, + dispatchMode: { type: "start_immediately" }, + }); + const ids = yield* IdAllocator.IdAllocatorV2; + // Settling detaches the thread from its session runtime, which forgets + // what the manager knew about the binding; the runtime stays up. + const detach: OrchestrationV2Command = { + type: "thread.settle", + commandId: CommandId.make("command:wake-admission:settle"), + threadId, + }; + const scopeId = yield* ids.allocate.checkpointScope({ threadId, name: "root" }); + const rollback: OrchestrationV2Command = { + type: "checkpoint.rollback", + restoreFiles: false, + commandId: CommandId.make("command:wake-admission:rollback"), + threadId, + scopeId, + checkpointId: yield* ids.allocate.checkpoint({ checkpointScopeId: scopeId, name: "1" }), + }; + const changed = message("changed", { + ...provider.modelSelection, + model: "opencode/mimo-v2.6-flash-free", + }); + const replaced = message("replaced", { ...provider.modelSelection, instanceId: OTHER }); + // The thread's session runtime, and another instance to move to. + const sessions: Array = []; + const harness: typeof OpenCode2OrchestratorReplayHarness = { + ...OpenCode2OrchestratorReplayHarness, + makeProviderAdapterRegistryLayer: (decoded, options) => + Layer.effect( + ProviderAdapterRegistry.ProviderAdapterRegistryV2, + Effect.gen(function* () { + const registry = yield* ProviderAdapterRegistry.ProviderAdapterRegistryV2; + const recorder = ( + adapter: ProviderAdapter.ProviderAdapterV2["Service"], + ): ProviderAdapter.ProviderAdapterV2["Service"] => ({ + ...adapter, + openSession: (input) => + adapter + .openSession(input) + .pipe(Effect.tap((session) => Effect.sync(() => void sessions.push(session)))), + }); + return ProviderAdapterRegistry.ProviderAdapterRegistryV2.of({ + ...registry, + get: (id) => + id === OTHER + ? registry.get(provider.modelSelection.instanceId).pipe( + Effect.map((adapter) => ({ + ...adapter, + instanceId: OTHER, + openSession: () => Effect.die("The other instance never opens."), + })), + ) + : registry.get(id).pipe(Effect.map(recorder)), + list: () => registry.list().pipe(Effect.map((instances) => [...instances, OTHER])), + }); + }), + ).pipe( + Layer.provide( + OpenCode2OrchestratorReplayHarness.makeProviderAdapterRegistryLayer(decoded, options), + ), + ), + }; + const result = yield* runOrchestratorV2ProviderReplayScenario( + { + name: `${FIXTURE_REVERT}/wake-admission`, + transcript, + commands: [...materialized.commands, detach, rollback, changed, replaced], + steps: [ + ...materialized.steps, + { type: "dispatch", command: detach }, + { type: "advance_clock", duration: "1 millis" }, + // The rollback attaches it again and OpenCode loads nothing new. + { type: "dispatch", command: rollback }, + { type: "advance_clock", duration: "1 millis" }, + { + type: "await_run_status", + threadId, + runId: ids.derive.run({ threadId, ordinal: 2 }), + status: "rolled_back", + }, + { type: "await_replay_gate", label: WAKE_HELD }, + { type: "dispatch_refused", command: changed, key: "changed" }, + { type: "dispatch_refused", command: replaced, key: "replaced" }, + { type: "release_replay_gate", label: WAKE_HELD }, + ], + projectionThreadIds: materialized.projectionThreadIds, + runtimePolicyOverride: { ...provider.runtimePolicyOverride, cwd: workspace }, + }, + harness, + ).pipe(provideDeterministicTestRuntime); + const projection = result.projections.get(threadId); + assert.isDefined(projection); + // Neither message made a run or changed the thread's selection. + assert.deepEqual( + projection.runs.map((run) => run.status), + ["completed", "rolled_back"], + ); + assert.deepEqual(projection.thread.modelSelection, provider.modelSelection); + for (const key of ["changed", "replaced"]) { + assert.include( + causeText(result.refusedDispatches.get(key)), + ProviderAdapter.PROVIDER_BUFFERED_OUTPUT_MESSAGE, + key, + ); + } + // The follow-up is still held for its continuation. + assert.lengthOf(sessions, 1); + const binding = projection.providerThreads[0]!; + assert.isTrue(yield* sessions[0]!.hasBufferedOutputForThread!(binding)); + }).pipe(Effect.scoped, Effect.provide(Layer.merge(NodeServices.layer, IdAllocator.layer))), + ); }); diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 5257330e2cb5d..766df933729f3 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -184,7 +184,7 @@ import * as ProviderContinuationRequests from "@t3tools/provider-core/server/Pro import { makeProviderFailure } from "@t3tools/provider-core/server/failure"; import * as RunExecutionService from "./RunExecutionService.ts"; import { ProviderSessionManagerV2 } from "./ProviderSessionManager.ts"; -import { ProviderSwitchServiceV2 } from "./ProviderSwitchService.ts"; +import { ProviderSwitchPlanError, ProviderSwitchServiceV2 } from "./ProviderSwitchService.ts"; import { isAutomaticCompletionRun, queuedRunsInDeliveryOrder } from "./QueuedRunOrder.ts"; import { isRuntimePolicyAboveCeilingError, RuntimePolicyV2 } from "./RuntimePolicy.ts"; import { @@ -201,7 +201,7 @@ import { ThreadForkServiceV2, } from "./ThreadForkService.ts"; import { planThreadDeletion } from "./ThreadDeletion.ts"; -import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import * as ServerSettings from "../serverSettings.ts"; const isMessageQuestionAnswer = Schema.is( @@ -1661,6 +1661,138 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ), ); + const planExecutionSelection = Effect.fnUntraced(function* (input: { + readonly projection: Pick< + OrchestrationV2ThreadProjection, + "thread" | "providerThreads" | "providerSessions" + >; + readonly targetModelSelection: ModelSelection; + readonly buffered?: boolean; + readonly admit?: boolean; + /** A wake that delivers the binding's output is already queued, so none is offered. */ + readonly drainQueued?: boolean; + }) { + const { projection, targetModelSelection } = input; + const binding = projection.providerThreads.find( + (thread) => thread.id === projection.thread.activeProviderThreadId, + ); + const sessionId = binding?.providerSessionId; + if (binding === undefined || sessionId === null || sessionId === undefined) { + return yield* providerSwitchService.plan({ + ...input, + executionSelection: null, + freshAttachment: true, + }); + } + const resident = yield* providerSessions.get(sessionId); + const current = + Option.isSome(resident) && resident.value.executionSelection !== undefined + ? yield* resident.value.executionSelection(binding) + : null; + const pending = Option.isSome(resident) + ? yield* ( + resident.value.hasPendingBackgroundWorkForThread?.(binding, { includeSubagents: true }) ?? + resident.value.hasPendingBackgroundWork ?? + Effect.succeed(false) + ) + : false; + // A provider wake only delivers output its binding already holds, so it never replaces + // the binding; refusing it would make the adapter drop that output. + if ( + input.buffered === true || + (input.admit === true && + Option.isSome(resident) && + (yield* ( + resident.value.canAdoptUnselectedBufferedOutput?.(binding, targetModelSelection) ?? + Effect.succeed(false) + ))) + ) { + return { + instanceChanged: false, + releaseProviderSessionIds: [], + transition: { type: "reuse" as const }, + }; + } + const attached = Option.isSome(resident) + ? yield* resident.value.hasResidentBinding?.(binding) ?? Effect.succeed(current !== null) + : false; + const projectedLive = projection.providerSessions.some( + (session) => + session.id === sessionId && session.status !== "stopped" && session.status !== "error", + ); + // Without a resident runtime there is no execution evidence to plan from. + if (Option.isNone(resident) && projectedLive) { + return yield* providerSwitchService.plan({ projection, targetModelSelection }); + } + const plan = yield* providerSwitchService + .plan({ + ...input, + executionSelection: attached && !projectedLive ? null : current, + freshAttachment: + !attached || + (Option.isSome(resident) && + (yield* ( + resident.value.reappliesFullSelection?.(targetModelSelection) ?? Effect.succeed(false) + ))), + }) + .pipe( + Effect.map((plan) => + attached && plan.transition.type === "restart_and_resume" + ? { ...plan, releaseProviderSessionIds: [sessionId] } + : plan, + ), + ); + const replacing = + plan.transition.type === "restart_and_resume" || + plan.transition.type === "create_with_handoff"; + // Native output and work are the resident runtime's, whether or not the + // manager has seen this binding load or run: a binding opened by a + // rollback can own a wake before any turn of T3's. + if (input.admit === true && Option.isSome(resident)) { + const buffered = yield* ( + resident.value.hasBufferedOutputForThread?.(binding) ?? Effect.succeed(false) + ); + const producingSelection = buffered + ? yield* resident.value.bufferedExecutionSelection?.(binding) ?? Effect.succeed(current) + : null; + if ( + buffered && + (current === null || + producingSelection === null || + replacing || + !modelSelectionsEqual(current, targetModelSelection) || + !modelSelectionsEqual(producingSelection, targetModelSelection)) + ) { + const runtime = resident.value; + if (input.drainQueued !== true) { + yield* continuationRequests.offer({ + threadId: projection.thread.id, + providerThreadId: binding.id, + driver: binding.driver, + detail: null, + dispatchIfCurrent: Effect.fn("Orchestrator.drainBufferedOutput")(function* (effect) { + if (!(yield* runtime.hasBufferedOutputForThread!(binding))) return Option.none(); + return Option.some(yield* effect); + }), + }); + } + return yield* new ProviderSwitchPlanError({ + threadId: projection.thread.id, + targetProviderInstanceId: targetModelSelection.instanceId, + cause: ProviderAdapter.PROVIDER_BUFFERED_OUTPUT_MESSAGE, + }); + } + if (pending && replacing) { + return yield* new ProviderSwitchPlanError({ + threadId: projection.thread.id, + targetProviderInstanceId: targetModelSelection.instanceId, + cause: ProviderAdapter.PROVIDER_RUNNING_WORK_MESSAGE, + }); + } + } + return plan; + }); + const providerSessionIdFor = (input: { readonly adapter: ProviderAdapter.ProviderAdapterV2["Service"]; readonly providerInstanceId: ProviderInstanceId; @@ -2398,7 +2530,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio return true; }); - const startNextQueuedRun = Effect.fn("Orchestrator.startNextQueuedRun")( + const startQueuedRun = Effect.fn("Orchestrator.startNextQueuedRun")( function* ( threadId: ThreadId, options?: { @@ -2631,7 +2763,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio storedProviderThread.nativeThreadRef !== null) ? undefined : storedProviderThread; - const queuedProviderThread: OrchestrationV2ProviderThread = targetProviderThread ?? { + let queuedProviderThread: OrchestrationV2ProviderThread = targetProviderThread ?? { id: idAllocator.derive.providerThread({ driver: targetAdapter.driver, // The queue reserved `pending:` for this run; another run may have taken it. @@ -2658,44 +2790,78 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio projection.thread.modelSelection, queuedRun.modelSelection, ); - // The picker may already have moved ahead of the native conversation. - const switchFromActiveProvider = - followsCurrent && - activeProviderThread !== undefined && - activeProviderThread.providerInstanceId !== queuedRun.providerInstanceId; - const switchPlan = - selectionChanged || switchFromActiveProvider - ? yield* providerSwitchService - .plan({ - projection: switchFromActiveProvider - ? { - ...projection, - thread: { - ...projection.thread, - modelSelection: { - ...projection.thread.modelSelection, - instanceId: activeProviderThread.providerInstanceId, - }, - }, - } - : projection, - targetModelSelection: queuedRun.modelSelection, - }) - .pipe( - Effect.mapError( - (cause) => - new OrchestratorDispatchError({ - commandId, - commandType: "message.dispatch", - cause, - }), - ), - ) - : null; + const queuedSelectionInput = { + projection: freshContext + ? { ...projection, thread: { ...projection.thread, activeProviderThreadId: null } } + : projection, + targetModelSelection: queuedRun.modelSelection, + buffered: + queuedMessage.createdBy === "agent" && queuedMessage.creationSource === "provider", + }; + const queuedDrain = queuedSelectionInput.buffered + ? undefined + : queuedRunsInDeliveryOrder(projection).find((run) => { + const message = projection.messages.find( + (candidate) => candidate.id === run.userMessageId, + ); + return ( + run.queueHeld !== true && + message !== undefined && + isBufferedProviderNotification(message) + ); + }); + // Undelivered output refuses this start. A wake already queued delivers it, so that wake + // goes first and this run keeps its place; with none queued the start fails, because + // waiting for the drain offered here could block the queue. Running work holds below. + const switchPlan = yield* planExecutionSelection({ + ...queuedSelectionInput, + admit: true, + drainQueued: queuedDrain !== undefined, + }).pipe( + Effect.catchTags({ + ProviderSwitchPlanError: (error) => { + if (error.cause === ProviderAdapter.PROVIDER_RUNNING_WORK_MESSAGE) { + return planExecutionSelection(queuedSelectionInput); + } + return queuedDrain !== undefined && + error.cause === ProviderAdapter.PROVIDER_BUFFERED_OUTPUT_MESSAGE + ? Effect.undefined + : Effect.fail(error); + }, + }), + Effect.mapError( + (cause) => + new OrchestratorDispatchError({ + commandId, + commandType: "message.dispatch", + cause, + }), + ), + ); + if (switchPlan === undefined) return queuedDrain?.id; + const queuedHandoff = + !freshContext && + switchPlan.transition.type === "create_with_handoff" && + !switchPlan.instanceChanged && + queuedProviderThread.nativeThreadRef !== null; + if (queuedHandoff) { + queuedProviderThread = { + ...queuedProviderThread, + id: idAllocator.derive.providerThread({ + driver: queuedProviderThread.driver, + nativeThreadId: `handoff:${queuedRun.id}`, + }), + providerSessionId: null, + nativeThreadRef: null, + nativeConversationHeadRef: null, + nativeMetadata: undefined, + }; + } const releasableSessions = uniqueSessions([ ...projection.providerSessions.filter( (session) => - switchPlan?.releaseProviderSessionIds.includes(session.id) && + (switchPlan.releaseProviderSessionIds.includes(session.id) || + (queuedHandoff && session.id === activeProviderThread?.providerSessionId)) && session.status !== "stopped" && session.status !== "error", ), @@ -2770,7 +2936,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio freshContext || canResumeAcrossInstances || latestHandoffRun === undefined || - latestHandoffRun.providerInstanceId === queuedRun.providerInstanceId + (latestHandoffRun.providerInstanceId === queuedRun.providerInstanceId && !queuedHandoff) ? [] : projection.runs.filter( (run) => @@ -2913,16 +3079,33 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const providerSessionId = (!canResumeAcrossInstances && queuedProviderThread.providerSessionId !== null && + Option.isSome( + yield* providerSessions.get(queuedProviderThread.providerSessionId).pipe( + Effect.mapError( + (cause) => + new OrchestratorDispatchError({ + commandId, + commandType: "message.dispatch", + cause, + }), + ), + ), + ) && !switchPlan?.releaseProviderSessionIds.includes(queuedProviderThread.providerSessionId) ? queuedProviderThread.providerSessionId : null) ?? (yield* providerAdapters.get(queuedRun.providerInstanceId).pipe( Effect.flatMap((adapter) => - providerSessionIdFor({ - adapter, - providerInstanceId: queuedRun.providerInstanceId, - threadId, - }), + switchPlan.transition.type === "restart_and_resume" || queuedHandoff + ? idAllocator.allocate.providerSession({ + providerInstanceId: queuedRun.providerInstanceId, + threadId, + }) + : providerSessionIdFor({ + adapter, + providerInstanceId: queuedRun.providerInstanceId, + threadId, + }), ), Effect.mapError( (cause) => @@ -3111,7 +3294,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio : "Provider or model selection changed."; yield* writeSystemEvents( [ - ...(selectionChanged + ...(selectionChanged && + queuedMessage.notification === undefined && + queuedMessage.delegatedCompletion === undefined ? [ { type: @@ -3251,6 +3436,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio request: { type: "provider-session.detach" as const, providerSessionId: session.id, + preserveBufferedOutput: true, detail: detachReason, }, })), @@ -3266,10 +3452,22 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }, (effect, threadId, options?) => effect.pipe( - Effect.catch((cause) => failQueuedRunStart(threadId, cause, options?.queuedRunId)), + Effect.catch((cause) => + failQueuedRunStart(threadId, cause, options?.queuedRunId).pipe(Effect.as(undefined)), + ), ), ); + // Untraced: each start it makes carries the span. + const startNextQueuedRun = Effect.fnUntraced(function* ( + ...[threadId, options]: Parameters + ) { + const drainFirst = yield* startQueuedRun(threadId, options); + if (drainFirst !== undefined) { + yield* startQueuedRun(threadId, { ...options, queuedRunId: drainFirst }); + } + }); + // Under load a resumed provider can time out before taking the prompt; the restart still owes the turn. const retryContinuationStart = Effect.fn("Orchestrator.retryContinuationStart")(function* ( threadId: ThreadId, @@ -4475,12 +4673,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }), ), ); - const plan = yield* providerSwitchService - .plan({ - projection: providerContext!, - targetModelSelection: command.modelSelection, - }) - .pipe(mapDispatchError(command)); + const plan = yield* planExecutionSelection({ + projection: providerContext!, + targetModelSelection: command.modelSelection, + }).pipe(mapDispatchError(command)); if (plan.instanceChanged || plan.releaseProviderSessionIds.length > 0) { const projection = yield* loadProjectionForCommand(command, ["runs"], { turnItemTypes: [], @@ -5482,6 +5678,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio : command.type === "thread.runtime-mode.set" ? "Runtime mode changed." : "Provider or model selection changed.", + ...(command.type === "thread.model-selection.set" || + command.type === "provider.switch" + ? { preserveBufferedOutput: true } + : {}), // Terminal detaches revoke MCP credentials; reattachments need other detach reasons to preserve them. ...(command.type === "thread.archive" ? { revokeMcpCredential: true } : {}), }, @@ -7082,6 +7282,28 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio projection = applyToProjection(projection, event); } let modelSelection = command.modelSelection ?? projection.thread.modelSelection; + // A provider wake delivers output an earlier turn produced, so its run keeps that turn's + // selection even when the thread's selection has moved on since. + if ( + command.createdBy === "agent" && + command.creationSource === "provider" && + command.notification !== undefined + ) { + const owner = projection.providerThreads.find( + (thread) => thread.id === projection.thread.activeProviderThreadId, + ); + const resident = + owner?.providerSessionId == null + ? Option.none() + : yield* providerSessions.get(owner.providerSessionId).pipe(mapDispatchError(command)); + if (owner !== undefined && Option.isSome(resident)) { + const runtime = resident.value; + modelSelection = + (yield* runtime.bufferedExecutionSelection?.(owner) ?? Effect.succeed(null)) ?? + (yield* runtime.executionSelection?.(owner) ?? Effect.succeed(null)) ?? + modelSelection; + } + } let dispatchMode = resolveMessageDispatchIntent( projection, command.dispatchMode, @@ -7870,8 +8092,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ); const now = yield* DateTime.now; if ( - !modelSelectionsEqual(projection.thread.modelSelection, modelSelection) || - projection.thread.providerInstanceId !== modelSelection.instanceId + command.notification === undefined && + command.delegatedCompletion === undefined && + (!modelSelectionsEqual(projection.thread.modelSelection, modelSelection) || + projection.thread.providerInstanceId !== modelSelection.instanceId) ) { yield* emit( events, @@ -7902,6 +8126,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio cause: "A fresh session can't start while a fork or merge is still being applied.", }); } + const admission = yield* planExecutionSelection({ + projection: freshContext + ? { ...projection, thread: { ...projection.thread, activeProviderThreadId: null } } + : projection, + targetModelSelection: modelSelection, + admit: true, + buffered: command.createdBy === "agent" && command.creationSource === "provider", + }).pipe(mapDispatchError(command)); const ordinal = nextRunOrdinal(projection); const runId = idAllocator.derive.run({ threadId: command.threadId, ordinal }); const contextBoundary = freshContext ? ordinal : contextBoundaryOrdinal(projection); @@ -7911,9 +8143,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio projection.thread.historyOrigin === "v1_import" && contextBoundary === 0 ? yield* readHandoffItems(command.threadId, [null]) : []; + const selectionHandoff = + activeProviderThread !== undefined && + activeProviderThread.providerInstanceId === modelSelection.instanceId && + admission.transition.type === "create_with_handoff"; const isProviderSwitch = activeProviderThread !== undefined && - activeProviderThread.providerInstanceId !== modelSelection.instanceId; + (activeProviderThread.providerInstanceId !== modelSelection.instanceId || selectionHandoff); // Account overlays share native history. Selection commands may already // have updated the app thread, so classify against the native thread's owner. const providerSwitchPlan = isProviderSwitch @@ -7941,7 +8177,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const releasableSessions = uniqueSessions([ ...projection.providerSessions.filter( (session) => - providerSwitchPlan?.releaseProviderSessionIds.includes(session.id) && + (providerSwitchPlan?.releaseProviderSessionIds.includes(session.id) || + (selectionHandoff && session.id === activeProviderThread?.providerSessionId)) && session.status !== "stopped" && session.status !== "error", ), @@ -7966,7 +8203,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio if ( pendingForkTransfer === undefined && pendingMergeBackTransfer === undefined && - (!isProviderSwitch || freshContext) + (freshContext || + (!isProviderSwitch && + (activeProviderThread === undefined || + admission.transition.type !== "create_with_handoff"))) ) { const reusableProviderThread = freshContext ? undefined : activeProviderThread; const adapter = yield* providerAdapters.get(modelSelection.instanceId).pipe( @@ -7979,15 +8219,60 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }), ), ); - const providerSessionId = - reusableProviderThread?.providerSessionId ?? - (yield* mapDispatchError(command)( - providerSessionIdFor({ - adapter, - providerInstanceId: modelSelection.instanceId, + const replacesBinding = !freshContext && admission.transition.type === "restart_and_resume"; + const providerSessionId = replacesBinding + ? yield* idAllocator.allocate + .providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId: command.threadId, + }) + .pipe(mapDispatchError(command)) + : reusableProviderThread?.providerSessionId != null && + Option.isSome( + yield* providerSessions + .get(reusableProviderThread.providerSessionId) + .pipe(mapDispatchError(command)), + ) + ? reusableProviderThread.providerSessionId + : yield* mapDispatchError(command)( + providerSessionIdFor({ + adapter, + providerInstanceId: modelSelection.instanceId, + threadId: command.threadId, + }), + ); + if (replacesBinding && activeProviderThread?.providerSessionId != null) { + const oldSessionId = activeProviderThread.providerSessionId; + yield* emit( + events, + command, + )({ + type: "provider-session.detached", + threadId: command.threadId, + driver: activeProviderThread.driver, + providerInstanceId: activeProviderThread.providerInstanceId, + occurredAt: now, + payload: { + providerSessionId: oldSessionId, + detachedAt: now, + reason: "Execution selection requires a fresh binding.", + }, + }); + yield* Ref.update(effects, (existing) => [ + ...existing, + { + id: `effect:${command.commandId}:provider-session.detach:${oldSessionId}`, + commandId: command.commandId, threadId: command.threadId, - }), - )); + request: { + type: "provider-session.detach", + providerSessionId: oldSessionId, + preserveBufferedOutput: true, + detail: "Execution selection requires a fresh binding.", + }, + } satisfies PendingOrchestrationEffectV2, + ]); + } const providerThreadId = reusableProviderThread?.id ?? idAllocator.derive.providerThread({ @@ -8412,11 +8697,20 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }), ), ); - const targetProviderThread = - isProviderSwitch && !canResumeAcrossInstances + const targetProviderThread = selectionHandoff + ? undefined + : isProviderSwitch && !canResumeAcrossInstances ? rootProviderThreadsForProvider(projection, modelSelection.instanceId)[0] : activeProviderThread; const providerSessionId = + (selectionHandoff || canResumeAcrossInstances + ? yield* idAllocator.allocate + .providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId: command.threadId, + }) + .pipe(mapDispatchError(command)) + : undefined) ?? (canResumeAcrossInstances ? undefined : targetProviderThread?.providerSessionId) ?? (yield* mapDispatchError(command)( providerSessionIdFor({ @@ -9357,6 +9651,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio request: { type: "provider-session.detach", providerSessionId: session.id, + preserveBufferedOutput: true, detail: detachDetail, }, } satisfies PendingOrchestrationEffectV2, diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 2e9adb9c8a54c..a141317e1c5d2 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -1,5 +1,6 @@ import { AgentScope } from "@t3tools/shared/AgentScope"; import * as KeyedLock from "@t3tools/shared/KeyedLock"; +import { modelSelectionsEqual } from "@t3tools/shared/model"; import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; import { isOrchestrationV2WorkActive, @@ -103,6 +104,13 @@ const WORKING_RUN_STATUSES: ReadonlySet = new Set([ "running", "waiting", ]); +const isProviderAdapterBufferedOutputError = Schema.is( + ProviderAdapter.ProviderAdapterBufferedOutputError, +); +const isProviderAdapterRunningWorkError = Schema.is( + ProviderAdapter.ProviderAdapterRunningWorkError, +); +const isProviderAdapterTurnStartError = Schema.is(ProviderAdapter.ProviderAdapterTurnStartError); /** Matches planDelegatedCompletionDelivery: true while a delegated task's result still wakes the thread. */ export const delegatedTaskWillWake = (records: { @@ -256,6 +264,7 @@ export interface ProviderSessionManagerV2Shape { readonly threadId: ThreadId; readonly detail?: string; readonly revokeMcpCredential?: boolean; + readonly preserveBufferedOutput?: boolean; }) => Effect.Effect; /** * Releases the session unless, at the moment of removal, a turn is busy on it other than one @@ -290,7 +299,27 @@ export class ProviderSessionManagerV2 extends Context.Service< ProviderSessionManagerV2Shape >()("t3/orchestration-v2/ProviderSessionManager/ProviderSessionManagerV2") {} +interface ExecutionSelectionEvidence { + readonly providerThread?: OrchestrationV2ProviderThread; + readonly providerThreadId: string; + readonly nativeThreadId: string | null; + readonly selection: ModelSelection | null; + readonly bufferedRunOrdinal?: number; + readonly operation?: symbol | undefined; + readonly runAttemptId?: string; + readonly providerTurnId?: string; + readonly loadKey?: string; +} + interface LiveSessionEntry { + readonly executionSelections: ReadonlyMap; + readonly preservedOutputThreadIds: ReadonlyMap< + ThreadId, + { + readonly binding: OrchestrationV2ProviderThread; + readonly operation: symbol | undefined; + } + >; readonly attachedThreadIds: ReadonlySet; readonly loadedProviderThreadKeyByThread: ReadonlyMap; /** null marks an unknown catalog; undefined means MCP refresh is not configured. */ @@ -1459,6 +1488,49 @@ export const layerWithOptions = ( ), ); + const finishDeferredDetaches = Effect.fnUntraced(function* (entry: LiveSessionEntry) { + const ownedEntry = (yield* Ref.get(sessions)).get( + sessionKey(entry.runtime.providerSessionId), + ); + if (ownedEntry?.runtime !== entry.runtime) return; + for (const [threadId, retained] of ownedEntry.preservedOutputThreadIds) { + const buffered = yield* ( + entry.runtime.hasBufferedOutputForThread?.(retained.binding) ?? Effect.succeed(false) + ); + if (buffered) continue; + yield* Ref.update(sessions, (current) => { + const key = sessionKey(entry.runtime.providerSessionId); + const latest = current.get(key); + if ( + latest?.runtime !== entry.runtime || + latest.preservedOutputThreadIds.get(threadId) !== retained + ) + return current; + const preservedOutputThreadIds = new Map(latest.preservedOutputThreadIds); + preservedOutputThreadIds.delete(threadId); + const executionSelections = new Map(latest.executionSelections); + const evidence = executionSelections.get(threadId); + const attachedThreadIds = new Set(latest.attachedThreadIds); + attachedThreadIds.delete(threadId); + const loadedProviderThreadKeyByThread = new Map(latest.loadedProviderThreadKeyByThread); + loadedProviderThreadKeyByThread.delete(threadId); + if (evidence !== undefined) + executionSelections.set(threadId, { + ...evidence, + selection: null, + operation: undefined, + }); + return new Map(current).set(key, { + ...latest, + executionSelections, + preservedOutputThreadIds, + attachedThreadIds, + loadedProviderThreadKeyByThread, + }); + }); + } + }); + const releaseIfStillIdle = Effect.fn("ProviderSessionManager.releaseIfStillIdle")( function* (input: { readonly providerSessionId: ProviderSessionId; @@ -1474,6 +1546,14 @@ export const layerWithOptions = ( ) { return; } + yield* finishDeferredDetaches(entry).pipe( + Effect.catchCause((cause) => + Effect.logWarning("orchestration-v2.driver-session.deferred-detach-failed", { + providerSessionId: entry.runtime.providerSessionId, + cause, + }), + ), + ); // Capture runtime identity before yielding; replacements can reuse the session id. const probedRuntime = entry.runtime; const hasPendingWork = @@ -1490,10 +1570,20 @@ export const layerWithOptions = ( ), ), ); - if (hasPendingWork) { + const afterProbe = (yield* Ref.get(sessions)).get(key); + if ( + afterProbe?.runtime !== probedRuntime || + afterProbe.busyTurns.size > 0 || + afterProbe.idleGeneration !== input.generation + ) + return; + if (hasPendingWork || afterProbe.preservedOutputThreadIds.size > 0) { const now = yield* Clock.currentTimeMillis; - const pinnedSinceMs = entry.pinnedSinceMs ?? now; - if (now - pinnedSinceMs < maxIdlePinMs) { + const pinnedSinceMs = afterProbe.pinnedSinceMs ?? now; + if ( + now - pinnedSinceMs < maxIdlePinMs || + afterProbe.preservedOutputThreadIds.size > 0 + ) { const shouldContinuePin = yield* Ref.modify(sessions, (latest) => { const latestEntry = latest.get(key); if ( @@ -2299,7 +2389,200 @@ export const layerWithOptions = ( >, ): ProviderAdapter.ProviderAdapterV2SessionRuntime => { const providerSessionId = runtime.providerSessionId; + const selectionEvidence = Effect.fnUntraced(function* (threadId: ThreadId) { + const entry = (yield* Ref.get(sessions)).get(sessionKey(providerSessionId)); + return entry?.runtime === runtime ? entry.executionSelections.get(threadId) : undefined; + }); + const setSelectionEvidence = ( + threadId: ThreadId, + evidence: ExecutionSelectionEvidence, + expectedOperation?: symbol, + ) => + Ref.update(sessions, (current) => { + const key = sessionKey(providerSessionId); + const entry = current.get(key); + if ( + entry?.runtime !== runtime || + (expectedOperation !== undefined && + entry.executionSelections.get(threadId)?.operation !== expectedOperation) + ) + return current; + return new Map(current).set(key, { + ...entry, + executionSelections: new Map(entry.executionSelections).set(threadId, evidence), + }); + }); const subscribeEvents = makeEventSubscription(eventSubscribers); + const observeSelection = Effect.fnUntraced(function* ( + threadId: ThreadId, + operation: Effect.Effect, + evidence: ExecutionSelectionEvidence, + acceptedSelection: ModelSelection | null, + drainOnly = false, + ) { + const token = Symbol(); + const previous = yield* selectionEvidence(threadId); + yield* setSelectionEvidence(threadId, { ...evidence, operation: token }); + const entry = (yield* Ref.get(sessions)).get(sessionKey(providerSessionId)); + const retained = entry?.preservedOutputThreadIds.get(threadId); + if (entry?.runtime === runtime && retained !== undefined) { + const preservedOutputThreadIds = new Map(entry.preservedOutputThreadIds); + if ( + evidence.bufferedRunOrdinal !== undefined && + retained.binding.id === evidence.providerThreadId && + retained.binding.nativeThreadRef?.nativeId === evidence.nativeThreadId + ) { + preservedOutputThreadIds.set(threadId, { ...retained, operation: token }); + } else { + preservedOutputThreadIds.delete(threadId); + } + yield* Ref.update(sessions, (current) => + current.get(sessionKey(providerSessionId)) === entry + ? new Map(current).set(sessionKey(providerSessionId), { + ...entry, + preservedOutputThreadIds, + }) + : current, + ); + } + const result = yield* Effect.exit(operation); + const current = yield* selectionEvidence(threadId); + if (current?.operation === token) { + let selection = Exit.isSuccess(result) ? acceptedSelection : null; + if (drainOnly) { + selection = + runtime.liveExecutionSelection !== undefined && + evidence.providerThread !== undefined + ? yield* runtime.liveExecutionSelection(evidence.providerThread) + : (previous?.selection ?? null); + } else if (selection !== null && runtime.selectionAcknowledged !== undefined) { + if (!(yield* runtime.selectionAcknowledged(selection))) selection = null; + } + if (!drainOnly && Exit.isFailure(result) && !Cause.hasInterrupts(result.cause)) { + const failure = Cause.findErrorOption(result.cause); + const error = + Option.isSome(failure) && isProviderAdapterTurnStartError(failure.value) + ? failure.value.cause + : Option.getOrUndefined(failure); + const refused = + isProviderAdapterRunningWorkError(error) || + isProviderAdapterBufferedOutputError(error); + const sameSelection = + runtime.preservesSelectionOnSameSelectionFailure === true && + previous?.selection != null && + acceptedSelection !== null && + modelSelectionsEqual(previous.selection, acceptedSelection); + if (refused || sameSelection) selection = previous?.selection ?? null; + if (selection === null && evidence.providerThread !== undefined) + selection = yield* ( + runtime.liveExecutionSelection?.(evidence.providerThread) ?? Effect.succeed(null) + ); + } + if ((yield* selectionEvidence(threadId))?.operation === token) { + yield* setSelectionEvidence( + threadId, + { + ...current, + ...(Exit.isFailure(result) && + selection !== null && + previous?.selection != null && + modelSelectionsEqual(selection, previous.selection) + ? previous + : {}), + operation: token, + selection, + }, + token, + ); + } + } + return yield* result; + }); + const protectBufferedOutput = Effect.fnUntraced(function* ( + thread: OrchestrationV2ProviderThread | undefined, + selection?: ModelSelection, + policy?: ProviderAdapter.ProviderAdapterV2RuntimePolicy, + starting = false, + ) { + if (thread === undefined || thread.appThreadId === null) return; + const buffered = yield* ( + runtime.hasBufferedOutputForThread?.(thread) ?? Effect.succeed(false) + ); + if (!buffered) return; + const evidence = yield* selectionEvidence(thread.appThreadId); + const bufferedSelection = yield* ( + runtime.bufferedExecutionSelection?.(thread) ?? Effect.succeed(null) + ); + if ( + starting && + selection !== undefined && + (yield* ( + runtime.canAdoptUnselectedBufferedOutput?.(thread, selection) ?? Effect.succeed(false) + )) + ) + return; + if ( + starting && + bufferedSelection !== null && + selection !== undefined && + modelSelectionsEqual(bufferedSelection, selection) + ) + return; + if ( + starting && + evidence?.selection !== null && + evidence?.loadKey === + providerThreadLoadKey({ + runtime, + providerThread: thread, + ...(selection === undefined ? {} : { modelSelection: selection }), + ...(policy === undefined ? {} : { runtimePolicy: policy }), + }) + ) + return; + return yield* new ProviderAdapter.ProviderAdapterBufferedOutputError({ + driver: runtime.driver, + providerThreadId: thread.id, + }); + }); + const rememberBinding = Effect.fnUntraced(function* ( + thread: OrchestrationV2ProviderThread, + ) { + if (thread.appThreadId === null) return; + const previous = yield* selectionEvidence(thread.appThreadId); + if ( + previous?.providerThreadId !== thread.id || + previous.nativeThreadId !== (thread.nativeThreadRef?.nativeId ?? null) + ) { + yield* setSelectionEvidence(thread.appThreadId, { + providerThread: thread, + providerThreadId: thread.id, + nativeThreadId: thread.nativeThreadRef?.nativeId ?? null, + selection: null, + }); + } + }); + const invalidateBinding = (threadId: ThreadId) => + Ref.update(sessions, (current) => { + const key = sessionKey(providerSessionId); + const entry = current.get(key); + if (entry?.runtime !== runtime) return current; + const executionSelections = new Map(entry.executionSelections); + const preservedOutputThreadIds = new Map(entry.preservedOutputThreadIds); + preservedOutputThreadIds.delete(threadId); + const previous = executionSelections.get(threadId); + if (previous !== undefined) + executionSelections.set(threadId, { + ...previous, + selection: null, + operation: undefined, + }); + return new Map(current).set(key, { + ...entry, + executionSelections, + preservedOutputThreadIds, + }); + }); const turnMetrics = (operation: string, model?: string) => withMetrics({ counter: providerTurnsTotal, @@ -2316,6 +2599,42 @@ export const layerWithOptions = ( get providerSession() { return runtime.providerSession; }, + // Only a binding this runtime loaded, resumed or ran has a selection + // that can be uncertain. A thread merely attached when the session + // opened (as a rollback opens one) is still a fresh attachment: its + // next turn applies the selection like any first turn. + hasResidentBinding: Effect.fn("ProviderSessionManager.hasResidentBinding")( + function* (thread) { + const entry = (yield* Ref.get(sessions)).get(sessionKey(providerSessionId)); + return ( + entry?.runtime === runtime && + thread.appThreadId !== null && + thread.providerSessionId === providerSessionId && + entry.executionSelections.has(thread.appThreadId) + ); + }, + ), + executionSelection: Effect.fn("ProviderSessionManager.executionSelection")( + function* (thread) { + const entry = (yield* Ref.get(sessions)).get(sessionKey(providerSessionId)); + if ( + entry?.runtime !== runtime || + thread.appThreadId === null || + thread.providerSessionId !== providerSessionId || + thread.providerInstanceId !== runtime.instanceId || + thread.driver !== runtime.driver || + (thread.nativeThreadRef !== null && + thread.nativeThreadRef.driver !== runtime.driver) || + !entry.attachedThreadIds.has(thread.appThreadId) + ) + return null; + const evidence = yield* selectionEvidence(thread.appThreadId); + return evidence?.providerThreadId === thread.id && + evidence.nativeThreadId === (thread.nativeThreadRef?.nativeId ?? null) + ? evidence.selection + : null; + }, + ), subscribeEvents, events: Stream.unwrap( subscribeEvents.pipe(Effect.map((subscription) => subscription.events)), @@ -2329,7 +2648,16 @@ export const layerWithOptions = ( providerInstanceId: runtime.instanceId, }), ).pipe( + Effect.andThen( + protectBufferedOutput( + input.existingProviderThread, + input.modelSelection, + input.runtimePolicy, + ), + ), + Effect.andThen(invalidateBinding(input.threadId)), Effect.andThen(loadNatively(runtime.ensureThread(input))), + Effect.tap(({ loaded }) => rememberBinding(loaded)), Effect.tap(({ loaded: providerThread, linkedEnvironments }) => markProviderThreadLoaded({ providerSessionId, @@ -2383,14 +2711,40 @@ export const layerWithOptions = ( Effect.andThen( isProviderThreadLoaded({ providerSessionId, threadId, providerThreadKey }), ), + Effect.flatMap((loaded) => + Effect.gen(function* () { + const buffered = yield* ( + runtime.bufferedExecutionSelection?.(input.providerThread) ?? + Effect.succeed(null) + ); + return ( + loaded || + (input.modelSelection !== undefined && + (yield* ( + runtime.canAdoptUnselectedBufferedOutput?.( + input.providerThread, + input.modelSelection, + ) ?? Effect.succeed(false) + ))) || + (buffered !== null && + input.modelSelection !== undefined && + modelSelectionsEqual(buffered, input.modelSelection)) + ); + }), + ), Effect.flatMap((loaded) => loaded - ? Effect.succeed({ - loaded: input.providerThread, - linkedEnvironments: undefined, - }) - : loadNatively(runtime.resumeThread(input)), + ? Effect.succeed({ loaded: input.providerThread, linkedEnvironments: undefined }) + : protectBufferedOutput( + input.providerThread, + input.modelSelection, + input.runtimePolicy, + ).pipe( + Effect.andThen(invalidateBinding(threadId)), + Effect.andThen(loadNatively(runtime.resumeThread(input))), + ), ), + Effect.tap(({ loaded }) => rememberBinding(loaded)), Effect.tap(({ loaded: providerThread, linkedEnvironments }) => markProviderThreadLoaded({ providerSessionId, @@ -2420,7 +2774,9 @@ export const layerWithOptions = ( providerInstanceId: runtime.instanceId, }), ).pipe( + Effect.andThen(invalidateBinding(input.targetThreadId)), Effect.andThen(loadNatively(runtime.forkThread(input))), + Effect.tap(({ loaded }) => rememberBinding(loaded)), Effect.tap(({ loaded: providerThread, linkedEnvironments }) => markProviderThreadLoaded({ providerSessionId, @@ -2440,64 +2796,94 @@ export const layerWithOptions = ( ), Effect.map(({ loaded }) => loaded), ), - startTurn: (input) => - observeActivity( + startTurn: Effect.fn("ProviderSessionManager.startTurn")(function* (input) { + const previous = yield* selectionEvidence(input.threadId); + const reapplySelection = + previous?.runAttemptId !== undefined && previous.selection == null; + const deliversBufferedOutput = + input.message.createdBy === "agent" && input.message.creationSource === "provider"; + if (!deliversBufferedOutput) { + yield* protectBufferedOutput( + input.providerThread, + input.modelSelection, + input.runtimePolicy, + true, + ); + } + yield* observeActivity( providerSessionId, ensureThreadAttached({ providerSessionId, threadId: input.threadId, providerInstanceId: runtime.instanceId, }), - ).pipe( - // Failed or stopped starts may omit turn.terminal; clear their turns so sessions can idle. - Effect.andThen( - holdThreadLoaded({ - providerSessionId, - threadId: input.threadId, - providerThread: input.providerThread, - }), - ), - Effect.andThen( - Effect.acquireUseRelease( - markBusy( + ); + yield* holdThreadLoaded({ + providerSessionId, + threadId: input.threadId, + providerThread: input.providerThread, + }); + // Failed or stopped starts may omit turn.terminal; clear their turns so sessions can idle. + return yield* Effect.acquireUseRelease( + markBusy( + providerSessionId, + busyTurnKey(input.providerThread.id, input.runOrdinal), + runtime, + ).pipe( + Effect.catchCause((cause) => + Effect.logWarning("orchestration-v2.driver-session.activity-failed", { providerSessionId, - busyTurnKey(input.providerThread.id, input.runOrdinal), - runtime, - ).pipe( - Effect.catchCause((cause) => - Effect.logWarning("orchestration-v2.driver-session.activity-failed", { - providerSessionId, - cause, - }).pipe(Effect.as(true)), - ), - // A release between attach and markBusy leaves nothing to mark: fail the - // start instead of running it on a runtime being torn down. - Effect.flatMap((marked) => - marked - ? Effect.void - : Effect.fail( - new ProviderAdapter.ProviderAdapterTurnStartError({ - driver: runtime.driver, - threadId: input.threadId, - providerThreadId: input.providerThread.id, - runId: input.runId, - cause: "The provider session was released while the turn started.", - }), - ), - ), - ), - () => - runtime.startTurn(input).pipe(turnMetrics("send", input.modelSelection.model)), - (_, exit) => - Exit.isFailure(exit) - ? observeActivity( - providerSessionId, - markIdle(providerSessionId, input.providerThread.id, input.runOrdinal), - ) - : Effect.void, + cause, + }).pipe(Effect.as(true)), + ), + // A release between attach and markBusy leaves nothing to mark: fail the + // start instead of running it on a runtime being torn down. + Effect.flatMap((marked) => + marked + ? Effect.void + : Effect.fail( + new ProviderAdapter.ProviderAdapterTurnStartError({ + driver: runtime.driver, + threadId: input.threadId, + providerThreadId: input.providerThread.id, + runId: input.runId, + cause: "The provider session was released while the turn started.", + }), + ), ), ), - ), + () => + observeSelection( + input.threadId, + runtime + .startTurn({ ...input, reapplySelection }) + .pipe(turnMetrics("send", input.modelSelection.model)), + { + providerThread: input.providerThread, + providerThreadId: input.providerThread.id, + nativeThreadId: input.providerThread.nativeThreadRef?.nativeId ?? null, + selection: null, + runAttemptId: input.attemptId, + loadKey: providerThreadLoadKey({ + runtime, + providerThread: input.providerThread, + modelSelection: input.modelSelection, + runtimePolicy: input.runtimePolicy, + }), + ...(deliversBufferedOutput ? { bufferedRunOrdinal: input.runOrdinal } : {}), + }, + input.modelSelection, + runtime.continuationDrainsOutput === true && deliversBufferedOutput, + ), + (_, exit) => + Exit.isFailure(exit) + ? observeActivity( + providerSessionId, + markIdle(providerSessionId, input.providerThread.id, input.runOrdinal), + ) + : Effect.void, + ); + }), steerTurn: (input) => observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe( Effect.andThen(runtime.steerTurn(input).pipe(turnMetrics("steer"))), @@ -2787,18 +3173,64 @@ export const layerWithOptions = ( lastSessionStatus = event.providerSession.status; if (lastSessionStatus === "stopped") stoppedByProvider = true; } - return observeActivity( - entry.runtime.providerSessionId, - event.type === "turn.terminal" - ? markIdle( - entry.runtime.providerSessionId, - event.providerThreadId, - event.runOrdinal, - ) - : metadataOnly - ? Effect.void - : touchActivity(entry.runtime.providerSessionId, { providerActivity: true }), - ).pipe( + const recordEvidence = Ref.update(sessions, (current) => { + const key = sessionKey(entry.runtime.providerSessionId); + const latest = current.get(key); + if (latest?.runtime !== entry.runtime) return current; + const executionSelections = new Map(latest.executionSelections); + if (event.type === "provider_turn.updated") { + for (const [threadId, evidence] of executionSelections) { + if ( + evidence.providerThreadId === event.providerTurn.providerThreadId && + evidence.runAttemptId === event.providerTurn.runAttemptId + ) { + executionSelections.set(threadId, { + ...evidence, + providerTurnId: event.providerTurn.id, + }); + } + } + } + if (event.type === "turn.terminal") { + for (const [threadId, evidence] of executionSelections) { + if ( + evidence.providerThreadId === event.providerThreadId && + evidence.providerTurnId === event.providerTurnId + ) { + const { bufferedRunOrdinal: _bufferedRunOrdinal, ...completed } = evidence; + executionSelections.set(threadId, completed); + } + } + } + return new Map(current).set(key, { ...latest, executionSelections }); + }); + return recordEvidence.pipe( + Effect.andThen( + observeActivity( + entry.runtime.providerSessionId, + event.type === "turn.terminal" + ? markIdle( + entry.runtime.providerSessionId, + event.providerThreadId, + event.runOrdinal, + ) + : metadataOnly + ? Effect.void + : touchActivity(entry.runtime.providerSessionId, { providerActivity: true }), + ), + ), + Effect.andThen( + event.type === "turn.terminal" + ? finishDeferredDetaches(entry).pipe( + Effect.catchCause((cause) => + Effect.logWarning( + "orchestration-v2.driver-session.deferred-detach-failed", + { providerSessionId: entry.runtime.providerSessionId, cause }, + ), + ), + ) + : Effect.void, + ), Effect.andThen(armRetryStall), Effect.andThen( event.type === "provider_session.updated" @@ -3066,12 +3498,15 @@ export const layerWithOptions = ( const eventSubscribers = yield* Ref.make< ReadonlyMap> >(new Map()); + const executionSelections = new Map(); const exposedRuntime = decorateRuntime(runtime, eventSubscribers); const entry: LiveSessionEntry = { attachedThreadIds: new Set([input.threadId]), loadedProviderThreadKeyByThread: new Map(), openedWithLinkedEnvironments, linkedEnvironmentsByThread: new Map(), + executionSelections, + preservedOutputThreadIds: new Map(), mcpCredentialIdByThread: mcpCredentialId === undefined ? new Map() @@ -3243,6 +3678,81 @@ export const layerWithOptions = ( const key = sessionKey(input.providerSessionId); const currentEntry = (yield* Ref.get(sessions)).get(key); let detachedProviderThreads: ReadonlyArray = []; + if (input.preserveBufferedOutput === true && currentEntry !== undefined) { + const evidence = currentEntry.executionSelections.get(input.threadId); + const projectedBinding = + evidence?.providerThread ?? + (yield* projectionStore.getThreadRecords(input.threadId, [ + "providerThreads", + ])).providerThreads.find( + (thread) => + thread.id === evidence?.providerThreadId || + thread.providerSessionId === input.providerSessionId, + ); + const binding = + projectedBinding === undefined + ? undefined + : { + ...projectedBinding, + providerSessionId: input.providerSessionId, + nativeThreadRef: + evidence?.nativeThreadId == null + ? projectedBinding.nativeThreadRef + : { + driver: currentEntry.runtime.driver, + nativeId: evidence.nativeThreadId, + strength: "strong" as const, + }, + }; + const buffered = + currentEntry.executionSelections.get(input.threadId)?.bufferedRunOrdinal !== + undefined; + const owned = + binding === undefined + ? false + : yield* ( + currentEntry.runtime.hasBufferedOutputForThread?.(binding) ?? + Effect.succeed(false) + ); + if ((buffered || owned) && binding !== undefined) { + const latest = (yield* Ref.get(sessions)).get(key); + if ( + latest?.runtime !== currentEntry.runtime || + latest.executionSelections.get(input.threadId)?.operation !== evidence?.operation + ) + return; + yield* Ref.update(sessions, (current) => { + const active = current.get(key); + if ( + active?.runtime !== currentEntry.runtime || + active.executionSelections.get(input.threadId)?.operation !== + evidence?.operation + ) + return current; + return new Map(current).set(key, { + ...active, + preservedOutputThreadIds: new Map(active.preservedOutputThreadIds).set( + input.threadId, + { binding, operation: evidence?.operation }, + ), + }); + }); + yield* scheduleIdleRelease(input.providerSessionId); + return; + } + const running = + binding === undefined + ? false + : yield* ( + currentEntry.runtime.hasPendingBackgroundWorkForThread?.(binding, { + includeSubagents: true, + }) ?? Effect.succeed(false) + ); + if (running) + return yield* new ProviderAdapter.ProviderAdapterRunningWorkError({ + driver: currentEntry.runtime.driver, + }); + } if (currentEntry?.supportsMultipleProviderThreads === true) { const projection = yield* Effect.option( projectionStore.getThreadRecords(input.threadId, [ @@ -3304,6 +3814,20 @@ export const layerWithOptions = ( loadedProviderThreadKeyByThread.delete(input.threadId); const linkedEnvironmentsByThread = new Map(entry.linkedEnvironmentsByThread); linkedEnvironmentsByThread.delete(input.threadId); + const executionSelections = new Map(entry.executionSelections); + const preservedOutputThreadIds = new Map(entry.preservedOutputThreadIds); + if (input.preserveBufferedOutput === true) { + const evidence = executionSelections.get(input.threadId); + if (evidence !== undefined) + executionSelections.set(input.threadId, { + ...evidence, + selection: null, + operation: undefined, + }); + } else { + executionSelections.delete(input.threadId); + } + preservedOutputThreadIds.delete(input.threadId); const mcpCredentialIdByThread = input.revokeMcpCredential === true ? (() => { @@ -3316,6 +3840,8 @@ export const layerWithOptions = ( idleThreadUnloads.delete(input.threadId); const updatedEntry = { ...entry, + executionSelections, + preservedOutputThreadIds, attachedThreadIds, loadedProviderThreadKeyByThread, linkedEnvironmentsByThread, diff --git a/apps/server/src/orchestration-v2/ProviderSwitchService.ts b/apps/server/src/orchestration-v2/ProviderSwitchService.ts index 69c3f351602b5..8c69dd58a0ee9 100644 --- a/apps/server/src/orchestration-v2/ProviderSwitchService.ts +++ b/apps/server/src/orchestration-v2/ProviderSwitchService.ts @@ -40,6 +40,8 @@ export interface ProviderSwitchServiceV2Shape { "thread" | "providerSessions" | "providerThreads" >; readonly targetModelSelection: ModelSelection; + readonly executionSelection?: ModelSelection | null; + readonly freshAttachment?: boolean; }) => Effect.Effect; } @@ -62,10 +64,19 @@ export const layer: Layer.Layer< const adapters = yield* ProviderAdapterRegistry.ProviderAdapterRegistryV2; return ProviderSwitchServiceV2.of({ plan: Effect.fn("ProviderSwitchService.plan")( - function* ({ projection, targetModelSelection }) { - const current = projection.thread.modelSelection; - const instanceChanged = current.instanceId !== targetModelSelection.instanceId; - const getMetadata = (instanceId: typeof current.instanceId) => + function* ({ projection, targetModelSelection, executionSelection, freshAttachment }) { + const current = + executionSelection === undefined + ? projection.thread.modelSelection + : executionSelection; + const currentInstanceId = + current?.instanceId ?? + projection.providerThreads.find( + (thread) => thread.id === projection.thread.activeProviderThreadId, + )?.providerInstanceId ?? + projection.thread.providerInstanceId; + const instanceChanged = currentInstanceId !== targetModelSelection.instanceId; + const getMetadata = (instanceId: ModelSelection["instanceId"]) => adapters.getMetadata !== undefined ? adapters.getMetadata(instanceId) : adapters.get(instanceId).pipe( @@ -80,26 +91,34 @@ export const layer: Layer.Layer< ), ), ); - const currentInstance = yield* Effect.option(getMetadata(current.instanceId)); + const currentInstance = yield* Effect.option(getMetadata(currentInstanceId)); const targetInstance = yield* Effect.option(getMetadata(targetModelSelection.instanceId)); const targetAdapter = yield* Effect.option(adapters.get(targetModelSelection.instanceId)); const currentSessions = projection.providerSessions - .filter((session) => session.providerInstanceId === current.instanceId) + .filter((session) => session.providerInstanceId === currentInstanceId) .toSorted( (left, right) => DateTime.toEpochMillis(right.updatedAt) - DateTime.toEpochMillis(left.updatedAt), ); - const currentSession = currentSessions.find(isLiveProviderSession); + const activeSessionId = projection.providerThreads.find( + (thread) => thread.id === projection.thread.activeProviderThreadId, + )?.providerSessionId; + const currentSession = currentSessions.find( + (session) => + isLiveProviderSession(session) && + (executionSelection === undefined || session.id === activeSessionId), + ); const negotiatedCapabilities = currentSession?.capabilities ?? currentSessions[0]?.capabilities; const currentProviderThread = projection.providerThreads.find( (thread) => thread.id === projection.thread.activeProviderThreadId && - thread.providerInstanceId === current.instanceId && + thread.providerInstanceId === currentInstanceId && thread.nativeThreadRef !== null, ); const selectionTransition = - current.instanceId === targetModelSelection.instanceId && + current !== null && + currentInstanceId === targetModelSelection.instanceId && !modelSelectionsEqual(current, targetModelSelection) && Option.isSome(targetAdapter) && Option.isSome(currentInstance) && @@ -117,42 +136,68 @@ export const layer: Layer.Layer< // workspace of whichever thread opened it, not a move of this thread. const workspace = projection.thread.worktreePath ?? currentSession?.cwd ?? ""; - const transition = - Option.isNone(targetInstance) || Option.isNone(targetAdapter) - ? ({ - type: "reject", - reason: "The target provider instance is unavailable.", - } as const) - : decideProviderSessionTransition({ - current: - Option.isNone(currentInstance) || - (currentSession === undefined && currentProviderThread === undefined) - ? null - : { - driver: currentInstance.value.driver, - continuationIdentity: { - driverKind: currentInstance.value.driver, - continuationKey: currentInstance.value.continuationKey, - }, - modelSelection: current, - runtimeMode: projection.thread.runtimeMode, - interactionMode: projection.thread.interactionMode, - workspace, - }, - target: { - driver: targetInstance.value.driver, - continuationIdentity: { - driverKind: targetInstance.value.driver, - continuationKey: targetInstance.value.continuationKey, + const canResumeUnknown = + currentProviderThread !== undefined && + Option.isSome(currentInstance) && + Option.isSome(targetInstance) && + currentInstance.value.driver === targetInstance.value.driver && + currentInstance.value.continuationKey === targetInstance.value.continuationKey; + const unknownTransition = (() => { + if (currentInstanceId === targetModelSelection.instanceId) { + return { + type: + freshAttachment === true ? ("reuse" as const) : ("restart_and_resume" as const), + }; + } + return { + type: canResumeUnknown + ? ("restart_and_resume" as const) + : ("create_with_handoff" as const), + }; + })(); + const transition = (() => { + if ( + Option.isNone(targetInstance) || + Option.isNone(targetAdapter) || + !targetInstance.value.enabled + ) { + return { + type: "reject" as const, + reason: "The target provider instance is unavailable.", + }; + } + if (current === null) return unknownTransition; + return decideProviderSessionTransition({ + current: + Option.isNone(currentInstance) || + (currentSession === undefined && currentProviderThread === undefined) + ? null + : { + driver: currentInstance.value.driver, + continuationIdentity: { + driverKind: currentInstance.value.driver, + continuationKey: currentInstance.value.continuationKey, + }, + modelSelection: current, + runtimeMode: projection.thread.runtimeMode, + interactionMode: projection.thread.interactionMode, + workspace, }, - modelSelection: targetModelSelection, - runtimeMode: projection.thread.runtimeMode, - interactionMode: projection.thread.interactionMode, - workspace, - available: targetInstance.value.enabled, - }, - ...(selectionTransition === undefined ? {} : { selectionTransition }), - }); + target: { + driver: targetInstance.value.driver, + continuationIdentity: { + driverKind: targetInstance.value.driver, + continuationKey: targetInstance.value.continuationKey, + }, + modelSelection: targetModelSelection, + runtimeMode: projection.thread.runtimeMode, + interactionMode: projection.thread.interactionMode, + workspace, + available: targetInstance.value.enabled, + }, + ...(selectionTransition === undefined ? {} : { selectionTransition }), + }); + })(); if (transition.type === "reject") { return yield* new ProviderSwitchPlanError({ threadId: projection.thread.id, diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 6587e742c7cd0..059903782ffd1 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -96,6 +96,9 @@ const isProviderAdapterProtocolError = Schema.is(ProviderAdapter.ProviderAdapter const isProviderAdapterOpenSessionError = Schema.is( ProviderAdapter.ProviderAdapterOpenSessionError, ); +const isProviderAdapterBufferedOutputError = Schema.is( + ProviderAdapter.ProviderAdapterBufferedOutputError, +); /** * Claude refuses these before it reads the prompt: replacing a process running background work, @@ -821,7 +824,8 @@ export const layer: Layer.Layer< ) { const loaded = yield* Effect.result(load); if (loaded._tag === "Success") return loaded.success; - if (input.willRetry === true) return yield* loaded.failure; + if (input.willRetry === true || isProviderAdapterBufferedOutputError(loaded.failure)) + return yield* loaded.failure; yield* settleStartFailure({ signal: "provider-thread-load-failure", title: "Provider turn failed to start", @@ -1109,7 +1113,7 @@ export const layer: Layer.Layer< let resumedHandoffContext: ReturnType | undefined; let portableForkFallback: OrchestrationV2ContextHandoff | undefined; let recoveryTurnItem: OrchestrationV2TurnItem | undefined; - const loadedProviderThread = yield* Effect.gen(function* () { + const bindingResult = yield* Effect.gen(function* () { if (nativeForkTransfer !== undefined) { const sourceProjection = yield* projectionStore.getThreadRecords( nativeForkTransfer.sourceThreadId, @@ -1385,6 +1389,8 @@ export const layer: Layer.Layer< ); } } + if (resumed._tag === "Failure" && isProviderAdapterBufferedOutputError(resumed.failure)) + return yield* resumed.failure; resumedHandoffContext = undefined; resumedRetryHandoff = []; if (!(yield* isCurrentAttemptInStatus("starting"))) return undefined; @@ -1509,7 +1515,19 @@ export const layer: Layer.Layer< }); recoveryTurnItem = handoffTurnItem; return replacement; - }); + }).pipe(Effect.result); + if (bindingResult._tag === "Failure") { + if (!isProviderAdapterBufferedOutputError(bindingResult.failure)) + return yield* bindingResult.failure; + yield* settleStartFailure({ + signal: "provider-buffered-output-owned", + title: "Provider output must be delivered first", + error: bindingResult.failure, + message: "Buffered provider output must be delivered before changing its binding.", + }); + return; + } + const loadedProviderThread = bindingResult.success; if (loadedProviderThread === undefined) return; if (!(yield* isCurrentAttemptInStatus("starting"))) { return; diff --git a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts index 7b12f72f4cead..12d8bd09826e6 100644 --- a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts +++ b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts @@ -16,6 +16,7 @@ import { ProviderTurnId, type RunId, ThreadId, + TurnItemId, } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -24,15 +25,18 @@ import * as Layer from "effect/Layer"; import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; import * as EffectWorker from "./EffectWorker.ts"; import * as EventSink from "./EventSink.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; +import * as ProviderSessionManager from "./ProviderSessionManager.ts"; import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts"; import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; const driver = ProviderDriverKind.make("codex"); const providerInstanceId = ProviderInstanceId.make("codex-restart-test"); @@ -1037,3 +1041,713 @@ it.live.each(["active", "idle", "selection-command", "pooled", "separate-home"] }).pipe(Effect.provide(ProviderReplayHarness.layerWithRegistry({ name }, layerRegistry))); }), ); + +const admissionInstanceId = ProviderInstanceId.make("codex"); +const effortSelection = (effort: string): ModelSelection => ({ + instanceId: admissionInstanceId, + model: "same-model", + options: [{ id: "effort", value: effort }], +}); +const effortOf = (selection: ModelSelection) => + selection.options?.find((option) => option.id === "effort")?.value; +const admissionCapabilities: OrchestrationV2ProviderCapabilities = { + ...CodexProviderCapabilitiesV2, + turns: { ...CodexProviderCapabilitiesV2.turns, supportsSteeringByInterruptRestart: false }, + sessions: { + ...CodexProviderCapabilitiesV2.sessions, + supportsMultipleProviderThreadsPerSession: false, + supportsModelSwitchInSession: false, + }, +}; +const lowToHighRestarts = ({ + current, + target, +}: { + readonly current: ModelSelection; + readonly target: ModelSelection; +}) => + Effect.succeed({ + type: + effortOf(current) === "low" && effortOf(target) === "high" + ? ("restart_session" as const) + : ("apply_on_next_turn" as const), + }); +const runningTurn = ( + events: Queue.Queue, + turn: ProviderAdapter.ProviderAdapterV2TurnInput, +) => + Effect.gen(function* () { + yield* Queue.offer(events, { + type: "provider_turn.updated", + driver, + providerTurn: { + id: ProviderTurnId.make(`provider-turn:${turn.attemptId}`), + providerThreadId: turn.providerThread.id, + nodeId: turn.rootNodeId, + runAttemptId: turn.attemptId, + nativeTurnRef: { driver, nativeId: `native:${turn.attemptId}`, strength: "strong" }, + ordinal: turn.providerTurnOrdinal, + status: "running", + startedAt: yield* DateTime.now, + completedAt: null, + }, + }); + }); +const admissionProviderThread = ( + providerSessionId: ProviderSessionId, + threadId: ThreadId, + existingProviderThread: OrchestrationV2ProviderThread | undefined, + nativeId: string, +) => + Effect.map(DateTime.now, (now): OrchestrationV2ProviderThread => ({ + id: existingProviderThread?.id ?? ProviderThreadId.make(`provider-thread:${threadId}`), + driver, + providerInstanceId: admissionInstanceId, + providerSessionId, + appThreadId: threadId, + ownerNodeId: null, + nativeThreadRef: { driver, nativeId, strength: "strong" }, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: null, + lastRunOrdinal: null, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + })); +const admissionSession = ( + providerSessionId: ProviderSessionId, + cwd: string, + model: string, + capabilities: OrchestrationV2ProviderCapabilities, +) => + Effect.map(DateTime.now, (now): OrchestrationV2ProviderSession => ({ + id: providerSessionId, + driver, + providerInstanceId: admissionInstanceId, + status: "ready", + cwd, + model, + capabilities, + createdAt: now, + updatedAt: now, + lastError: null, + })); +const admissionThread = (threadId: ThreadId, modelSelection: ModelSelection, cwd: string) => ({ + type: "thread.create" as const, + commandId: CommandId.make(`create:${threadId}`), + threadId, + projectId: ProjectId.make("project:admission"), + title: "Admission", + modelSelection, + runtimeMode: "full-access" as const, + interactionMode: "default" as const, + branch: null, + worktreePath: cwd, + createdBy: "user" as const, + creationSource: "web" as const, +}); +const userMessage = (threadId: ThreadId, name: string, modelSelection?: ModelSelection) => ({ + type: "message.dispatch" as const, + commandId: CommandId.make(name), + threadId, + messageId: MessageId.make(`message:${name}`), + text: name, + attachments: [], + ...(modelSelection === undefined ? {} : { modelSelection }), + dispatchMode: { type: "start_immediately" as const }, + createdBy: "user" as const, + creationSource: "web" as const, +}); +/** Starts the thread's next turn and waits until the provider reports it running. */ +const startAdmissionTurn = (threadId: ThreadId, name: string, modelSelection?: ModelSelection) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const running = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.threadId === threadId && + event.type === "provider-turn.updated" && + event.payload.status === "running", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* orchestrator.dispatch(userMessage(threadId, name, modelSelection)); + yield* worker.drain(); + yield* Fiber.join(running); + }); +/** Completes the thread's latest (or given) run through the event stream its attempt started on. */ +const completeAdmissionTurn = ( + threadId: ThreadId, + eventsFor: (attemptId: string) => Queue.Queue, + runId?: RunId, +) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const projection = yield* orchestrator.getThreadProjection(threadId); + const run = + runId === undefined ? projection.runs.at(-1)! : projection.runs.find((r) => r.id === runId)!; + const turn = projection.providerTurns.find( + (candidate) => candidate.runAttemptId === run.activeAttemptId, + )!; + const waiting = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "run.updated" && + event.payload.id === run.id && + event.payload.status === "waiting", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + const events = eventsFor(run.activeAttemptId!); + yield* Queue.offer(events, { + type: "provider_turn.updated", + driver, + providerTurn: { ...turn, status: "completed", completedAt: yield* DateTime.now }, + }); + yield* Queue.offer(events, { + type: "turn.terminal", + driver, + providerThreadId: turn.providerThreadId, + providerTurnId: turn.id, + runOrdinal: run.ordinal, + status: "completed", + failure: null, + threadDisposition: "reusable", + }); + yield* Fiber.join(waiting); + yield* worker.drain(); + }); + +it.effect("plans a selection change from the selection that last ran, not a steered one", () => + Effect.scoped( + Effect.gen(function* () { + const cwd = yield* checkpointWorkspace("execution-admission"); + const threadId = ThreadId.make("thread:execution-admission"); + const A = effortSelection("low"); + const B = effortSelection("medium"); + const C = effortSelection("high"); + const queues = new Map>(); + const calls: ProviderAdapter.ProviderAdapterV2TurnInput[] = []; + const opens: string[] = []; + const closes: string[] = []; + const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { + driver, + instanceId: admissionInstanceId, + getCapabilities: () => Effect.succeed(admissionCapabilities), + planSelectionTransition: lowToHighRestarts, + openSession: (input) => + Effect.gen(function* () { + const events = yield* Queue.unbounded(); + let loaded = input.modelSelection; + opens.push(input.providerSessionId); + yield* Effect.addFinalizer(() => + Effect.sync(() => void closes.push(input.providerSessionId)), + ); + return { + driver, + instanceId: admissionInstanceId, + providerSessionId: input.providerSessionId, + providerSession: yield* admissionSession( + input.providerSessionId, + cwd, + input.modelSelection.model, + admissionCapabilities, + ), + events: Stream.fromQueue(events), + ensureThread: ({ threadId, existingProviderThread }) => + admissionProviderThread( + input.providerSessionId, + threadId, + existingProviderThread, + `native:${input.providerSessionId}`, + ), + resumeThread: ({ providerThread }) => Effect.succeed(providerThread), + startTurn: (turn) => + Effect.gen(function* () { + assert.isFalse( + effortOf(loaded) === "low" && effortOf(turn.modelSelection) === "high", + "an incompatible native operation must not reach the adapter", + ); + calls.push(turn); + loaded = turn.modelSelection; + queues.set(turn.attemptId, events); + yield* runningTurn(events, turn); + }), + steerTurn: () => Effect.void, + interruptTurn: () => Effect.void, + respondToRuntimeRequest: () => Effect.void, + readThreadSnapshot: () => Effect.die("unused"), + rollbackThread: () => Effect.die("unused"), + forkThread: () => Effect.die("unused"), + }; + }), + }; + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + yield* orchestrator.dispatch(admissionThread(threadId, A, cwd)); + yield* startAdmissionTurn(threadId, "first"); + const first = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!; + // The steer saves B for the next turn; no turn ever runs on B. + yield* orchestrator.dispatch({ + ...userMessage(threadId, "desired", B), + dispatchMode: { type: "steer_active", targetRunId: first.id }, + }); + yield* worker.drain(); + yield* completeAdmissionTurn(threadId, (attemptId) => queues.get(attemptId)!); + yield* orchestrator.dispatch(userMessage(threadId, "following", C)); + yield* worker.drain(); + const after = yield* orchestrator.getThreadProjection(threadId); + assert.equal(calls.length, 2); + assert.deepEqual(calls[1]!.modelSelection, C); + assert.deepEqual(after.thread.modelSelection, C); + assert.equal(opens.length, 2, "A to C restarts even though B to C would not"); + assert.equal( + calls[1]!.providerThread.nativeThreadRef?.nativeId, + calls[0]!.providerThread.nativeThreadRef?.nativeId, + ); + assert.include(closes, opens[0]!); + assert.notInclude(closes, opens[1]!, "the old session's detach spares its replacement"); + }).pipe( + Effect.provide( + ProviderReplayHarness.layerWithRegistry( + { name: "execution-admission" }, + ProviderAdapterRegistry.layerSingle(adapter), + { runEffectWorker: false }, + ), + ), + ); + }), + ), +); + +it.effect("opens a fresh session after a start that failed partway, without resending it", () => + Effect.scoped( + Effect.gen(function* () { + const cwd = yield* checkpointWorkspace("admission-failed-start"); + const threadId = ThreadId.make("thread:admission-failed-start"); + const A = effortSelection("low"); + const B = effortSelection("medium"); + const C = effortSelection("high"); + const queues = new Map>(); + const closed = new Set(); + let pending = false; + let opens = 0; + const calls: ProviderAdapter.ProviderAdapterV2TurnInput[] = []; + const plans: Array<{ readonly current: ModelSelection; readonly target: ModelSelection }> = + []; + const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { + driver, + instanceId: admissionInstanceId, + getCapabilities: () => Effect.succeed(admissionCapabilities), + planSelectionTransition: (input) => + Effect.suspend(() => { + plans.push(input); + return lowToHighRestarts(input); + }), + openSession: (input) => + Effect.gen(function* () { + opens += 1; + const events = yield* Queue.unbounded(); + yield* Effect.addFinalizer(() => + Effect.sync(() => void closed.add(input.providerSessionId)), + ); + return { + driver, + instanceId: admissionInstanceId, + providerSessionId: input.providerSessionId, + providerSession: yield* admissionSession( + input.providerSessionId, + cwd, + input.modelSelection.model, + admissionCapabilities, + ), + events: Stream.fromQueue(events), + hasPendingBackgroundWork: Effect.sync(() => pending), + ensureThread: ({ threadId, existingProviderThread }) => + admissionProviderThread( + input.providerSessionId, + threadId, + existingProviderThread, + "native:failed-start", + ), + resumeThread: ({ providerThread }) => Effect.succeed(providerThread), + startTurn: (turn) => + Effect.gen(function* () { + calls.push(turn); + queues.set(turn.attemptId, events); + // The error does not say whether the native selection changed first. + if (turn.message.text === "reject") + return yield* new ProviderAdapter.ProviderAdapterTurnStartError({ + driver, + threadId: turn.threadId, + providerThreadId: turn.providerThread.id, + runId: turn.runId, + cause: "controlled request rejection", + }); + yield* runningTurn(events, turn); + }), + steerTurn: () => Effect.void, + interruptTurn: () => Effect.void, + respondToRuntimeRequest: () => Effect.void, + readThreadSnapshot: () => Effect.die("unused"), + rollbackThread: () => Effect.die("unused"), + forkThread: () => Effect.die("unused"), + }; + }), + }; + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + yield* orchestrator.dispatch(admissionThread(threadId, A, cwd)); + yield* startAdmissionTurn(threadId, "first"); + yield* completeAdmissionTurn(threadId, (attemptId) => queues.get(attemptId)!); + yield* TestClock.adjust("1 second"); + yield* orchestrator.dispatch(userMessage(threadId, "reject", B)); + yield* worker.drain(); + const failed = yield* orchestrator.getThreadProjection(threadId); + assert.equal(failed.runs.at(-1)!.status, "failed"); + pending = true; + const blocked = yield* Effect.exit( + orchestrator.dispatch(userMessage(threadId, "blocked-replacement", C)), + ); + assert.equal( + blocked._tag, + "Failure", + "an unknown selection cannot replace a runtime with pending work", + ); + assert.equal(opens, 1); + assert.equal(closed.size, 0); + pending = false; + yield* orchestrator.dispatch({ + type: "thread.model-selection.set", + commandId: CommandId.make("set-c"), + threadId, + modelSelection: C, + }); + yield* worker.drain(); + yield* startAdmissionTurn(threadId, "following"); + assert.equal(opens, 2, "the uncertain resident session is not reused"); + assert.deepEqual( + calls.map((call) => call.message.messageId), + ["message:first", "message:reject", "message:following"], + ); + assert.deepEqual(calls.at(-1)!.modelSelection, C); + const oldSessionId = calls[0]!.providerThread.providerSessionId!; + const newSessionId = calls.at(-1)!.providerThread.providerSessionId!; + assert.notEqual(oldSessionId, newSessionId); + assert.isTrue(closed.has(oldSessionId)); + assert.isFalse(closed.has(newSessionId)); + assert.isFalse( + plans.some( + (plan) => effortOf(plan.current) === "medium" && effortOf(plan.target) === "high", + ), + "the failed B start is not execution evidence", + ); + }).pipe( + Effect.provide( + ProviderReplayHarness.layerWithRegistry( + { name: "admission-failed-start" }, + ProviderAdapterRegistry.layerSingle(adapter), + { runEffectWorker: false }, + ), + ), + ); + }), + ), +); + +const drainVariants = ["idle", "queued", "queued-wake"] as const; +it.effect.each(drainVariants)("drains output before changing selection (%s)", (variant) => + Effect.scoped( + Effect.gen(function* () { + const queued = variant !== "idle"; + const cwd = yield* checkpointWorkspace("buffered-output-admission"); + const ownerId = ThreadId.make("thread:buffer-owner"); + const siblingId = ThreadId.make("thread:buffer-sibling"); + const A = effortSelection("low"); + const C = effortSelection("high"); + const queues = new Map>(); + const buffers = new Map(); + const continuations: ProviderContinuationRequests.ProviderContinuationRequest[] = []; + const calls: ProviderAdapter.ProviderAdapterV2TurnInput[] = []; + const opens: string[] = []; + const closes: string[] = []; + const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { + driver, + instanceId: admissionInstanceId, + getCapabilities: () => Effect.succeed(pooledCapabilities), + planSelectionTransition: lowToHighRestarts, + openSession: (input) => + Effect.gen(function* () { + const events = yield* Queue.unbounded(); + opens.push(input.providerSessionId); + yield* Effect.addFinalizer(() => + Effect.sync(() => void closes.push(input.providerSessionId)), + ); + const bindings = new Set(); + const bind = (thread: OrchestrationV2ProviderThread) => + Effect.sync(() => { + bindings.add(thread.id); + return thread; + }); + return { + driver, + instanceId: admissionInstanceId, + providerSessionId: input.providerSessionId, + providerSession: yield* admissionSession( + input.providerSessionId, + cwd, + input.modelSelection.model, + pooledCapabilities, + ), + events: Stream.fromQueue(events), + hasPendingBackgroundWork: Effect.sync(() => + [...bindings].some((id) => buffers.has(id)), + ), + hasPendingBackgroundWorkForThread: () => Effect.succeed(false), + hasBufferedOutputForThread: (thread) => Effect.sync(() => buffers.has(thread.id)), + continuationDrainsOutput: true, + liveExecutionSelection: () => Effect.succeed(null), + ensureThread: ({ threadId, existingProviderThread }) => + admissionProviderThread( + input.providerSessionId, + threadId, + existingProviderThread, + `native:${threadId}`, + ).pipe(Effect.flatMap(bind)), + resumeThread: ({ providerThread }) => bind(providerThread), + startTurn: (turn) => + Effect.gen(function* () { + calls.push(turn); + queues.set(turn.attemptId, events); + yield* runningTurn(events, turn); + if (turn.message.creationSource !== "provider") return; + const text = buffers.get(turn.providerThread.id); + assert.isDefined(text, "the original runtime owns the output"); + buffers.delete(turn.providerThread.id); + const at = yield* DateTime.now; + yield* Queue.offer(events, { + type: "turn_item.updated", + driver, + turnItem: { + id: TurnItemId.make(`item:buffer:${turn.message.messageId}`), + threadId: turn.threadId, + runId: turn.runId, + nodeId: turn.rootNodeId, + providerThreadId: turn.providerThread.id, + providerTurnId: ProviderTurnId.make(`provider-turn:${turn.attemptId}`), + nativeItemRef: null, + parentItemId: null, + ordinal: turn.runOrdinal * 100 + 1, + status: "completed", + title: null, + startedAt: at, + completedAt: at, + updatedAt: at, + type: "assistant_message", + text: text!, + messageId: MessageId.make(`assistant:${turn.message.messageId}`), + streaming: false, + }, + }); + }), + steerTurn: () => Effect.void, + interruptTurn: () => Effect.void, + respondToRuntimeRequest: () => Effect.void, + readThreadSnapshot: () => Effect.die("unused"), + rollbackThread: () => Effect.die("unused"), + forkThread: () => Effect.die("unused"), + }; + }), + }; + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const eventsFor = (attemptId: string) => queues.get(attemptId)!; + for (const id of [ownerId, siblingId]) + yield* orchestrator.dispatch(admissionThread(id, A, cwd)); + yield* startAdmissionTurn(ownerId, "first"); + if (!queued) yield* completeAdmissionTurn(ownerId, eventsFor); + yield* startAdmissionTurn(siblingId, "sibling"); + assert.lengthOf(opens, 1, "both threads share one provider process"); + const owner = (yield* orchestrator.getThreadProjection(ownerId)).providerThreads[0]!; + const oldSession = owner.providerSessionId!; + buffers.set(owner.id, "Owned original output"); + const wake = (name: string) => ({ + type: "message.dispatch" as const, + commandId: CommandId.make(name), + threadId: ownerId, + messageId: MessageId.make(`message:${name}`), + text: "Do not execute this placeholder", + attachments: [], + dispatchMode: { type: "queue_after_active" as const }, + createdBy: "agent" as const, + creationSource: "provider" as const, + notification: { + source: { kind: "background_task" as const }, + outcome: "updated" as const, + summary: "Owned output", + }, + }); + const runOf = (name: string) => + Effect.map(orchestrator.getThreadProjection(ownerId), (projection) => + projection.runs.find((run) => run.userMessageId === `message:${name}`)!, + ); + /** Completes `ending`, then waits for the queue to start `next` on its own. */ + const queueStarts = (next: string, ending: string) => + Effect.gen(function* () { + const leftQueue = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "run.updated" && + event.payload.userMessageId === `message:${next}` && + event.payload.status !== "queued", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + const running = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.threadId === ownerId && + event.type === "provider-turn.updated" && + event.payload.status === "running", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* completeAdmissionTurn(ownerId, eventsFor, (yield* runOf(ending)).id); + yield* Fiber.join(leftQueue); + yield* worker.drain(); + yield* Fiber.join(running); + }); + if (variant === "queued-wake") { + // The wake is already queued behind the message when the message's start is refused. + yield* orchestrator.dispatch({ + ...userMessage(ownerId, "queued-selection", C), + dispatchMode: { type: "queue_after_active" }, + }); + yield* orchestrator.dispatch(wake("queued-wake")); + yield* queueStarts("queued-wake", "first"); + assert.equal(calls.at(-1)!.message.messageId, "message:queued-wake"); + assert.equal(calls.at(-1)!.providerThread.providerSessionId, oldSession); + assert.isFalse(buffers.has(owner.id), "the queued wake delivers the output unprompted"); + assert.lengthOf(continuations, 0, "the queued wake is the drain"); + assert.equal((yield* runOf("queued-selection")).status, "queued"); + yield* queueStarts("queued-selection", "queued-wake"); + assert.equal(calls.at(-1)!.message.messageId, "message:queued-selection"); + assert.deepEqual(calls.at(-1)!.modelSelection, C); + assert.lengthOf(opens, 2); + assert.notInclude(closes, oldSession, "the other thread on the process keeps running"); + return; + } + if (queued) { + yield* orchestrator.dispatch({ + ...userMessage(ownerId, "queued-selection", C), + dispatchMode: { type: "queue_after_active" }, + }); + yield* completeAdmissionTurn( + ownerId, + eventsFor, + (yield* orchestrator.getThreadProjection(ownerId)).runs.find( + (run) => run.userMessageId === "message:first", + )!.id, + ); + yield* worker.drain(); + const refused = (yield* orchestrator.getThreadProjection(ownerId)).runs.find( + (run) => run.userMessageId === "message:queued-selection", + )!; + assert.equal(refused.status, "failed", "a refused start must not block the queue"); + assert.lengthOf(calls, 2, "the buffered binding is not replaced"); + } + yield* orchestrator.dispatch({ + type: "thread.model-selection.set", + commandId: CommandId.make("set-c"), + threadId: ownerId, + modelSelection: C, + }); + yield* worker.drain(); + assert.notInclude(closes, oldSession, "the process holding the output stays up"); + const blocked = yield* Effect.exit( + orchestrator.dispatch(userMessage(ownerId, "blocked-following")), + ); + assert.equal(blocked._tag, "Failure", "no new execution before the output is delivered"); + const drain = continuations.at(-1); + assert.isDefined(drain, "admission offers a drain without another native frame"); + assert.equal(drain!.providerThreadId, owner.id); + const delivered = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "turn-item.updated" && + event.payload.type === "assistant_message" && + event.payload.text === "Owned original output", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* drain!.dispatchIfCurrent!(orchestrator.dispatch(wake("deliver"))); + yield* worker.drain(); + yield* Fiber.join(delivered); + assert.equal((yield* drain!.dispatchIfCurrent!(Effect.die("stale drain")))._tag, "None"); + assert.equal(calls.at(-1)!.providerThread.providerSessionId, oldSession); + assert.deepEqual(calls.at(-1)!.modelSelection, A); + const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; + const resident = yield* sessions.get(oldSession); + assert.equal(resident._tag, "Some"); + if (resident._tag !== "Some") return; + assert.isNull( + yield* resident.value.executionSelection!(owner), + "a drain whose query is no longer acknowledged must not retain its selection", + ); + assert.deepEqual( + (yield* orchestrator.getThreadProjection(ownerId)).thread.modelSelection, + C, + ); + yield* completeAdmissionTurn( + ownerId, + eventsFor, + (yield* orchestrator.getThreadProjection(ownerId)).runs.find( + (run) => run.userMessageId === "message:deliver", + )!.id, + ); + yield* startAdmissionTurn(ownerId, "following"); + assert.lengthOf(opens, 2); + assert.deepEqual(calls.at(-1)!.modelSelection, C); + yield* completeAdmissionTurn(siblingId, eventsFor); + yield* startAdmissionTurn(siblingId, "sibling-following"); + assert.equal(calls.at(-1)!.providerThread.providerSessionId, oldSession); + assert.notInclude(closes, oldSession, "the other thread on the process keeps running"); + }).pipe( + Effect.provide( + ProviderReplayHarness.layerWithRegistry( + { name: "buffered-output-admission" }, + ProviderAdapterRegistry.layerSingle(adapter), + { runEffectWorker: false }, + ).pipe( + Layer.provide( + Layer.succeed(ProviderContinuationRequests.ProviderContinuationRequests, { + offer: (request) => + Effect.sync(() => { + continuations.push(request); + }), + take: Effect.never, + }), + ), + ), + ), + ); + }), + ), +); diff --git a/apps/server/src/orchestration-v2/testkit/OrchestratorScenario.ts b/apps/server/src/orchestration-v2/testkit/OrchestratorScenario.ts index 0d428e1a6e1e0..1fce5747d3f4e 100644 --- a/apps/server/src/orchestration-v2/testkit/OrchestratorScenario.ts +++ b/apps/server/src/orchestration-v2/testkit/OrchestratorScenario.ts @@ -35,6 +35,12 @@ export type OrchestratorV2ScenarioStep = readonly await?: boolean; readonly key?: string; } + | { + /** Dispatches a command that must be refused; its error is kept under `key`. */ + readonly type: "dispatch_refused"; + readonly command: OrchestrationV2Command; + readonly key: string; + } | { readonly type: "advance_clock"; readonly duration: Duration.Input; @@ -93,6 +99,11 @@ export type OrchestratorV2ScenarioStep = readonly type: "release_replay_gate"; readonly label: string; } + | { + /** Waits until the replay holds the entry labelled `label`, and keeps holding it. */ + readonly type: "await_replay_gate"; + readonly label: string; + } | { readonly type: "capture_shell_snapshot"; readonly key: string; @@ -121,6 +132,8 @@ export interface OrchestratorV2ScenarioResult { readonly capturedShellSnapshots: ReadonlyMap; /** Live output each command streamed (never persisted). Provider replays fill it in. */ readonly commandOutput?: ReadonlyMap; + /** The errors of `dispatch_refused` steps, by key. */ + readonly refusedDispatches: ReadonlyMap; } export class OrchestratorV2ScenarioStepError extends Schema.TaggedError()( @@ -297,6 +310,7 @@ export function runOrchestratorV2Scenario( Fiber.Fiber, Orchestrator.OrchestratorV2Error> >(); const capturedShellSnapshots = new Map(); + const refusedDispatches = new Map(); let anonymousBackgroundDispatchIndex = 0; const awaitDispatch = Effect.fn("awaitDispatch")(function* (key: string) { @@ -629,17 +643,23 @@ export function runOrchestratorV2Scenario( } }); - const releaseReplayGate = Effect.fn("scenario.releaseReplayGate")(function* (label: string) { + const awaitReplayGate = Effect.fn("scenario.awaitReplayGate")(function* ( + label: string, + step: "await_replay_gate" | "release_replay_gate", + ) { const gate = options.replayGate; const reached = gate === undefined ? false : yield* Effect.promise(() => gate.waitForReached(label)); if (!reached) { return yield* new OrchestratorV2ScenarioStepError({ scenario: scenario.name, - step: `release_replay_gate:${label}:not_configured`, + step: `${step}:${label}:not_configured`, }); } - gate?.release(label); + }); + const releaseReplayGate = Effect.fn("scenario.releaseReplayGate")(function* (label: string) { + yield* awaitReplayGate(label, "release_replay_gate"); + options.replayGate?.release(label); }); for (const step of scenarioSteps(scenario)) { @@ -662,6 +682,17 @@ export function runOrchestratorV2Scenario( ); break; } + case "dispatch_refused": { + const result = yield* Effect.result(orchestrator.dispatch(step.command)); + if (result._tag === "Success") { + return yield* new OrchestratorV2ScenarioStepError({ + scenario: scenario.name, + step: `dispatch_refused:${step.key}:accepted`, + }); + } + refusedDispatches.set(step.key, result.failure); + break; + } case "advance_clock": { if (step.awaitProviderSessionRelease === undefined) { yield* TestClock.adjust(step.duration); @@ -722,6 +753,9 @@ export function runOrchestratorV2Scenario( case "release_replay_gate": yield* releaseReplayGate(step.label); break; + case "await_replay_gate": + yield* awaitReplayGate(step.label, "await_replay_gate"); + break; case "capture_shell_snapshot": capturedShellSnapshots.set(step.key, yield* orchestrator.getShellSnapshot()); break; @@ -776,6 +810,7 @@ export function runOrchestratorV2Scenario( projections, shellSnapshot, capturedShellSnapshots, + refusedDispatches, }; }), ).pipe( diff --git a/packages/provider-acp/src/server/AcpSessionRuntime.ts b/packages/provider-acp/src/server/AcpSessionRuntime.ts index 21ae7f7fb1990..d5ebf1d1e9365 100644 --- a/packages/provider-acp/src/server/AcpSessionRuntime.ts +++ b/packages/provider-acp/src/server/AcpSessionRuntime.ts @@ -1181,6 +1181,7 @@ export class AcpSessionRuntime extends Context.Service< readonly terminateProcessGroup?: Effect.Effect; readonly setMode: ( modeId: string, + force?: boolean, ) => Effect.Effect; readonly setSessionModel: ( modelId: string, @@ -1201,7 +1202,10 @@ export class AcpSessionRuntime extends Context.Service< readonly resendCurrentValue?: boolean; }, ) => Effect.Effect; - readonly setModel: (model: string) => Effect.Effect; + readonly setModel: ( + model: string, + force?: boolean, + ) => Effect.Effect; readonly request: ( method: string, payload: unknown, @@ -2749,9 +2753,9 @@ export const make = Effect.fn("AcpSessionRuntime.make")(function* ( // A session's mode is its `category: "mode"` config option. ACP v1 agents // that only advertise `modes` (gemini-cli) take `session/set_mode` // instead, which ACP v2 removed. - setMode: Effect.fn("AcpSessionRuntime.setMode")(function* (modeId) { + setMode: Effect.fn("AcpSessionRuntime.setMode")(function* (modeId, force = false) { const modeState = yield* Ref.get(modeStateRef); - if (modeState?.currentModeId === modeId) { + if (!force && modeState?.currentModeId === modeId) { return {} satisfies EffectAcpSchema.SetSessionModeResponse; } const modeConfigOption = (yield* SubscriptionRef.get(configOptionsRef))?.find( @@ -2764,7 +2768,9 @@ export const make = Effect.fn("AcpSessionRuntime.make")(function* ( yield* updateCurrentModeId(modeId); return {} satisfies EffectAcpSchema.SetSessionModeResponse; } - const response = yield* setConfigOption(modeConfigOption?.id ?? "mode", modeId); + const response = yield* setConfigOption(modeConfigOption?.id ?? "mode", modeId, { + resendCurrentValue: force, + }); const reported = parseSessionModeState({ configOptions: response.configOptions }); if (reported === undefined) { yield* updateCurrentModeId(modeId); @@ -2785,9 +2791,19 @@ export const make = Effect.fn("AcpSessionRuntime.make")(function* ( }), ), setConfigOption, - setModel: (model) => + setModel: (model, force = false) => getStartedState.pipe( - Effect.flatMap((started) => setConfigOption(started.modelConfigId ?? "model", model)), + Effect.flatMap((started) => + SubscriptionRef.get(configOptionsRef).pipe( + Effect.flatMap((configOptions) => + setConfigOption( + findModelConfigOption(configOptions)?.id ?? started.modelConfigId ?? "model", + model, + { resendCurrentValue: force }, + ), + ), + ), + ), Effect.asVoid, ), diff --git a/packages/provider-acp/src/server/adapter.ts b/packages/provider-acp/src/server/adapter.ts index b2a28c68c6c08..fe076e3e5110d 100644 --- a/packages/provider-acp/src/server/adapter.ts +++ b/packages/provider-acp/src/server/adapter.ts @@ -438,6 +438,7 @@ export interface AcpAdapterV2Flavor { readonly runtime: AcpSessionRuntime.AcpSessionRuntime["Service"]; readonly startResult: AcpSessionRuntime.AcpSessionRuntimeStartResult; readonly modelSelection: ModelSelection; + readonly force?: boolean; }) => Effect.Effect; /** * Model-selection option ids `applyModelSelection` sets itself under a @@ -445,6 +446,7 @@ export interface AcpAdapterV2Flavor { * `reasoningEffort` is its `effortLevel`). */ readonly ownedModelOptionIds?: ReadonlyArray; + readonly canForceModelSelection?: boolean; readonly sessionModeForPolicy?: ( policy: ProviderAdapter.ProviderAdapterV2RuntimePolicy, ) => string | undefined; @@ -1200,6 +1202,7 @@ function structuredFileChanges(toolCall: AcpToolCallState) { // Bound diff search to protect the event loop; empty-sided diffs need no search. const ACP_V1_DIFF_MAX_EDITS = 1_000; +const isAcpRequestError = Schema.is(EffectAcpErrors.AcpRequestError); /** * A `snippet` patch has no known location. Keep its recorded text, so no view numbers it @@ -1656,6 +1659,35 @@ function acpCarryoverTerminalShouldClearContinuation(input: { return !input.continuationOffered && input.wakeBufferLength === 0; } +function acpInitialAppliedSelection( + setup: AcpSessionRuntime.AcpSessionRuntimeStartResult["sessionSetupResult"], + selection: ModelSelection, + initialModeId: string | undefined, + attemptsModel: boolean, +): Omit & { readonly model?: string } { + const initialModel = + setup.models?.currentModelId ?? + setup.configOptions?.find((option) => option.category === "model")?.currentValue; + return { + instanceId: selection.instanceId, + ...(typeof initialModel === "string" + ? { model: initialModel } + : !attemptsModel + ? { model: selection.model } + : {}), + ...(selection.options === undefined + ? {} + : { + options: selection.options.flatMap((option) => { + const value = + setup.configOptions?.find((candidate) => candidate.id === option.id)?.currentValue ?? + (option.id === ACP_SESSION_MODE_OPTION_ID ? initialModeId : undefined); + return value === undefined ? [] : [{ id: option.id, value }]; + }), + }), + }; +} + function acpTurnStartShouldPreserveContinuation(input: { readonly continuationRequested: boolean; readonly isContinuationTurn: boolean; @@ -2090,6 +2122,10 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( yield* Ref.set(activeSessionSetup, setup); }); const activeSelection = yield* Ref.make(null); + const appliedSelections = yield* Ref.make< + ReadonlyMap & { readonly model?: string }> + >(new Map()); + const configurationAcknowledged = yield* Ref.make(false); const activeInteractionMode = yield* Ref.make(null); const promptInstructionStates = yield* Ref.make(new Map()); const promptRuntimeInstructions = yield* Ref.make(new Map()); @@ -2380,6 +2416,20 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( offered, })); }); + const wakeSelections = new WeakMap(); + const bufferWakeNotification = Effect.fnUntraced(function* ( + notification: EffectAcpSchema.SessionNotification, + ) { + const producingSelection = (yield* Ref.get(appliedSelections)).get( + notification.sessionId, + ); + if (producingSelection?.model !== undefined) + wakeSelections.set(notification, { + ...producingSelection, + model: producingSelection.model, + }); + yield* Ref.update(wakeBuffer, (current) => [...current, notification]); + }); const continuationRequested = yield* Ref.make(false); const continuationGeneration = yield* Ref.make(0); const continuationPermit = yield* Semaphore.make(1); @@ -5083,7 +5133,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( !isInTurnHandledAgentChatter && acpPostSettleWakeShouldBuffer(notification, backgroundWorkRunning) ) { - yield* Ref.update(wakeBuffer, (current) => [...current, notification]); + yield* bufferWakeNotification(notification); buffered = true; } if (!acpPostSettleContinuationOfferEvidence(notification, flavor)) { @@ -5389,7 +5439,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( update.sessionUpdate === "user_message_chunk" || (update.sessionUpdate === "state_update" && update.state === "running")) ) { - yield* Ref.update(wakeBuffer, (current) => [...current, notification]); + yield* bufferWakeNotification(notification); yield* offerContinuationRun(notification.sessionId); return; } @@ -5656,7 +5706,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( flavor.isProviderWakeNotification?.(notification) === true ) { if (hasDeferredBackgroundWork(context)) { - yield* Ref.update(wakeBuffer, (current) => [...current, notification]); + yield* bufferWakeNotification(notification); return; } yield* finalizeTurn(context, context.promptSettledStatus ?? "completed"); @@ -7607,15 +7657,136 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( } }); + const applyConfigurationStep = Effect.fnUntraced(function* ( + sessionId: string, + optionId: string, + value: string | boolean | ((result: A) => string | boolean | undefined), + operation: Effect.Effect, + recordSelection = true, + ) { + const result = yield* Effect.exit(operation); + if (Exit.isFailure(result)) { + const error = Cause.findErrorOption(result.cause); + if (Option.isNone(error) || !isAcpRequestError(error.value)) { + yield* Effect.logWarning("orchestration-v2.acp-selection-step-indeterminate", { + driver, + sessionId, + optionId, + }); + // The write may have taken effect, so later output has no known producing selection. + yield* Ref.update(appliedSelections, (current) => { + const previous = current.get(sessionId); + if (previous === undefined) return current; + const { model: _, ...unknown } = previous; + return new Map(current).set(sessionId, unknown); + }); + } + return yield* result; + } + const appliedValue = typeof value === "function" ? value(result.value) : value; + yield* Ref.update(appliedSelections, (current) => { + const previous = current.get(sessionId); + if (previous === undefined || !recordSelection || appliedValue === undefined) + return current; + const selection = + optionId === "model" + ? { ...previous, model: String(appliedValue) } + : { + ...previous, + options: [ + ...(previous.options ?? []).filter((option) => option.id !== optionId), + { id: optionId, value: appliedValue }, + ], + }; + return new Map(current).set(sessionId, selection); + }); + return result.value; + }); + const configureSession = Effect.fnUntraced(function* ( startResult: AcpSessionRuntime.AcpSessionRuntimeStartResult, modelSelection: ModelSelection, runtimePolicy: ProviderAdapter.ProviderAdapterV2RuntimePolicy, unsetOptionDefaults: NonNullable = [], + force = false, ) { + yield* Ref.set(configurationAcknowledged, false); yield* Ref.set(activeSelection, null); yield* validateModelSelection(modelSelection); const requestedModel = flavor.resolveModelId?.(modelSelection) ?? modelSelection.model; + const attemptsModel = + flavor.applyModelSelection !== undefined || + (requestedModel.length > 0 && + !["auto", "default"].includes(requestedModel) && + (startResult.sessionSetupResult.models != null || + startResult.sessionSetupResult.configOptions?.some( + (option) => option.category === "model", + ) === true)); + if (!(yield* Ref.get(appliedSelections)).has(startResult.sessionId)) { + const initialMode = yield* runtime.getModeState; + yield* Ref.update(appliedSelections, (current) => + new Map(current).set( + startResult.sessionId, + acpInitialAppliedSelection( + startResult.sessionSetupResult, + modelSelection, + initialMode?.currentModeId, + attemptsModel, + ), + ), + ); + } + const step = ( + id: string, + value: string | boolean | ((result: A) => string | boolean | undefined), + operation: Effect.Effect, + ) => applyConfigurationStep(startResult.sessionId, id, value, operation); + const rememberOption = (id: string, value: string | boolean | undefined) => + Ref.update(appliedSelections, (current) => { + const previous = current.get(startResult.sessionId); + if ( + previous === undefined || + value === undefined || + previous.options?.some((option) => option.id === id) + ) + return current; + return new Map(current).set(startResult.sessionId, { + ...previous, + options: [...(previous.options ?? []), { id, value }], + }); + }); + const setConfig = Effect.fnUntraced(function* (id: string, value: string | boolean) { + const previous = (yield* runtime.getConfigOptions).find( + (option) => option.id === id, + )?.currentValue; + yield* rememberOption(id, previous); + return yield* step( + id, + value, + runtime.setConfigOption(id, value, { resendCurrentValue: force }), + ); + }); + const setMode = Effect.fnUntraced(function* (mode: string) { + if (modelSelection.options?.some((option) => option.id === ACP_SESSION_MODE_OPTION_ID)) + yield* rememberOption( + ACP_SESSION_MODE_OPTION_ID, + (yield* runtime.getModeState)?.currentModeId, + ); + const prior = (yield* Ref.get(appliedSelections)).get(startResult.sessionId); + const nativeModeId = (yield* runtime.getConfigOptions).find( + (option) => option.category === "mode", + )?.id; + const modeId = prior?.options?.find( + (option) => option.id === ACP_SESSION_MODE_OPTION_ID || option.id === nativeModeId, + )?.id; + return yield* applyConfigurationStep( + startResult.sessionId, + modeId ?? ACP_SESSION_MODE_OPTION_ID, + mode, + runtime.setMode(mode, force), + modeId !== undefined, + ); + }); // Invalid params is the agent refusing the value, and the runtime // raises the same code when the session's model option does not offer // it. Other failures, such as sign-in or a lost connection, keep their @@ -7626,13 +7797,18 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( : error; let appliedModel: string | undefined; if (flavor.applyModelSelection !== undefined) { - appliedModel = yield* flavor - .applyModelSelection({ - runtime, - startResult, - modelSelection, - }) - .pipe(Effect.mapError(rejectedModel)); + appliedModel = yield* step( + "model", + modelSelection.model, + flavor + .applyModelSelection({ + runtime, + startResult, + modelSelection, + force, + }) + .pipe(Effect.mapError(rejectedModel)), + ); } else if (requestedModel.length > 0) { // Stored catalog choices can expire; explicitly configured custom IDs // are validated by the agent instead. @@ -7646,11 +7822,15 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( (choice) => choice.value === requestedModel, ) ) { - yield* runtime - .setConfigOption(modelOption.id, requestedModel, { - resendCurrentValue: flavor.alwaysSetModel === true, - }) - .pipe(Effect.mapError(rejectedModel)); + yield* step( + "model", + modelSelection.model, + runtime + .setConfigOption(modelOption.id, requestedModel, { + resendCurrentValue: force || flavor.alwaysSetModel === true, + }) + .pipe(Effect.mapError(rejectedModel)), + ); appliedModel = requestedModel; } } else if ( @@ -7658,8 +7838,12 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( (isCustomModel || modelState.availableModels.some((model) => model.modelId === requestedModel)) ) { - if (modelState.currentModelId !== requestedModel) { - yield* runtime.setSessionModel(requestedModel).pipe(Effect.mapError(rejectedModel)); + if (force || modelState.currentModelId !== requestedModel) { + yield* step( + "model", + modelSelection.model, + runtime.setSessionModel(requestedModel).pipe(Effect.mapError(rejectedModel)), + ); } appliedModel = requestedModel; } @@ -7757,7 +7941,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( ); if (!advertisedValues.includes(selection.value)) continue; } - yield* runtime.setConfigOption(selection.id, selection.value).pipe( + yield* setConfig(selection.id, selection.value).pipe( Effect.catchTags({ AcpRequestError: (error) => error.code === -32602 @@ -7783,16 +7967,16 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( } const policyMode = flavor.sessionModeForPolicy?.(runtimePolicy); if (policyMode !== undefined) { - yield* runtime.setMode(policyMode); + yield* setMode(policyMode); } const modeState = yield* runtime.getModeState; if ( modeSelection !== undefined && typeof modeSelection.value === "string" && modeState?.availableModes.some((mode) => mode.id === modeSelection.value) === true && - modeState.currentModeId !== modeSelection.value + (force || modeState.currentModeId !== modeSelection.value) ) { - yield* runtime.setMode(modeSelection.value); + yield* setMode(modeSelection.value); } const effectiveModeState = yield* runtime.getModeState; const effectiveConfigOptions = yield* runtime.getConfigOptions; @@ -7821,16 +8005,19 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( const planMode = effectiveModeState?.availableModes.find((mode) => isAcpPlanModeId(mode.id), ); - if (planMode !== undefined && effectiveModeState?.currentModeId !== planMode.id) { - yield* runtime.setMode(planMode.id); + if ( + planMode !== undefined && + (force || effectiveModeState?.currentModeId !== planMode.id) + ) { + yield* setMode(planMode.id); } for (const option of planSensitiveOptions) { const choices = option.options.flatMap((entry) => "value" in entry ? [entry.value] : entry.options.map((choice) => choice.value), ); const requested = choices.find(isAcpPlanModeId); - if (requested !== undefined && option.currentValue !== requested) { - yield* runtime.setConfigOption(option.id, requested); + if (requested !== undefined && (force || option.currentValue !== requested)) { + yield* setConfig(option.id, requested); } } } else { @@ -7838,12 +8025,12 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( if (nativeBuild !== undefined) { if ( nativeBuild.modeId !== undefined && - effectiveModeState?.currentModeId !== nativeBuild.modeId && + (force || effectiveModeState?.currentModeId !== nativeBuild.modeId) && effectiveModeState?.availableModes.some( (mode) => mode.id === nativeBuild.modeId, ) === true ) { - yield* runtime.setMode(nativeBuild.modeId); + yield* setMode(nativeBuild.modeId); } for (const saved of nativeBuild.configOptions) { const option = effectiveConfigOptions.find( @@ -7853,14 +8040,20 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( const choices = option.options.flatMap((entry) => "value" in entry ? [entry.value] : entry.options.map((choice) => choice.value), ); - if (option.currentValue !== saved.value && choices.includes(saved.value)) { - yield* runtime.setConfigOption(option.id, saved.value); + if ( + (force || option.currentValue !== saved.value) && + choices.includes(saved.value) + ) { + yield* setConfig(option.id, saved.value); } } nativeBuildConfigurationBySessionId.delete(startResult.sessionId); } } yield* validateModelSelection(modelSelection); + // Skipped and explicitly rejected options are settled outcomes. Only + // a step without a response leaves planning evidence unknown. + yield* Ref.set(configurationAcknowledged, true); yield* ( flavor.onSessionConfigurationUpdate?.( yield* runtime.getConfigOptions, @@ -8469,6 +8662,8 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( if (!restartRequired) return false; yield* restartAcpRuntime(threadId); yield* Ref.set(runtimeRestartRequired, false); + yield* Ref.set(appliedSelections, new Map()); + yield* Ref.set(configurationAcknowledged, false); yield* Ref.set(activeSessionId, null); yield* Ref.set(activeSessionSetup, null); yield* Ref.set(activeSelection, null); @@ -8500,19 +8695,35 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( const restartAfterInterrupt = yield* restartRuntimeAfterTeardownIfRequired( turnInput.threadId, ); + const isContinuationTurn = + (postSettleContinuationEnabled || followsAgentWorkState) && + acpIsProviderContinuationMessage(turnInput.message); const needsSessionActivation = (yield* Ref.get(activeSessionId)) !== requestedSessionId || restartAfterInterrupt; - if (needsSessionActivation) { + if (isContinuationTurn && needsSessionActivation) { + return yield* new ProviderAdapter.ProviderAdapterProtocolError({ + driver, + detail: "Buffered output belongs to another native session", + }); + } + if (!isContinuationTurn && needsSessionActivation) { const activated = yield* activateSession(requestedSessionId, turnInput.threadId); yield* Ref.set(activeSessionId, activated.sessionId); yield* setActiveSessionSetup(activated); - yield* configureSession(activated, turnInput.modelSelection, turnInput.runtimePolicy); + yield* configureSession( + activated, + turnInput.modelSelection, + turnInput.runtimePolicy, + [], + turnInput.reapplySelection, + ); yield* Ref.set(activeSelection, turnInput.modelSelection); yield* Ref.set(activeInteractionMode, turnInput.runtimePolicy.interactionMode); - } else { + } else if (!isContinuationTurn) { const configuredSelection = yield* Ref.get(activeSelection); const configuredInteractionMode = yield* Ref.get(activeInteractionMode); if ( + turnInput.reapplySelection === true || configuredSelection === null || !modelSelectionsEqual(configuredSelection, turnInput.modelSelection) || configuredInteractionMode !== turnInput.runtimePolicy.interactionMode @@ -8528,6 +8739,8 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( currentSessionSetup, turnInput.modelSelection, turnInput.runtimePolicy, + [], + turnInput.reapplySelection, ); yield* Ref.set(activeSelection, turnInput.modelSelection); yield* Ref.set(activeInteractionMode, turnInput.runtimePolicy.interactionMode); @@ -8556,10 +8769,6 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( }); yield* Ref.set(suppressPostSettleMonitorPrompt, false); yield* Ref.set(handledBackgroundTaskIdsInActiveTurn, new Set()); - // Continuations consume existing wake traffic without re-prompting ACP. - const isContinuationTurn = - (postSettleContinuationEnabled || followsAgentWorkState) && - acpIsProviderContinuationMessage(turnInput.message); const isAppOwnedWakeTurn = acpIsAppOwnedWakeTurn(turnInput.message); if (!isAppOwnedWakeTurn) { yield* Ref.set(midTurnUnreportedCompletedTaskIds, new Set()); @@ -8737,12 +8946,19 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( reports: new Map([...reports].filter(([key]) => !offered.has(key))), offered: noWakeReports.offered, })); + // Output with no known producing selection is still this session's, so any wake + // drains it; output another selection produced waits for that selection's wake. const takeWakeBuffer = Ref.modify(wakeBuffer, (current) => { - const next: Array = []; - return [ - current.filter((notification) => notification.sessionId === requestedSessionId), - next, - ] as const; + const boundary = current.findIndex((notification) => { + const selection = wakeSelections.get(notification); + return ( + notification.sessionId !== requestedSessionId || + (selection !== undefined && + !modelSelectionsEqual(selection, turnInput.modelSelection)) + ); + }); + const count = boundary < 0 ? current.length : boundary; + return [current.slice(0, count), current.slice(count)] as const; }); if (followsAgentWorkState) { yield* runtimeCallbackPermit.withPermit( @@ -9072,6 +9288,30 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( providerSession, canResumeAfterRestart: canLoadSession || canResumeSession, events: Stream.fromEffectRepeat(Queue.take(events)), + selectionAcknowledged: Effect.fn("AcpAdapter.selectionAcknowledged")( + function* (selection) { + const acknowledged = yield* Ref.get(activeSelection); + return ( + (yield* Ref.get(configurationAcknowledged)) && + acknowledged !== null && + modelSelectionsEqual(acknowledged, selection) + ); + }, + ), + reappliesFullSelection: Effect.fnUntraced(function* (selection) { + const sessionId = yield* Ref.get(activeSessionId); + const applied = + sessionId === null ? undefined : (yield* Ref.get(appliedSelections)).get(sessionId); + return ( + (flavor.applyModelSelection === undefined || + flavor.canForceModelSelection === true) && + !["auto", "default", "grok-build"].includes(selection.model) && + applied !== undefined && + (applied.options ?? []).every((option) => + selection.options?.some((next) => next.id === option.id), + ) + ); + }), ...(flavor.getModelContextWindow === undefined ? {} : { @@ -9084,6 +9324,28 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( }), ...(postSettleContinuationEnabled || followsAgentWorkState ? { + continuationDrainsOutput: true, + bufferedExecutionSelection: Effect.fn("AcpAdapter.bufferedExecutionSelection")( + function* (providerThread) { + const first = (yield* Ref.get(wakeBuffer))[0]; + if ( + first === undefined || + first.sessionId !== providerThread.nativeThreadRef?.nativeId + ) + return null; + return wakeSelections.get(first) ?? null; + }, + ), + hasBufferedOutputForThread: Effect.fn("AcpAdapter.hasBufferedOutputForThread")( + function* (providerThread) { + const sessionId = yield* Ref.get(activeSessionId); + return ( + sessionId !== null && + providerThread.nativeThreadRef?.nativeId === sessionId && + (yield* Ref.get(wakeBuffer)).length > 0 + ); + }, + ), hasPendingBackgroundWork: Effect.gen(function* () { const rootSessionId = yield* Ref.get(activeSessionId); if ( @@ -9447,6 +9709,20 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( teardownState: (yield* Ref.get(runtimeTeardownState))._tag, }, ); + if ( + context === null && + !containOrphanRuntime && + turnInput.requestRuntimeRestart === true && + turnInput.providerThread.nativeThreadRef?.nativeId === + (yield* Ref.get(activeSessionId)) + ) { + yield* runtimeCallbackPermit.withPermit( + Effect.gen(function* () { + yield* awaitAdmittedNativeResponses; + yield* quarantineStoppedRun(); + }), + ); + } if (containOrphanRuntime) { const teardownBarrier = yield* Deferred.make< void, diff --git a/packages/provider-core/src/server/ProviderAdapter.ts b/packages/provider-core/src/server/ProviderAdapter.ts index 80d3ef8a5285a..62fa7b5535191 100644 --- a/packages/provider-core/src/server/ProviderAdapter.ts +++ b/packages/provider-core/src/server/ProviderAdapter.ts @@ -489,7 +489,32 @@ export class ProviderAdapterSessionStoppedError extends Schema.TaggedError()( + "ProviderAdapterBufferedOutputError", + { driver: ProviderDriverKind, providerThreadId: ProviderThreadId }, +) { + override get message(): string { + return PROVIDER_BUFFERED_OUTPUT_MESSAGE; + } +} + +export class ProviderAdapterRunningWorkError extends Schema.TaggedError()( + "ProviderAdapterRunningWorkError", + { driver: ProviderDriverKind }, +) { + override get message(): string { + return PROVIDER_RUNNING_WORK_MESSAGE; + } +} + export const ProviderAdapterV2Error = Schema.Union([ + ProviderAdapterBufferedOutputError, + ProviderAdapterRunningWorkError, ProviderAdapterCapabilitiesError, ProviderAdapterOpenSessionError, ProviderAdapterCloseSessionError, @@ -536,6 +561,8 @@ export interface ProviderAdapterV2EnsureThreadInput { } export interface ProviderAdapterV2TurnInput { + /** The manager has no acknowledged selection for this resident binding. */ + readonly reapplySelection?: boolean; readonly appThread: OrchestrationV2AppThread; readonly threadId: ThreadId; readonly runId: RunId; @@ -699,6 +726,43 @@ export interface ProviderAdapterV2SessionRuntime { readonly events: Stream.Stream; /** Manager runtimes broadcast independent subscriptions; adapters may expose only a single-consumer stream. */ readonly subscribeEvents?: Effect.Effect; + /** Resident native binding, including a detached binding awaiting replacement. */ + readonly hasResidentBinding?: ( + providerThread: OrchestrationV2ProviderThread, + ) => Effect.Effect; + /** Output retained by this native binding, not its running-work roster. */ + readonly hasBufferedOutputForThread?: ( + providerThread: OrchestrationV2ProviderThread, + ) => Effect.Effect; + /** Complete producing-turn selection retained with native output, even after query exit. */ + readonly bufferedExecutionSelection?: ( + providerThread: OrchestrationV2ProviderThread, + ) => Effect.Effect; + /** Adopt owned native output with unknown selection without replacing its binding. + * Draining it cannot establish selection; a user prompt still needs native acknowledgement. + */ + readonly canAdoptUnselectedBufferedOutput?: ( + providerThread: OrchestrationV2ProviderThread, + selection: ModelSelection, + ) => Effect.Effect; + /** Last acknowledged complete selection on the same still-live native query. */ + readonly liveExecutionSelection?: ( + providerThread: OrchestrationV2ProviderThread, + ) => Effect.Effect; + /** A failed same-selection start cannot change native selection configuration. */ + readonly preservesSelectionOnSameSelectionFailure?: boolean; + /** True only when every supported selection field is explicit in the next wire request. */ + readonly reappliesFullSelection?: (selection: ModelSelection) => Effect.Effect; + /** Continuations only deliver existing output and cannot acknowledge configuration. */ + readonly continuationDrainsOutput?: boolean; + /** Settled configuration check after a real start succeeds. Deterministic + * degradation can acknowledge the request; buffered output keeps its applied label. + */ + readonly selectionAcknowledged?: (selection: ModelSelection) => Effect.Effect; + /** Residency-owned evidence; absent or null means selection is not established. */ + readonly executionSelection?: ( + providerThread: OrchestrationV2ProviderThread, + ) => Effect.Effect; readonly hasPendingBackgroundWork?: Effect.Effect; /** Root-subscription probe; replacement checks opt in to subagents owned by the live process. */ readonly hasPendingBackgroundWorkForThread?: ( @@ -733,6 +797,11 @@ export interface ProviderAdapterV2SessionRuntime { readonly providerThread: OrchestrationV2ProviderThread; }, ) => Effect.Effect; + /** + * Success acknowledges the full requested selection after native configuration or prompt + * acceptance; deterministically skipped or rejected options settle without claiming application. + * Caching or scheduling setup does not acknowledge; failure or interruption establishes nothing. + */ readonly startTurn: ( input: ProviderAdapterV2TurnInput, ) => Effect.Effect; diff --git a/packages/provider-core/src/server/failure.ts b/packages/provider-core/src/server/failure.ts index 4131e470fa2f5..be8832ebd01d6 100644 --- a/packages/provider-core/src/server/failure.ts +++ b/packages/provider-core/src/server/failure.ts @@ -16,6 +16,10 @@ import * as Cause from "effect/Cause"; import * as IdAllocator from "./IdAllocator.ts"; import type { ProviderAdapterFailureReason } from "./ProviderAdapter.ts"; import * as Schema from "effect/Schema"; +import { + PROVIDER_BUFFERED_OUTPUT_MESSAGE, + PROVIDER_RUNNING_WORK_MESSAGE, +} from "./ProviderAdapter.ts"; export class ContextHandoffBudgetError extends Schema.TaggedError()( "ContextHandoffBudgetError", @@ -108,6 +112,10 @@ function curatedCauseMessages(cause: unknown): { if (typeof cause !== "object") break; const tag = (cause as Record)._tag; switch (tag) { + case "ProviderAdapterRunningWorkError": + return { known: PROVIDER_RUNNING_WORK_MESSAGE, category: categoryMessage() }; + case "ProviderAdapterBufferedOutputError": + return { known: PROVIDER_BUFFERED_OUTPUT_MESSAGE, category: categoryMessage() }; case "ContextHandoffBudgetError": return { known: new ContextHandoffBudgetError().message, category: categoryMessage() }; case "ContextHandoffBackgroundWorkError": diff --git a/packages/provider-grok/src/server/adapter.ts b/packages/provider-grok/src/server/adapter.ts index 0e8d9b6e04714..04873db9ce725 100644 --- a/packages/provider-grok/src/server/adapter.ts +++ b/packages/provider-grok/src/server/adapter.ts @@ -238,8 +238,11 @@ const applyGrokSessionModelSelection: NonNullable applyGrokAcpModelSelection({ runtime, - currentModelId, + currentModelId: resend ? undefined : currentModelId, requestedModelId, requestedReasoningEffort, mapError: (cause) => cause, @@ -290,8 +293,10 @@ const applyGrokSessionModelSelection: NonNullable runtime.setModel(model).pipe(Effect.as({})) }, - currentModelId: findModelConfigOption(yield* runtime.getConfigOptions)?.currentValue, + runtime: { setSessionModel: (model) => runtime.setModel(model, resend).pipe(Effect.as({})) }, + currentModelId: resend + ? undefined + : findModelConfigOption(yield* runtime.getConfigOptions)?.currentValue, requestedModelId, mapError: (cause) => cause, }); @@ -362,6 +367,7 @@ export function makeGrokAcpAdapterFlavor( ? window : undefined; }, + canForceModelSelection: true, applyModelSelection: applyGrokSessionModelSelection, ownedModelOptionIds: [GROK_REASONING_EFFORT_OPTION_ID], makeRuntime: diff --git a/packages/provider-muse/src/server/adapter.ts b/packages/provider-muse/src/server/adapter.ts index dbf27ebea95a7..3a305abf36871 100644 --- a/packages/provider-muse/src/server/adapter.ts +++ b/packages/provider-muse/src/server/adapter.ts @@ -356,6 +356,7 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* ( let nativeSessionId: string | undefined; let thread: OrchestrationV2ProviderThread | undefined; let active: ActiveTurn | undefined; + let executionSelection: ModelSelection | null = null; let lastProviderTurn: OrchestrationV2ProviderTurn | undefined; let latestSubscriptionUsage: MuseSubscriptionUsage | undefined; const agentEndedAt = new Map(); @@ -371,7 +372,11 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* ( const agentTitles = new Map(); const finishedBackground: Array = []; let wake: - | { readonly nativeId: string; readonly events: Array<[string, unknown]> } + | { + readonly nativeId: string; + readonly selection: ModelSelection | null; + readonly events: Array<[string, unknown]>; + } | undefined; const emit = (event: ProviderAdapter.ProviderAdapterV2Event) => Queue.offer(events, event).pipe(Effect.asVoid); @@ -1441,7 +1446,7 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* ( ) { if (active && !active.compact) active.joined.add(params.turnId); else if (!active && !wake) { - const held = { nativeId: params.turnId, events: [] }; + const held = { nativeId: params.turnId, selection: executionSelection, events: [] }; wake = held; const notification = backgroundWorkNotification(finishedBackground.splice(0)); if (thread?.appThreadId && options.continuationRequests) @@ -2057,6 +2062,8 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* ( }); const validateThread = (candidate: OrchestrationV2ProviderThread) => thread !== undefined && + candidate.id === thread.id && + candidate.providerSessionId === input.providerSessionId && candidate.driver === MUSE_PROVIDER && candidate.providerInstanceId === options.instanceId && candidate.appThreadId === thread.appThreadId && @@ -2097,6 +2104,7 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* ( const targetModel = continuation ? undefined : yield* resolveModel(turnInput.modelSelection); + if (!continuation && !compact) executionSelection = null; if (targetModel && session.model !== targetModel) { yield* request("session/setModel", { model: { modelId: targetModel, providerId: "meta" }, @@ -2208,6 +2216,7 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* ( ); return yield* protocolError("Muse admitted the turn under an unexpected identity"); } + executionSelection = turnInput.modelSelection; } }).pipe( Effect.onError((cause) => { @@ -2273,6 +2282,23 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* ( providerSession: session, events: Stream.fromQueue(events), backgroundTasksEndWithProcess: true, + continuationDrainsOutput: true, + hasBufferedOutputForThread: (candidate) => + Effect.sync(() => validateThread(candidate) && wake !== undefined), + bufferedExecutionSelection: (candidate) => + Effect.sync(() => (validateThread(candidate) ? (wake?.selection ?? null) : null)), + canAdoptUnselectedBufferedOutput: (candidate, selection) => + Effect.sync( + () => + validateThread(candidate) && + !broken && + !closed && + wake !== undefined && + wake.selection === null && + executionSelection === null && + selection.instanceId === options.instanceId && + selection.model === session.model, + ), // A running workflow or a held Muse turn keeps the host: idle release must wait. hasPendingBackgroundWork: Effect.sync( () => observedChildren.size > 0 || wake !== undefined, diff --git a/packages/provider-opencode/src/server/adapter.ts b/packages/provider-opencode/src/server/adapter.ts index 602977628612d..9773a015d8cdc 100644 --- a/packages/provider-opencode/src/server/adapter.ts +++ b/packages/provider-opencode/src/server/adapter.ts @@ -3588,6 +3588,15 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function const runtimeSession: ProviderAdapter.ProviderAdapterV2SessionRuntime = { instanceId: options.instanceId, + preservesSelectionOnSameSelectionFailure: true, + reappliesFullSelection: (selection) => + Effect.succeed( + getModelSelectionStringOptionValue(selection, "agent") !== undefined && + getModelSelectionStringOptionValue(selection, "variant") !== undefined && + (selection.options ?? []).every((option) => + ["agent", "variant"].includes(option.id), + ), + ), driver: OPENCODE_PROVIDER, providerSessionId: input.providerSessionId, providerSession: sessionEntity, diff --git a/packages/provider-opencode/src/server/v2/adapter.ts b/packages/provider-opencode/src/server/v2/adapter.ts index 3a83c81557d5a..2c7d2ce3312f2 100644 --- a/packages/provider-opencode/src/server/v2/adapter.ts +++ b/packages/provider-opencode/src/server/v2/adapter.ts @@ -4608,6 +4608,13 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr const state = nativeId == null ? undefined : threads.get(nativeId); return state !== undefined && hasBackground(state); }), + hasBufferedOutputForThread: (providerThread) => + Effect.sync(() => { + const nativeId = providerThread.nativeThreadRef?.nativeId; + const state = nativeId == null ? undefined : threads.get(nativeId); + return state !== undefined && state.wakes.length > 0; + }), + continuationDrainsOutput: true, getModelContextWindow: (selection, cwd) => selection.instanceId === instanceId ? windowOf(cwd === undefined ? session.cwd : cwd, selection.model)