diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 27f7aa82fd25c..37e71cea00d4e 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -1030,6 +1030,7 @@ function queueTailModelSelection( return ( queuedRunsInDeliveryOrder(projection).findLast( (run) => + run.threadPickSuperseded !== true && !followsCurrentConversation( projection.messages.find((message) => message.id === run.userMessageId), ), @@ -2658,9 +2659,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio projection.thread.modelSelection, queuedRun.modelSelection, ); - // The picker may already have moved ahead of the native conversation. + // The picker, or a superseded queued start, may have left the thread ahead of the native conversation. const switchFromActiveProvider = - followsCurrent && activeProviderThread !== undefined && activeProviderThread.providerInstanceId !== queuedRun.providerInstanceId; const switchPlan = @@ -3111,7 +3111,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio : "Provider or model selection changed."; yield* writeSystemEvents( [ - ...(selectionChanged + // A scheduled task's pinned model still moves the thread back when it starts. + ...(selectionChanged && + (queuedRun.threadPickSuperseded !== true || + (queuedMessage.scheduledTaskId !== undefined && + queuedMessage.followsThreadSelection !== true)) ? [ { type: @@ -5269,6 +5273,25 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio occurredAt: now, payload: updatedThread, }); + if (command.type === "thread.model-selection.set" || command.type === "provider.switch") { + const { runs } = yield* loadProjectionForCommand(command, ["runs"], { + runStatuses: ["queued"], + }); + for (const run of runs) { + if (run.threadPickSuperseded === true) continue; + yield* emit( + events, + command, + )({ + type: "run.updated", + threadId: command.threadId, + runId: run.id, + providerInstanceId: run.providerInstanceId, + occurredAt: now, + payload: { ...run, threadPickSuperseded: true }, + }); + } + } // Archiving ends the guest share, so its waiting files go too. yield* releaseGuestAttachments(command, thread.guestShare, updatedThread.guestShare, effects); diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index 0614a14861abd..c299648e61c5c 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -8998,6 +8998,83 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => { }), ); + it.effect("keeps a model picked after a message was queued when that message starts", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const threadId = ThreadId.make("runtime-layer-pick-after-queued-message"); + const pickedSelection = { ...modelSelection, model: "gpt-5.5" }; + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make(`${threadId}:create`), + threadId, + projectId: ProjectId.make(`${threadId}:project`), + createdBy: "user", + creationSource: "web", + title: "Pick after a queued message", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: process.cwd(), + }); + for (const key of ["active", "queued"] as const) { + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`${threadId}:${key}`), + threadId, + messageId: MessageId.make(`${threadId}:${key}`), + text: key, + attachments: [], + modelSelection, + dispatchMode: { type: key === "active" ? "start_immediately" : "queue_after_active" }, + createdBy: "user", + creationSource: "web", + }); + } + // The web picker persists a same-instance pick at once (useApplyThreadModelPick). + yield* orchestrator.dispatch({ + type: "thread.model-selection.set", + commandId: CommandId.make(`${threadId}:pick`), + threadId, + modelSelection: pickedSelection, + }); + // A message queued after the pick without its own model takes the pick, not the older queue tail. + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`${threadId}:implicit`), + threadId, + messageId: MessageId.make(`${threadId}:implicit`), + text: "implicit", + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "user", + creationSource: "web", + }); + const [active, queued, implicit] = (yield* orchestrator.getThreadProjection(threadId)).runs; + assert.isDefined(active); + assert.isDefined(queued); + assert.deepEqual(implicit?.modelSelection, pickedSelection); + const now = yield* DateTime.now; + yield* eventSink.write({ + events: [ + { + id: EventId.make(`${active.id}:completed`), + type: "run.updated", + threadId, + runId: active.id, + occurredAt: now, + payload: { ...active, status: "completed", completedAt: now }, + }, + ], + }); + yield* orchestrator.resumeQueuedRuns; + const after = yield* orchestrator.getThreadProjection(threadId); + assert.equal(after.runs.find((run) => run.id === queued.id)?.status, "starting"); + assert.deepEqual(after.thread.modelSelection, pickedSelection); + }), + ); + it.effect("keeps a buffered provider wake on its own provider behind a queued pick", () => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; diff --git a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts index edc12dcd98840..9f55c03d0c367 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts @@ -2373,185 +2373,235 @@ describe("orchestration v2 provider switching", () => { }), ); - it.live("resumes a queued account switch without requiring a portable handoff", () => - Effect.gen(function* () { - const cwd = yield* checkpointWorkspace("queued-account-switch"); - const capturedTurns = yield* Ref.make>([]); - const started = yield* Deferred.make(); - const alternateSelection: ModelSelection = { - ...CODEX_MODEL_SELECTION, - instanceId: ProviderInstanceId.make("codex-alternate"), - }; - const alternateCapabilities = { - ...CodexProviderCapabilitiesV2, - canConsumeHandoffSummaries: false, - }; - const adapters = [ - makeTestAdapter({ - instanceId: CODEX_MODEL_SELECTION.instanceId, - driver: CODEX_DRIVER, - capabilities: CodexProviderCapabilitiesV2, - modelSelection: CODEX_MODEL_SELECTION, - responseByRunOrdinal: {}, - capturedTurns, - holdFirstTurn: started, - }), - makeTestAdapter({ - instanceId: alternateSelection.instanceId, - driver: CODEX_DRIVER, - capabilities: alternateCapabilities, - modelSelection: alternateSelection, - responseByRunOrdinal: { 2: "Alternate account complete" }, - capturedTurns, - }), - ]; - const layerRegistry = Layer.succeed( - ProviderAdapterRegistry.ProviderAdapterRegistryV2, - ProviderAdapterRegistry.ProviderAdapterRegistryV2.of({ - get: (instanceId) => { - const adapter = adapters.find((candidate) => candidate.instanceId === instanceId); - return adapter === undefined - ? Effect.fail( - new ProviderAdapterRegistry.ProviderAdapterRegistryLookupError({ instanceId }), - ) - : Effect.succeed(adapter); - }, - list: () => Effect.succeed(adapters.map((adapter) => adapter.instanceId)), - getMetadata: (instanceId) => { - const adapter = adapters.find((candidate) => candidate.instanceId === instanceId); - return adapter === undefined - ? Effect.fail( - new ProviderAdapterRegistry.ProviderAdapterRegistryLookupError({ instanceId }), - ) - : Effect.succeed({ - driver: CODEX_DRIVER, - continuationKey: "codex:shared-native-account-history", - enabled: true, - capabilities: - instanceId === alternateSelection.instanceId - ? alternateCapabilities - : CodexProviderCapabilitiesV2, - }); - }, - }), - ); - const queuedThreadId = ThreadId.make("thread:queued-account-switch"); - const projection = yield* Effect.gen(function* () { - const orchestrator = yield* Orchestrator.OrchestratorV2; - const eventSink = yield* EventSink.EventSinkV2; - const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; - yield* orchestrator.dispatch({ - type: "thread.create", - createdBy: "user", - creationSource: "web", - commandId: CommandId.make("command:queued-account-switch:create"), - threadId: queuedThreadId, - projectId: ProjectId.make("project:queued-account-switch"), - title: "Queued account switch", - modelSelection: CODEX_MODEL_SELECTION, - runtimeMode: "full-access", - interactionMode: "default", - branch: null, - worktreePath: cwd, - }); - yield* orchestrator.dispatch({ - type: "message.dispatch", - createdBy: "user", - creationSource: "web", - commandId: CommandId.make("command:queued-account-switch:first"), - threadId: queuedThreadId, - messageId: MessageId.make("message:queued-account-switch:first"), - text: "First account turn", - attachments: [], - modelSelection: CODEX_MODEL_SELECTION, - dispatchMode: { type: "start_immediately" }, - }); - yield* Deferred.await(started); - yield* orchestrator.dispatch({ - type: "message.dispatch", - createdBy: "user", - creationSource: "web", - commandId: CommandId.make("command:queued-account-switch:second"), - threadId: queuedThreadId, - messageId: MessageId.make("message:queued-account-switch:second"), - text: "Alternate account turn", - attachments: [], - modelSelection: alternateSelection, - dispatchMode: { type: "queue_after_active" }, - }); - const queued = yield* orchestrator.getThreadProjection(queuedThreadId); - assert.deepEqual( - queued.runs.map((run) => run.status), - ["running", "queued"], + it.live.each(["behind a running turn", "after a turn queued before a newer pick"] as const)( + "resumes a queued account switch without requiring a portable handoff %s", + (scenario) => + Effect.gen(function* () { + const supersededPick = scenario !== "behind a running turn"; + const cwd = yield* checkpointWorkspace("queued-account-switch"); + const capturedTurns = yield* Ref.make>([]); + const started = yield* Deferred.make(); + const alternateSelection: ModelSelection = { + ...CODEX_MODEL_SELECTION, + instanceId: ProviderInstanceId.make("codex-alternate"), + }; + const alternateCapabilities = { + ...CodexProviderCapabilitiesV2, + canConsumeHandoffSummaries: false, + }; + const adapters = [ + makeTestAdapter({ + instanceId: CODEX_MODEL_SELECTION.instanceId, + driver: CODEX_DRIVER, + capabilities: CodexProviderCapabilitiesV2, + modelSelection: CODEX_MODEL_SELECTION, + responseByRunOrdinal: {}, + capturedTurns, + holdFirstTurn: started, + }), + makeTestAdapter({ + instanceId: alternateSelection.instanceId, + driver: CODEX_DRIVER, + capabilities: alternateCapabilities, + modelSelection: alternateSelection, + responseByRunOrdinal: { 2: "Alternate account complete" }, + capturedTurns, + }), + ]; + const layerRegistry = Layer.succeed( + ProviderAdapterRegistry.ProviderAdapterRegistryV2, + ProviderAdapterRegistry.ProviderAdapterRegistryV2.of({ + get: (instanceId) => { + const adapter = adapters.find((candidate) => candidate.instanceId === instanceId); + return adapter === undefined + ? Effect.fail( + new ProviderAdapterRegistry.ProviderAdapterRegistryLookupError({ instanceId }), + ) + : Effect.succeed(adapter); + }, + list: () => Effect.succeed(adapters.map((adapter) => adapter.instanceId)), + getMetadata: (instanceId) => { + const adapter = adapters.find((candidate) => candidate.instanceId === instanceId); + return adapter === undefined + ? Effect.fail( + new ProviderAdapterRegistry.ProviderAdapterRegistryLookupError({ instanceId }), + ) + : Effect.succeed({ + driver: CODEX_DRIVER, + continuationKey: "codex:shared-native-account-history", + enabled: true, + capabilities: + instanceId === alternateSelection.instanceId + ? alternateCapabilities + : CodexProviderCapabilitiesV2, + }); + }, + }), ); - assert.equal(queued.thread.providerInstanceId, CODEX_MODEL_SELECTION.instanceId); - const sourceNativeRef = queued.providerThreads.find( - (providerThread) => providerThread.id === queued.runs[0]?.providerThreadId, - )?.nativeThreadRef; - assert.isNotNull(sourceNativeRef); - const now = yield* DateTime.now; - yield* eventSink.write({ - events: [ - { - id: EventId.make("event:queued-account-switch:first-complete"), - type: "run.updated", + const queuedThreadId = ThreadId.make("thread:queued-account-switch"); + const projection = yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + yield* orchestrator.dispatch({ + type: "thread.create", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make("command:queued-account-switch:create"), + threadId: queuedThreadId, + projectId: ProjectId.make("project:queued-account-switch"), + title: "Queued account switch", + modelSelection: CODEX_MODEL_SELECTION, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: cwd, + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make("command:queued-account-switch:first"), + threadId: queuedThreadId, + messageId: MessageId.make("message:queued-account-switch:first"), + text: "First account turn", + attachments: [], + modelSelection: CODEX_MODEL_SELECTION, + dispatchMode: { type: "start_immediately" }, + }); + yield* Deferred.await(started); + if (supersededPick) { + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make("command:queued-account-switch:earlier"), + threadId: queuedThreadId, + messageId: MessageId.make("message:queued-account-switch:earlier"), + text: "Earlier account turn", + attachments: [], + modelSelection: CODEX_MODEL_SELECTION, + dispatchMode: { type: "queue_after_active" }, + }); + } + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make("command:queued-account-switch:second"), + threadId: queuedThreadId, + messageId: MessageId.make("message:queued-account-switch:second"), + text: "Alternate account turn", + attachments: [], + modelSelection: alternateSelection, + dispatchMode: { type: "queue_after_active" }, + }); + const queued = yield* orchestrator.getThreadProjection(queuedThreadId); + assert.deepEqual( + queued.runs.map((run) => run.status), + supersededPick ? ["running", "queued", "queued"] : ["running", "queued"], + ); + assert.equal(queued.thread.providerInstanceId, CODEX_MODEL_SELECTION.instanceId); + const sourceNativeRef = queued.providerThreads.find( + (providerThread) => providerThread.id === queued.runs[0]?.providerThreadId, + )?.nativeThreadRef; + assert.isNotNull(sourceNativeRef); + if (supersededPick) { + // A held queue lets the pick land before the earlier Codex turn starts. That start keeps + // the pick, so the thread is on the alternate account while Codex owns the native thread. + yield* orchestrator.dispatch({ + type: "run.interrupt", + commandId: CommandId.make("command:queued-account-switch:interrupt"), threadId: queuedThreadId, runId: queued.runs[0]!.id, - providerInstanceId: CODEX_MODEL_SELECTION.instanceId, - occurredAt: now, - payload: { ...queued.runs[0]!, status: "completed", completedAt: now }, - }, - ], - }); - yield* orchestrator.resumeQueuedRuns; - yield* orchestrator.streamStoredEvents.pipe( - Stream.filter( - (event) => - event.event.type === "run.updated" && - event.event.runId === queued.runs[1]?.id && - event.event.payload.status === "completed", - ), - Stream.runHead, - ); - yield* worker.drain(); - const delivered = yield* orchestrator.getThreadProjection(queuedThreadId); - const targetNativeRef = delivered.providerThreads.find( - (providerThread) => providerThread.id === delivered.runs[1]?.providerThreadId, - )?.nativeThreadRef; - assert.deepEqual(targetNativeRef, sourceNativeRef); - return delivered; - }).pipe( - Effect.provide( - ProviderReplayHarness.layerWithRegistry( - { - name: "queued-account-switch", - runtimePolicyOverride: { - cwd, - approvalPolicy: "never", - sandboxPolicy: { - type: "readOnly", - access: { type: "fullAccess" }, - networkAccess: false, + holdQueue: true, + }); + yield* orchestrator.dispatch({ + type: "thread.model-selection.set", + commandId: CommandId.make("command:queued-account-switch:pick"), + threadId: queuedThreadId, + modelSelection: alternateSelection, + }); + yield* orchestrator.dispatch({ + type: "queue.resume", + commandId: CommandId.make("command:queued-account-switch:resume"), + threadId: queuedThreadId, + }); + } else { + const now = yield* DateTime.now; + yield* eventSink.write({ + events: [ + { + id: EventId.make("event:queued-account-switch:first-complete"), + type: "run.updated", + threadId: queuedThreadId, + runId: queued.runs[0]!.id, + providerInstanceId: CODEX_MODEL_SELECTION.instanceId, + occurredAt: now, + payload: { ...queued.runs[0]!, status: "completed", completedAt: now }, + }, + ], + }); + } + const alternateRun = queued.runs.at(-1); + yield* orchestrator.resumeQueuedRuns; + yield* orchestrator.streamStoredEvents.pipe( + Stream.filter( + (event) => + event.event.type === "run.updated" && + event.event.runId === alternateRun?.id && + event.event.payload.status === "completed", + ), + Stream.runHead, + ); + yield* worker.drain(); + const delivered = yield* orchestrator.getThreadProjection(queuedThreadId); + const targetNativeRef = delivered.providerThreads.find( + (providerThread) => providerThread.id === delivered.runs.at(-1)?.providerThreadId, + )?.nativeThreadRef; + assert.deepEqual(targetNativeRef, sourceNativeRef); + return delivered; + }).pipe( + Effect.provide( + ProviderReplayHarness.layerWithRegistry( + { + name: "queued-account-switch", + runtimePolicyOverride: { + cwd, + approvalPolicy: "never", + sandboxPolicy: { + type: "readOnly", + access: { type: "fullAccess" }, + networkAccess: false, + }, }, }, - }, - layerRegistry, + layerRegistry, + ), ), - ), - ); - assert.deepEqual( - projection.runs.map((run) => [run.providerInstanceId, run.status]), - [ - [CODEX_MODEL_SELECTION.instanceId, "completed"], - [alternateSelection.instanceId, "completed"], - ], - ); - assert.lengthOf(projection.contextHandoffs, 0); - assert.deepEqual( - (yield* Ref.get(capturedTurns)).map((turn) => turn.text), - ["First account turn", "Alternate account turn"], - ); - }), + ); + assert.deepEqual( + projection.runs.map((run) => [run.providerInstanceId, run.status]), + [ + [CODEX_MODEL_SELECTION.instanceId, supersededPick ? "interrupted" : "completed"], + ...(supersededPick ? [[CODEX_MODEL_SELECTION.instanceId, "completed"]] : []), + [alternateSelection.instanceId, "completed"], + ], + ); + // The interrupted turn reaches the earlier Codex turn as a same-thread delta, never the switch. + assert.deepEqual( + projection.contextHandoffs.map((handoff) => handoff.targetRunId), + supersededPick ? [projection.runs[1]!.id] : [], + ); + const turnTexts = (yield* Ref.get(capturedTurns)).map((turn) => turn.text); + assert.equal(turnTexts.at(-1), "Alternate account turn"); + assert.deepEqual( + turnTexts.map((text) => text.split("\n").at(-1)), + supersededPick + ? ["First account turn", "Earlier account turn", "Alternate account turn"] + : ["First account turn", "Alternate account turn"], + ); + }), ); it.live("finishes earlier queued Codex turns before handing context to queued Claude", () => diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 38d2e98ef72c2..b203ffcd741f6 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -734,6 +734,8 @@ export const OrchestrationV2Run = Schema.Struct({ status: OrchestrationV2RunStatus, queuePosition: Schema.optional(Schema.NullOr(PositiveInt)), queueHeld: Schema.optional(Schema.Boolean), + /** A thread model pick landed while this run was queued, so its start keeps that pick. */ + threadPickSuperseded: Schema.optional(Schema.Boolean), steerWhenReadyRunId: Schema.optional(RunId), requestedAt: Schema.DateTimeUtc, startedAt: Schema.NullOr(Schema.DateTimeUtc),