diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index d1ddd569e60d5..7a1d3e9dd2f8d 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -272,6 +272,7 @@ export const CodexProviderCapabilitiesV2 = { canRollbackThread: true, canForkThread: true, canForkFromTurn: true, + canForkWhileSourceRunning: true, canForkFromSubagentThread: true, exposesNativeThreadId: true, }, diff --git a/apps/server/src/orchestration-v2/CommandPolicy.test.ts b/apps/server/src/orchestration-v2/CommandPolicy.test.ts index 087b24646ef4e..e9f829524948b 100644 --- a/apps/server/src/orchestration-v2/CommandPolicy.test.ts +++ b/apps/server/src/orchestration-v2/CommandPolicy.test.ts @@ -312,6 +312,7 @@ layer("CommandPolicyV2", (it) => { capabilities: CursorProviderCapabilitiesV2, sameProvider: true, hasStrongNativeSource: true, + hasNativeTurnBoundary: false, sourceRunStatus: "completed", fromSpecificTurn: true, sourceBusy: false, @@ -321,29 +322,60 @@ layer("CommandPolicyV2", (it) => { }), ); - it.effect("forks a busy source natively only where the provider supports it", () => - Effect.gen(function* () { - const policy = yield* CommandPolicy.CommandPolicyV2; - const decide = ( - provider: "codex" | "claudeAgent", - capabilities: OrchestrationV2ProviderCapabilities, - sourceBusy: boolean, - ) => - policy.decideForkExecution({ - commandId, - threadId, - providerInstanceId: ProviderInstanceId.make(provider), - capabilities, - sameProvider: true, - hasStrongNativeSource: true, - sourceRunStatus: "completed", - fromSpecificTurn: true, - sourceBusy, - }); + it.effect.each([ + { + provider: "codex", + capabilities: CodexProviderCapabilitiesV2, + hasNativeTurnBoundary: true, + busyStrategy: "native_fork", + }, + { + provider: "codex without busy forks", + capabilities: { + ...CodexProviderCapabilitiesV2, + threads: { ...CodexProviderCapabilitiesV2.threads, canForkWhileSourceRunning: undefined }, + }, + hasNativeTurnBoundary: true, + busyStrategy: "portable_context", + }, + { + provider: "codex", + capabilities: CodexProviderCapabilitiesV2, + hasNativeTurnBoundary: false, + busyStrategy: "portable_context", + }, + { + provider: "claudeAgent", + capabilities: ClaudeProviderCapabilitiesV2, + hasNativeTurnBoundary: false, + busyStrategy: "native_fork", + }, + ] satisfies ReadonlyArray<{ + provider: string; + capabilities: OrchestrationV2ProviderCapabilities; + hasNativeTurnBoundary: boolean; + busyStrategy: CommandPolicy.ForkExecutionPolicyV2; + }>)( + "forks a busy $provider source with hasNativeTurnBoundary=$hasNativeTurnBoundary as $busyStrategy", + ({ capabilities, hasNativeTurnBoundary, busyStrategy }) => + Effect.gen(function* () { + const policy = yield* CommandPolicy.CommandPolicyV2; + const decide = (sourceBusy: boolean) => + policy.decideForkExecution({ + commandId, + threadId, + providerInstanceId: ProviderInstanceId.make("custom-instance"), + capabilities, + sameProvider: true, + hasStrongNativeSource: true, + hasNativeTurnBoundary, + sourceRunStatus: "completed", + fromSpecificTurn: true, + sourceBusy, + }); - assert.equal(yield* decide("codex", CodexProviderCapabilitiesV2, false), "native_fork"); - assert.equal(yield* decide("codex", CodexProviderCapabilitiesV2, true), "portable_context"); - assert.equal(yield* decide("claudeAgent", ClaudeProviderCapabilitiesV2, true), "native_fork"); - }), + assert.equal(yield* decide(false), "native_fork"); + assert.equal(yield* decide(true), busyStrategy); + }), ); }); diff --git a/apps/server/src/orchestration-v2/CommandPolicy.ts b/apps/server/src/orchestration-v2/CommandPolicy.ts index 69c5497d3b876..953ad7b565062 100644 --- a/apps/server/src/orchestration-v2/CommandPolicy.ts +++ b/apps/server/src/orchestration-v2/CommandPolicy.ts @@ -126,6 +126,20 @@ export function isActiveRun(run: OrchestrationV2Run): boolean { ); } +/** + * Providers that key forks on native turn ids can only fork a busy source from a turn with one; + * a synthetic or missing boundary would fork the live conversation, including the running turn. + */ +export function canForkBusySourceNatively( + capabilities: OrchestrationV2ProviderCapabilities, + hasNativeTurnBoundary: boolean, +): boolean { + return ( + capabilities.threads.canForkWhileSourceRunning === true && + (hasNativeTurnBoundary || !capabilities.turns.exposesNativeTurnId) + ); +} + /** The run a follow-up to a finished child binds to: the latest active one, else the last that ran. */ export function followUpOwnerRun( runs: ReadonlyArray, @@ -219,9 +233,10 @@ export interface CommandPolicyV2Shape { input: CapabilityCheckInput & { readonly sameProvider: boolean; readonly hasStrongNativeSource: boolean; + readonly hasNativeTurnBoundary: boolean; readonly sourceRunStatus: OrchestrationV2Run["status"]; readonly fromSpecificTurn: boolean; - /** Later source runs are still active; providers cannot fork a thread mid-turn. */ + /** Later source runs are still active. */ readonly sourceBusy: boolean; }, ) => Effect.Effect; @@ -388,7 +403,8 @@ const decideForkExecution: CommandPolicyV2Shape["decideForkExecution"] = (input) (input.sourceRunStatus === "completed" || input.sourceRunStatus === "waiting") && input.sameProvider && input.hasStrongNativeSource && - (!input.sourceBusy || input.capabilities.threads.canForkWhileSourceRunning === true) && + (!input.sourceBusy || + canForkBusySourceNatively(input.capabilities, input.hasNativeTurnBoundary)) && input.capabilities.threads.canForkThread && (!input.fromSpecificTurn || input.capabilities.threads.canForkFromTurn) && input.capabilities.identity.nativeThreadIds === "strong"; diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 0f6b0e1f20b61..2d0dd5a74e8e0 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -8460,6 +8460,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio sameProvider: pendingForkTransfer.sourceProviderInstanceId === modelSelection.instanceId, hasStrongNativeSource: sourceProviderThread?.nativeThreadRef?.strength === "strong", + hasNativeTurnBoundary: + sourceProjection?.providerTurns.find((turn) => turn.id === sourceProviderTurnId) + ?.nativeTurnRef != null, sourceRunStatus: sourceRun.status, fromSpecificTurn: sourceRun !== null, sourceBusy: sourceRunsAfterFork.some(isActiveRun), diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts index edec4cad58790..b4aeef19f849f 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts @@ -19,6 +19,7 @@ import { RunAttemptId, RunId, ThreadId, + TurnItemId, ProjectId, type OrchestrationV2ThreadProjection, OrchestrationV2DomainEvent, @@ -395,6 +396,7 @@ function makeLocalCommandHarness(input: { readonly steeringRestart?: boolean; readonly nativeGoals?: boolean; readonly canForkWhileSourceRunning?: boolean; + readonly exposesNativeTurnId?: boolean; readonly goalDriver?: "opencode" | "antigravity"; readonly worktreePath?: string; readonly worktreesDir?: string; @@ -436,6 +438,8 @@ function makeLocalCommandHarness(input: { readonly acceptedResume?: boolean; readonly restartedResume?: boolean; readonly nativeForkFrom?: "idle" | "busy"; + readonly sourceBoundary?: "native" | "synthetic"; + readonly sourceRolledBack?: boolean; readonly switchedProvider?: boolean; readonly threadRuntimeMode?: OrchestrationV2ThreadProjection["thread"]["runtimeMode"]; }; @@ -641,11 +645,14 @@ function makeLocalCommandHarness(input: { providerThreadId: parentProviderThread.id, nodeId: rootNodeId, runAttemptId: RunAttemptId.make(`side-chat-parent-attempt-${ordinal}`), - nativeTurnRef: { - driver: providerThread.driver, - nativeId: `side-chat-parent-native-turn-${ordinal}`, - strength: "strong" as const, - }, + nativeTurnRef: + ordinal === 1 && input.sideChat?.sourceBoundary === "synthetic" + ? null + : { + driver: providerThread.driver, + nativeId: `side-chat-parent-native-turn-${ordinal}`, + strength: "strong" as const, + }, ordinal, status: "completed" as const, startedAt: now, @@ -668,7 +675,10 @@ function makeLocalCommandHarness(input: { ordinal: 1, providerThreadId: parentProviderThread.id, activeAttemptId: parentTurns[0]!.attempt.id, - status: "completed" as const, + status: + input.sideChat?.sourceRolledBack === true + ? ("rolled_back" as const) + : ("completed" as const), completedAt: now, }, { @@ -1040,10 +1050,14 @@ function makeLocalCommandHarness(input: { ...CodexProviderCapabilitiesV2, threads: { ...CodexProviderCapabilitiesV2.threads, - canForkWhileSourceRunning: input.canForkWhileSourceRunning, + canForkWhileSourceRunning: + input.canForkWhileSourceRunning ?? + CodexProviderCapabilitiesV2.threads.canForkWhileSourceRunning, }, turns: { ...CodexProviderCapabilitiesV2.turns, + exposesNativeTurnId: + input.exposesNativeTurnId ?? CodexProviderCapabilitiesV2.turns.exposesNativeTurnId, ...(input.nativeGoals === undefined ? {} : { supportsNativeGoals: input.nativeGoals }), }, }, @@ -1074,7 +1088,7 @@ function makeLocalCommandHarness(input: { strength: "strong" as const, }, }; - const forkThread = vi.fn((_fork: ProviderAdapter.ProviderAdapterV2ForkThreadInput) => + const forkThread = vi.fn(() => Effect.succeed(sideChatThread), ); const sideChatSession = { @@ -1197,6 +1211,21 @@ function makeLocalCommandHarness(input: { const getThreadRecords = vi.fn((id) => Effect.succeed(id === parentThreadId ? parentRecords : projection), ); + const prepareProviderHandoff = vi.fn< + ContextHandoffService.ContextHandoffServiceV2["Service"]["prepareProviderHandoff"] + >((handoff) => + input.sideChat?.nativeForkFrom === undefined + ? Effect.die("history read must fail first") + : Effect.succeed({ + ...handoff, + id: ContextHandoffId.make("side-chat-fork-context"), + createdByProviderInstanceId: handoff.toProviderInstanceId, + status: "ready", + summaryMessageId: null, + summaryText: "PARENT_HISTORY", + updatedAt: handoff.createdAt, + }), + ); const getTurnStartContext = vi.fn(() => Effect.succeed({ ...projection, @@ -1236,18 +1265,7 @@ function makeLocalCommandHarness(input: { Layer.provide( Layer.mergeAll( Layer.mock(ContextHandoffService.ContextHandoffServiceV2)({ - prepareProviderHandoff: (handoff) => - input.sideChat?.nativeForkFrom === undefined - ? Effect.die("history read must fail first") - : Effect.succeed({ - ...handoff, - id: ContextHandoffId.make("side-chat-fork-context"), - createdByProviderInstanceId: handoff.toProviderInstanceId, - status: "ready", - summaryMessageId: null, - summaryText: "PARENT_HISTORY", - updatedAt: handoff.createdAt, - }), + prepareProviderHandoff, }), Layer.mock(EventSink.EventSinkV2)({ writeIfRunCurrent, @@ -1342,6 +1360,7 @@ function makeLocalCommandHarness(input: { startRootRun, startTurn, forkThread, + prepareProviderHandoff, interruptTurn, tryHandlePromptCommand, events, @@ -2219,9 +2238,34 @@ effectIt.effect.each([ ); effectIt.effect.each([ - { nativeForkFrom: "idle", forks: true, transferStatus: "consumed", handoffIds: [] }, + { + nativeForkFrom: "idle", + canForkWhileSourceRunning: false, + sourceBoundary: "native", + forks: true, + transferStatus: "consumed", + handoffIds: [], + }, { nativeForkFrom: "busy", + canForkWhileSourceRunning: true, + sourceBoundary: "native", + forks: true, + transferStatus: "consumed", + handoffIds: [], + }, + { + nativeForkFrom: "busy", + canForkWhileSourceRunning: false, + sourceBoundary: "native", + forks: false, + transferStatus: "resolved_portable", + handoffIds: ["side-chat-fork-context"], + }, + { + nativeForkFrom: "busy", + canForkWhileSourceRunning: true, + sourceBoundary: "synthetic", forks: false, transferStatus: "resolved_portable", handoffIds: ["side-chat-fork-context"], @@ -2229,17 +2273,19 @@ effectIt.effect.each([ { nativeForkFrom: "busy", canForkWhileSourceRunning: true, + exposesNativeTurnId: false, + sourceBoundary: "synthetic", forks: true, transferStatus: "consumed", handoffIds: [], }, ] as const)( - "runs a planned native side chat fork when the parent is $nativeForkFrom (fork while running: $canForkWhileSourceRunning)", - ({ nativeForkFrom, forks, transferStatus, handoffIds, ...capability }) => + "runs a planned native side chat fork when the parent is $nativeForkFrom with canForkWhileSourceRunning=$canForkWhileSourceRunning, sourceBoundary=$sourceBoundary and exposesNativeTurnId=$exposesNativeTurnId", + ({ nativeForkFrom, sourceBoundary, forks, transferStatus, handoffIds, ...capability }) => Effect.gen(function* () { const harness = makeLocalCommandHarness({ text: "What is the main thread doing?", - sideChat: { contextRun: "current", nativeForkFrom }, + sideChat: { contextRun: "current", nativeForkFrom, sourceBoundary }, ...capability, }); @@ -2250,6 +2296,11 @@ effectIt.effect.each([ expect( harness.forkThread.mock.calls[0]?.[0].sourceProviderTurns?.map((turn) => turn.id), ).toEqual(forks ? ["side-chat-parent-turn-1"] : undefined); + if (forks) { + expect(harness.forkThread).toHaveBeenCalledWith( + expect.objectContaining({ providerTurnId: "side-chat-parent-turn-1" }), + ); + } expect(harness.projection().contextTransfers).toMatchObject([ { id: "side-chat-fork", status: transferStatus }, ]); @@ -2304,6 +2355,109 @@ effectIt.effect.each( }), ); +effectIt.effect.each(["before-start", "during-fork"] as const)( + "recovers a planned native fork after the parent rolls back %s", + (rollback) => + Effect.gen(function* () { + const harness = makeLocalCommandHarness({ + text: "Continue from the selected turn", + sideChat: { + contextRun: "current", + nativeForkFrom: "idle", + sourceRolledBack: rollback === "before-start", + }, + }); + const parentThreadId = ThreadId.make("side-chat-parent"); + let parent = { + ...harness.projection(), + ...(yield* harness.getThreadRecords(parentThreadId, [ + "runs", + "attempts", + "providerThreads", + "providerTurns", + "turnItems", + ])), + }; + const selectedRun = parent.runs[0]!; + const earlierRun = { + ...selectedRun, + id: RunId.make("side-chat-kept-run"), + ordinal: 1, + activeAttemptId: null, + status: "completed" as const, + }; + const items = [earlierRun, selectedRun].map((sourceRun, ordinal) => ({ + type: "user_message" as const, + id: TurnItemId.make(`parent-item-${ordinal}`), + threadId: parentThreadId, + runId: sourceRun.id, + nodeId: null, + providerThreadId: sourceRun.providerThreadId, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal, + status: "completed" as const, + title: null, + startedAt: parent.updatedAt, + completedAt: parent.updatedAt, + updatedAt: parent.updatedAt, + createdBy: "user" as const, + creationSource: "server" as const, + messageId: MessageId.make(`parent-message-${ordinal}`), + text: ordinal === 0 ? "KEPT_HISTORY" : "ROLLED_BACK_HISTORY", + inputIntent: "turn_start" as const, + attachments: [], + })); + parent = { + ...parent, + runs: [ + earlierRun, + ...parent.runs.map((sourceRun) => ({ ...sourceRun, ordinal: sourceRun.ordinal + 1 })), + ], + turnItems: items, + }; + harness.getThreadRecords.mockImplementation((id) => + Effect.succeed(id === parentThreadId ? parent : harness.projection()), + ); + harness.forkThread.mockImplementation(() => + Effect.sync(() => { + parent = { + ...parent, + runs: parent.runs.map((sourceRun) => + sourceRun.id === selectedRun.id ? { ...sourceRun, status: "rolled_back" } : sourceRun, + ), + }; + }).pipe( + Effect.andThen( + Effect.fail( + new ProviderAdapter.ProviderAdapterForkThreadError({ + driver: ProviderDriverKind.make("codex"), + providerThreadId: ProviderThreadId.make("side-chat-parent-provider-thread"), + cause: "Selected native turn was removed by rollback", + }), + ), + ), + ), + ); + + yield* harness.start; + + expect(harness.forkThread).toHaveBeenCalledTimes(rollback === "before-start" ? 0 : 1); + expect(harness.projection().contextTransfers).toMatchObject([ + { id: "side-chat-fork", status: "resolved_portable" }, + ]); + expect(harness.startTurn.mock.calls[0]?.[0].message.text).toContain("PARENT_HISTORY"); + expect(harness.projection().providerThreads[0]?.handoffIds).toEqual([ + "side-chat-fork-context", + ]); + expect(harness.settledRuns).toEqual([]); + expect( + harness.prepareProviderHandoff.mock.calls[0]?.[0].items.map((item) => item.id), + ).toEqual([items[0]!.id]); + }), +); + effectIt.effect.each([ { runStatus: "failed", turnStatus: "completed" }, { runStatus: "completed", turnStatus: "completed" }, diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 7e6fe6a206c64..c37b55cc29adb 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -43,7 +43,7 @@ import { makeRestoreMissingWorktree } from "../workspace/restoreMissingWorktree. import * as EventSink from "./EventSink.ts"; import * as ProviderUsageAnalytics from "../telemetry/ProviderUsageAnalytics.ts"; import * as ContextHandoffService from "./ContextHandoffService.ts"; -import { isActiveRun } from "./CommandPolicy.ts"; +import { canForkBusySourceNatively, isActiveRun } from "./CommandPolicy.ts"; import { contextBoundaryOrdinal } from "./contextBoundary.ts"; import { DEFAULT_HANDOFF_TOKEN_CAP, @@ -1187,7 +1187,7 @@ export const layer: Layer.Layer< let recoveryTurnItem: OrchestrationV2TurnItem | undefined; const loadedProviderThread = yield* Effect.gen(function* () { if (nativeForkTransfer !== undefined) { - const sourceProjection = yield* projectionStore.getThreadRecords( + let sourceProjection = yield* projectionStore.getThreadRecords( nativeForkTransfer.sourceThreadId, ["runs", "providerThreads", "attempts", "providerTurns"], ); @@ -1211,15 +1211,19 @@ export const layer: Layer.Layer< cause: `Native fork transfer ${nativeForkTransfer.id} has no source provider execution.`, }); } - // Most providers refuse to fork a busy thread, so a source that started a turn after - // dispatch planned this fork gets the portable context dispatch gives a busy source. - const sourceBusy = - session.providerSession.capabilities.threads.canForkWhileSourceRunning !== true && - sourceProjection.runs.some( - (candidate) => candidate.ordinal > sourceRun.ordinal && isActiveRun(candidate), - ); + // The source may have started a turn after dispatch planned this fork. + const sourceBusy = sourceProjection.runs.some( + (candidate) => candidate.ordinal > sourceRun.ordinal && isActiveRun(candidate), + ); let nativeForkFailure: string | undefined; - if (!sourceBusy) { + if ( + (sourceRun.status === "completed" || sourceRun.status === "waiting") && + (!sourceBusy || + canForkBusySourceNatively( + session.providerSession.capabilities, + sourceProviderTurn?.nativeTurnRef != null, + )) + ) { // Rolled-back turns are no longer in the source conversation, so they cannot bound a fork. const rolledBackAttemptIds = new Set( sourceProjection.attempts @@ -1262,6 +1266,13 @@ export const layer: Layer.Layer< message: startFailureMessage(forked.failure), class: "provider_error", }).message; + sourceProjection = { + ...sourceProjection, + ...(yield* projectionStore.getThreadRecords(nativeForkTransfer.sourceThreadId, [ + "runs", + "attempts", + ])), + }; } const nativeForkFailed = nativeForkFailure !== undefined; const sourceItems = yield* ContextHandoffService.readForkHistoryItems(