From 02a2479673db5ac04e4ca446eb0d1f637993f2e0 Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Mon, 26 Oct 2026 15:07:17 -0500 Subject: [PATCH 1/6] fix(server): retry Codex capacity failures with empty input - Continue failed overloaded turns from persisted history, with bounded backoff - Keep retries interruptible and preserve one user message across continuations --- .../Adapters/CodexAdapterV2.test.ts | 334 ++++++++++++++++++ .../Adapters/CodexAdapterV2.ts | 296 ++++++++++++---- 2 files changed, 569 insertions(+), 61 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index 602e1a4d56f29..5d2e29365623f 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -2214,6 +2214,340 @@ describe("CodexAdapterV2 post-settle continuation", () => { ), ); + it.effect.each([ + "recover", + "completion-only", + "exhaust", + "stop", + "stop-continued", + "steer", + "start-fails", + ] as const)("handles terminal Codex capacity failures when retries %s", (outcome) => + Effect.scoped( + Effect.gen(function* () { + const scenario = `capacity-${outcome}`; + const nativeThreadId = `native-${scenario}`; + const nativeTurnId = (attempt: number) => `${nativeThreadId}-${attempt}`; + const prompt = "Continue the task."; + const capacityError = { + message: "Selected model is at capacity. Please try a different model.", + codexErrorInfo: "serverOverloaded", + }; + const failureEntries = (attempt: number): Array => [ + { + type: "emit_inbound", + frame: { + method: "error", + params: { + threadId: nativeThreadId, + turnId: nativeTurnId(attempt), + willRetry: false, + error: capacityError, + }, + }, + }, + { + type: "emit_inbound", + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: { + ...makeCodexReplayTurn({ id: nativeTurnId(attempt), status: "failed" }), + error: capacityError, + }, + }, + }, + }, + ]; + const entries = codexReplayPreamble({ + nativeThreadId, + nativeTurnId: nativeTurnId(0), + prompt, + }); + entries.push( + ...(outcome === "completion-only" ? failureEntries(0).slice(1) : failureEntries(0)), + ); + if (outcome === "completion-only") entries.push(...failureEntries(0).slice(1)); + const retries = outcome === "exhaust" ? 5 : outcome === "stop" ? 0 : 1; + const steeredText = "Use the smaller change."; + for (let attempt = 1; attempt <= retries; attempt++) { + const start = codexReplayPreamble({ + nativeThreadId, + nativeTurnId: nativeTurnId(attempt), + prompt, + }).slice(-3); + for (const entry of start) { + if (entry.type === "expect_outbound") { + entries.push({ + ...entry, + frame: { + id: 3 + attempt, + method: "turn/start", + params: { + threadId: nativeThreadId, + input: [], + cwd: "/workspace", + model: "gpt-5.4", + approvalPolicy: "never", + approvalsReviewer: "user", + sandboxPolicy: { type: "dangerFullAccess" }, + summary: "detailed", + collaborationMode: { mode: "default", settings: { model: "gpt-5.4" } }, + }, + }, + }); + } else if (entry.type === "emit_inbound" && entry.label === "turn/start") { + entries.push( + outcome === "start-fails" + ? { + type: "emit_inbound", + frame: { + id: 3 + attempt, + error: { code: -32603, message: "Start failed" }, + }, + } + : withReplayRequestId(entry, 3 + attempt), + ); + } else if (outcome !== "start-fails") { + entries.push(entry); + } + } + if (outcome === "exhaust") entries.push(...failureEntries(attempt)); + if (outcome === "recover" || outcome === "completion-only") { + entries.push({ + type: "emit_inbound", + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: nativeTurnId(attempt), status: "completed" }), + }, + }, + }); + } + if (outcome === "stop-continued" || outcome === "steer") { + const steering = outcome === "steer"; + entries.push( + { + type: "expect_outbound", + frame: { + id: 5, + method: steering ? "turn/steer" : "turn/interrupt", + params: steering + ? { + threadId: nativeThreadId, + expectedTurnId: nativeTurnId(attempt), + input: [{ type: "text", text: steeredText }], + } + : { threadId: nativeThreadId, turnId: nativeTurnId(attempt) }, + }, + }, + { + type: "emit_inbound", + frame: { id: 5, result: steering ? { turnId: nativeTurnId(attempt) } : {} }, + }, + { + type: "emit_inbound", + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ + id: nativeTurnId(attempt), + status: steering ? "completed" : "interrupted", + }), + }, + }, + }, + ); + } + } + const milestones = yield* Queue.unbounded(); + const starts: Array = []; + const harness = yield* makeCodexReplayHarness( + makeCodexReplayTranscript({ scenario, entries }), + (event) => + event.type === "turn.terminal" || + (event.type === "turn_item.updated" && + event.turnItem.type === "error" && + event.turnItem.status === "running") + ? Queue.offer(milestones, event) + : Effect.void, + (method, params) => + Effect.sync(() => { + if (method === "turn/start") starts.push(params); + }), + ); + const turnInput = makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make(scenario), + text: prompt, + }); + yield* harness.runtime.startTurn(turnInput); + for (let attempt = 1; attempt <= Math.max(retries, 1); attempt++) { + const milestone = yield* Queue.take(milestones); + assert.equal(milestone.type, "turn_item.updated"); + if (milestone.type !== "turn_item.updated" || milestone.turnItem.type !== "error") return; + const providerTurnId = milestone.turnItem.providerTurnId; + assert.isNotNull(providerTurnId); + if (providerTurnId === null) return; + assert.deepEqual(milestone.turnItem.retry, { + attempt, + maxAttempts: 5, + retryDelayMs: Math.min(2_000 * 2 ** (attempt - 1), 30_000), + }); + assert.isEmpty(harness.terminalEvents()); + if (outcome === "stop") { + yield* harness.runtime.interruptTurn({ + providerThread: harness.providerThread, + providerTurnId, + }); + } + const steering = + outcome === "steer" + ? yield* harness.runtime + .steerTurn({ + ...turnInput, + providerTurnId, + message: { ...turnInput.message, text: steeredText }, + }) + .pipe(Effect.forkScoped) + : undefined; + yield* TestClock.adjust(Math.min(2_000 * 2 ** (attempt - 1), 30_000)); + if (steering !== undefined) yield* Fiber.join(steering); + if (outcome === "stop-continued") { + yield* harness.until(() => + harness.events.some( + (event) => + event.type === "provider_turn.updated" && + event.providerTurn.nativeTurnRef?.nativeId === nativeTurnId(1), + ), + ); + yield* harness.runtime.interruptTurn({ + providerThread: harness.providerThread, + providerTurnId, + }); + } + } + yield* harness.awaitTerminal(1); + const terminal = harness.terminalEvents()[0]; + assert.equal( + terminal?.status, + outcome === "recover" || outcome === "completion-only" || outcome === "steer" + ? "completed" + : outcome === "stop" || outcome === "stop-continued" + ? "interrupted" + : "failed", + ); + assert.lengthOf(starts, 1 + retries); + assert.lengthOf(harness.terminalEvents(), 1); + const retryItems = harness.events.flatMap((event) => + event.type === "turn_item.updated" && event.turnItem.type === "error" + ? [event.turnItem] + : [], + ); + assert.isTrue( + ["completed", "failed", "interrupted"].includes(retryItems.at(-1)?.status ?? ""), + ); + assert.lengthOf(new Set(retryItems.map((item) => item.id)), 1); + + if (outcome === "recover") { + const fs = yield* FileSystem.FileSystem; + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-capacity-retry-" }); + const localTranscript = yield* decodeReplayTranscriptJson( + (yield* encodeReplayTranscriptJson( + makeCodexReplayTranscript({ scenario, entries }), + )).replaceAll(yield* encodeStringJson("/workspace"), yield* encodeStringJson(cwd)), + ); + const driver = yield* CodexReplay.makeReplayDriver(localTranscript); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const threadId = ThreadId.make("thread:capacity-retry"); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("create-capacity-retry"), + threadId, + projectId: ProjectId.make("project:capacity-retry"), + title: "Capacity retry", + modelSelection: CODEX_TEST_MODEL_SELECTION, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: cwd, + createdBy: "user", + creationSource: "web", + }); + const retrying = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "turn-item.updated" && + event.payload.type === "error" && + event.payload.retry != null && + event.payload.status === "running", + ), + Stream.runHead, + Effect.forkChild({ startImmediately: true }), + ); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("start-capacity-retry"), + threadId, + messageId: MessageId.make("message:capacity-retry"), + text: prompt, + attachments: [], + createdBy: "user", + creationSource: "web", + dispatchMode: { type: "start_immediately" }, + }); + yield* worker.drain(); + yield* Fiber.join(retrying); + assert.equal( + (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)?.status, + "running", + ); + const completed = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => event.type === "run.updated" && event.payload.status === "waiting", + ), + Stream.runHead, + Effect.forkChild({ startImmediately: true }), + ); + yield* TestClock.adjust("2 seconds"); + yield* Fiber.join(completed); + yield* worker.drain(); + const projection = yield* orchestrator.getThreadProjection(threadId); + assert.equal(projection.runs.at(-1)?.status, "completed"); + assert.deepEqual( + projection.providerTurns.map((turn) => turn.status), + ["failed", "completed"], + ); + assert.lengthOf( + projection.messages.filter((message) => message.role === "user"), + 1, + ); + assert.isNull((yield* Ref.get(driver.state)).failure); + }).pipe( + Effect.provide( + ProviderReplayHarness.layerWithRegistry( + { name: "codex-capacity-retry", runtimePolicyOverride: { cwd } }, + CodexAdapterV2Testkit.layer({ transcript: localTranscript, driver }), + { runEffectWorker: false }, + ), + ), + ); + } + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + it.effect("resolves retryable app-server errors on resumed provider activity", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 6dba639b63b01..2c9cdcb97fb50 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -86,6 +86,7 @@ import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; +import * as Function from "effect/Function"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; @@ -152,6 +153,7 @@ import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; const CODEX_PROVIDER = ProviderDriverKind.make("codex"); +const CODEX_CAPACITY_MAX_RETRIES = 5; export const CODEX_DRIVER_KIND = CODEX_PROVIDER; export const CODEX_DEFAULT_INSTANCE_ID = defaultInstanceIdForDriver(CODEX_DRIVER_KIND); @@ -1109,6 +1111,8 @@ function withCodexManagedFailure( } interface ActiveCodexTurnContext { + capacityRetryAttempt?: number; + capacityRetryPending?: Deferred.Deferred; closedCommandItemIds?: ReadonlySet; latestProviderFailure?: { readonly nativeMessage: string; @@ -1141,6 +1145,7 @@ interface ActiveCodexTurnContext { } interface ActiveCodexProviderRetry { + readonly context?: ActiveCodexTurnContext; readonly nativeMessage: string; readonly nativeCode: string | null; readonly retry: OrchestrationV2ProviderRetry; @@ -1149,6 +1154,30 @@ interface ActiveCodexProviderRetry { readonly itemOrdinal: number; } +function codexRetryTurnItem( + activeContext: ActiveCodexTurnContext, + state: ActiveCodexProviderRetry, + status: Parameters[0]["status"], + updatedAt: DateTime.Utc, +) { + // A continuation keeps the retry row attached to the native turn that failed. + const context = state.context ?? activeContext; + return makeProviderRetryTurnItem({ + driver: CODEX_PROVIDER, + threadId: context.projectionThreadId, + runId: context.projectionRunId, + nodeId: context.providerNodeId, + providerThreadId: context.providerThread.id, + providerTurnId: context.providerTurnId, + itemOrdinal: state.itemOrdinal, + failure: state.failure, + retry: state.retry, + status, + startedAt: state.startedAt, + updatedAt, + }); +} + interface TrackedRunningCommandItem { readonly id: string; readonly command: string; @@ -1815,12 +1844,16 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( const goalsByNativeThread = new Map(); const rootProviderThreads = new Map(); const goalHolds = new Map(); - const goalRuns = new Map>(); + const continuedRuns = new Map>(); + const capacityContinuations = new Map< + string, + { readonly context: ActiveCodexTurnContext; readonly retry: ActiveCodexProviderRetry } + >(); const goalActivations = new Map(); // Held runs must not complete while Stop resolves its target. const goalStops = new Map(); - const latestGoalTurnId = (providerTurnId: ProviderTurnId) => - goalRuns.get(providerTurnId)?.at(-1)?.providerTurnId ?? providerTurnId; + const latestRunTurnId = (providerTurnId: ProviderTurnId) => + continuedRuns.get(providerTurnId)?.at(-1)?.providerTurnId ?? providerTurnId; const emitProviderEvent = (event: ProviderAdapter.ProviderAdapterV2Event) => Queue.offer(events, event).pipe(Effect.asVoid); @@ -1882,20 +1915,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( yield* emitProviderEvent({ type: "turn_item.updated", driver: CODEX_PROVIDER, - turnItem: makeProviderRetryTurnItem({ - driver: CODEX_PROVIDER, - threadId: context.projectionThreadId, - runId: context.projectionRunId, - nodeId: context.providerNodeId, - providerThreadId: context.providerThread.id, - providerTurnId: context.providerTurnId, - itemOrdinal: providerRetry.itemOrdinal, - failure: providerRetry.failure, - retry: providerRetry.retry, - status: "completed", - startedAt: providerRetry.startedAt, - updatedAt, - }), + turnItem: codexRetryTurnItem(context, providerRetry, "completed", updatedAt), }); }); @@ -1934,6 +1954,24 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( fileChangeKindsByItem: new Map(), runningCompactionItemIds: new Set(), }; + const nativeThreadId = input.turnInput.providerThread.nativeThreadRef?.nativeId; + const continuation = + nativeThreadId == null ? undefined : capacityContinuations.get(nativeThreadId); + if (continuation !== undefined) { + context.capacityRetryAttempt = continuation.context.capacityRetryAttempt ?? 0; + yield* Ref.update(providerRetries, (current) => + new Map(current).set(providerTurnId, continuation.retry), + ); + const turns = continuedRuns.get(continuation.context.providerTurnId) ?? [ + continuation.context, + ]; + turns.push(context); + continuedRuns.set(continuation.context.providerTurnId, turns); + continuedRuns.set(providerTurnId, turns); + if (continuation.context.capacityRetryPending !== undefined) { + yield* Deferred.succeed(continuation.context.capacityRetryPending, context); + } + } yield* Ref.update(limitedTurnItems, (current) => { const next = new Map(current); next.delete(context.providerThread.id); @@ -4380,10 +4418,12 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( nativeTurnId: payload.turn.id, startedAt: codexTimestamp(payload.turn.startedAt), }); - const goalRun = goalRuns.get(goalHold.context.providerTurnId) ?? [goalHold.context]; + const goalRun = continuedRuns.get(goalHold.context.providerTurnId) ?? [ + goalHold.context, + ]; goalRun.push(next); - goalRuns.set(goalHold.context.providerTurnId, goalRun); - goalRuns.set(next.providerTurnId, goalRun); + continuedRuns.set(goalHold.context.providerTurnId, goalRun); + continuedRuns.set(next.providerTurnId, goalRun); yield* Deferred.succeed(goalHold.next, next); return; } @@ -4493,6 +4533,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( previous?.itemOrdinal ?? (yield* resolveItemOrdinal(context, `terminal-failure:${context.providerTurnId}`)); const state: ActiveCodexProviderRetry = { + ...(previous?.context === undefined ? {} : { context: previous.context }), nativeMessage: payload.error.message, nativeCode: code, retry, @@ -4508,20 +4549,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( yield* emitProviderEvent({ type: "turn_item.updated", driver: CODEX_PROVIDER, - turnItem: makeProviderRetryTurnItem({ - driver: CODEX_PROVIDER, - threadId: context.projectionThreadId, - runId: context.projectionRunId, - nodeId: context.providerNodeId, - providerThreadId: context.providerThread.id, - providerTurnId: context.providerTurnId, - itemOrdinal, - failure, - retry, - status: "running", - startedAt: state.startedAt, - updatedAt, - }), + turnItem: codexRetryTurnItem(context, state, "running", updatedAt), }); }).pipe(Effect.orDie), ); @@ -5617,8 +5645,8 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( } : event; yield* emitProviderEvent(current); - for (const turn of goalRuns.get(context.providerTurnId) ?? []) { - goalRuns.delete(turn.providerTurnId); + for (const turn of continuedRuns.get(context.providerTurnId) ?? []) { + continuedRuns.delete(turn.providerTurnId); } const nativeThreadId = context.providerThread.nativeThreadRef?.nativeId; const rootProviderThread = @@ -5803,8 +5831,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( readonly completedAt: DateTime.Utc; readonly failureMessage?: string; readonly failureErrorInfo?: CodexErrorInfo; + readonly continueRun?: boolean; }) => - turnTerminalizationPermit.withPermits(1)( + // Capacity continuation already holds the permit across finalization and restart. + (input.continueRun ? Function.identity : turnTerminalizationPermit.withPermits(1))( Effect.gen(function* () { const current = (yield* Ref.get(activeTurns)).get(input.nativeTurnId); if (current !== input.context) { @@ -5822,25 +5852,24 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }); if ( providerRetry !== undefined && - (input.status !== "failed" || input.context.subagent !== null) + !input.continueRun && + (providerRetry.context !== undefined || + input.status !== "failed" || + input.context.subagent !== null) ) { yield* emitProviderEvent({ type: "turn_item.updated", driver: CODEX_PROVIDER, - turnItem: makeProviderRetryTurnItem({ - driver: CODEX_PROVIDER, - threadId: input.context.projectionThreadId, - runId: input.context.projectionRunId, - nodeId: input.context.providerNodeId, - providerThreadId: input.context.providerThread.id, - providerTurnId: input.context.providerTurnId, - itemOrdinal: providerRetry.itemOrdinal, - failure: providerRetry.failure, - retry: providerRetry.retry, - status: providerTurnStatusToTerminal(input.status), - startedAt: providerRetry.startedAt, - updatedAt: input.completedAt, - }), + turnItem: codexRetryTurnItem( + input.context, + providerRetry, + input.status === "failed" && + providerRetry.context !== input.context && + providerRetry.context !== undefined + ? "interrupted" + : providerTurnStatusToTerminal(input.status), + input.completedAt, + ), }); } if (input.status !== "completed") { @@ -6017,7 +6046,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( input.completedAt, input.status === "interrupted" || input.status === "failed", ); - if (input.context.subagent === null) { + if (input.context.subagent === null && !input.continueRun) { yield* emitOrDeferRootTerminal({ ...input, ...(providerRetry === undefined ? {} : { providerRetry }), @@ -6051,6 +6080,9 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( if (input.context.nativeStartReady !== undefined) { yield* Deferred.succeed(input.context.nativeStartReady, undefined); } + if (!input.continueRun && input.context.capacityRetryPending !== undefined) { + yield* Deferred.succeed(input.context.capacityRetryPending, undefined); + } yield* flushReadyRootTerminals(); if (!retainSettledContext && !interruptInProgress) { yield* Ref.update(runningCommandItemsByTurn, (current) => { @@ -6092,6 +6124,135 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }), ); + // Codex can end an overloaded turn without retry advice. Continue its + // persisted history with empty input instead of resending the user message. + const retryCapacityFailure = Effect.fn("CodexAdapterV2.retryCapacityFailure")(function* ( + terminal: Parameters[0], + ) { + const context = terminal.context; + const code = + terminal.failureErrorInfo == null + ? context.latestProviderFailure?.nativeCode + : codexErrorInfoCode(terminal.failureErrorInfo); + if ( + terminal.status !== "failed" || + code !== "serverOverloaded" || + context.subagent !== null || + (context.input.message.attachments.length === 0 && + context.input.message.text.trim().toLowerCase() === "/compact") + ) + return false; + + return yield* turnTerminalizationPermit.withPermits(1)( + Effect.gen(function* () { + if ((yield* Ref.get(activeTurns)).get(terminal.nativeTurnId) !== context) return true; + if (context.capacityRetryPending !== undefined) return true; + if ( + (context.capacityRetryAttempt ?? 0) >= CODEX_CAPACITY_MAX_RETRIES || + (yield* Ref.get(interruptingNativeTurns)).has(terminal.nativeTurnId) + ) + return false; + + const attempt = (context.capacityRetryAttempt ?? 0) + 1; + context.capacityRetryAttempt = attempt; + context.capacityRetryPending = yield* Deferred.make< + ActiveCodexTurnContext | undefined + >(); + const retryDelayMs = Math.min(2_000 * 2 ** (attempt - 1), 30_000); + const now = yield* DateTime.now; + const previous = (yield* Ref.get(providerRetries)).get(context.providerTurnId); + const state: ActiveCodexProviderRetry = { + context: previous?.context ?? context, + nativeMessage: + terminal.failureMessage ?? + context.latestProviderFailure?.nativeMessage ?? + "Selected model is at capacity.", + nativeCode: code, + failure: makeProviderFailure({ + message: + context.latestProviderFailure?.failure.message ?? terminal.failureMessage, + code, + class: "provider_error", + retryable: true, + }), + retry: { attempt, maxAttempts: CODEX_CAPACITY_MAX_RETRIES, retryDelayMs }, + itemOrdinal: + previous?.itemOrdinal ?? + (yield* resolveItemOrdinal( + context, + `terminal-failure:${context.providerTurnId}`, + )), + startedAt: previous?.startedAt ?? now, + }; + yield* Ref.update(providerRetries, (current) => + new Map(current).set(context.providerTurnId, state), + ); + yield* emitProviderEvent({ + type: "turn_item.updated", + driver: CODEX_PROVIDER, + turnItem: codexRetryTurnItem(context, state, "running", now), + }); + yield* Effect.sleep(retryDelayMs).pipe( + Effect.andThen( + turnTerminalizationPermit.withPermits(1)( + Effect.gen(function* () { + if ( + (yield* Ref.get(activeTurns)).get(terminal.nativeTurnId) !== context || + (yield* Ref.get(interruptingNativeTurns)).has(terminal.nativeTurnId) + ) + return; + const nativeThreadId = yield* getNativeThreadId(context.providerThread); + capacityContinuations.set(nativeThreadId, { context, retry: state }); + yield* finalizeCodexTurn({ ...terminal, continueRun: true }); + yield* startNativeTurn( + { + ...context.input, + providerTurnOrdinal: context.providerTurnOrdinal + 1, + }, + [], + ).pipe( + Effect.catch((cause) => + Effect.gen(function* () { + yield* Effect.logWarning( + "orchestration-v2.codex-capacity-retry-start-failed", + { + providerTurnId: context.providerTurnId, + cause, + }, + ); + yield* emitProviderEvent({ + type: "turn_item.updated", + driver: CODEX_PROVIDER, + turnItem: codexRetryTurnItem( + context, + state, + "failed", + yield* DateTime.now, + ), + }); + if (context.capacityRetryPending !== undefined) { + yield* Deferred.succeed(context.capacityRetryPending, undefined); + } + yield* emitRootTerminal( + context, + yield* makeRootTerminalEvent({ ...terminal, providerRetry: state }), + ); + }), + ), + Effect.ensuring( + Effect.sync(() => capacityContinuations.delete(nativeThreadId)), + ), + ); + }), + ), + ), + Effect.forkIn(scope, { startImmediately: true }), + ); + return true; + }), + ); + }); + yield* client.handleServerNotification("turn/completed", (payload) => Effect.gen(function* () { const context = (yield* Ref.get(activeTurns)).get(payload.turn.id); @@ -6104,7 +6265,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( (yield* Ref.get(interruptingNativeTurns)).has(payload.turn.id) ? "interrupted" : nativeStatus; - yield* finalizeCodexTurn({ + const terminal = { context, nativeTurnId: payload.turn.id, status, @@ -6117,7 +6278,8 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( ? {} : { failureErrorInfo: payload.turn.error.codexErrorInfo }), }), - }); + }; + if (!(yield* retryCapacityFailure(terminal))) yield* finalizeCodexTurn(terminal); }), ); @@ -6231,10 +6393,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( goalStops.set(goalThreadId, (goalStops.get(goalThreadId) ?? 0) + 1); return yield* Effect.gen(function* () { // The run can settle while the pause is in flight, which drops its record. - const requestedTurnId = latestGoalTurnId(turnInput.providerTurnId); + const requestedTurnId = latestRunTurnId(turnInput.providerTurnId); const heldBefore = goalHolds.get(goalThreadId)?.context; const turnsBefore = [ - ...(goalRuns.get(requestedTurnId) ?? []), + ...(continuedRuns.get(requestedTurnId) ?? []), ...Array.from((yield* Ref.get(activeTurns)).values()).filter( (context) => context.providerTurnId === requestedTurnId, ), @@ -6258,13 +6420,13 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( } return yield* turnTerminalizationPermit.withPermits(1)( Effect.gen(function* () { - const providerTurnId = latestGoalTurnId(turnInput.providerTurnId); + const providerTurnId = latestRunTurnId(turnInput.providerTurnId); const hold = goalHolds.get(goalThreadId); const held = hold !== undefined && hold.context.providerTurnId === providerTurnId; const goalTurns: ReadonlyArray = Array.from( new Set([ ...turnsBefore, - ...(goalRuns.get(providerTurnId) ?? []), + ...(continuedRuns.get(providerTurnId) ?? []), ...(held ? [hold.context] : []), ]), ); @@ -6764,15 +6926,18 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( steerTurn: Effect.fn("CodexAdapterV2.steerTurn")( function* (turnInput) { const threadId = yield* getNativeThreadId(turnInput.providerThread); - const latestTurnId = latestGoalTurnId(turnInput.providerTurnId); + const latestTurnId = latestRunTurnId(turnInput.providerTurnId); const held = heldGoalTurn(turnInput.providerThread, latestTurnId); const providerTurnId = held === undefined ? latestTurnId : ((yield* Deferred.await(held.hold.next))?.providerTurnId ?? latestTurnId); - const activeTurn = Array.from((yield* Ref.get(activeTurns)).values()).find( + let activeTurn = Array.from((yield* Ref.get(activeTurns)).values()).find( (candidate) => candidate.providerTurnId === providerTurnId, ); + while (activeTurn?.capacityRetryPending !== undefined) { + activeTurn = yield* Deferred.await(activeTurn.capacityRetryPending); + } if (activeTurn === undefined) { return yield* toProtocolError( `Provider turn ${turnInput.providerTurnId} is not active and cannot be steered.`, @@ -7053,6 +7218,15 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( yield* Effect.gen(function* () { for (const target of interruptTargets) { const context = target.context; + if (context.capacityRetryPending !== undefined) { + yield* finalizeCodexTurn({ + context, + nativeTurnId: context.nativeTurnId, + status: "interrupted", + completedAt: yield* DateTime.now, + }); + continue; + } // Null start timestamps ACK queued turns; interruption must wait for turn/started. if (context.nativeStartReady !== undefined) { const ready = yield* Deferred.await(context.nativeStartReady).pipe( From 9819538c11282098a942177206ce0309930be9b0 Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Mon, 26 Oct 2026 16:54:20 -0500 Subject: [PATCH 2/6] fix(server): keep Codex capacity retry startup responsive --- .../Adapters/CodexAdapterV2.test.ts | 208 +++++++++++++++++- .../Adapters/CodexAdapterV2.ts | 150 ++++++++----- 2 files changed, 299 insertions(+), 59 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index 5d2e29365623f..8b76955aeebe8 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -2220,6 +2220,9 @@ describe("CodexAdapterV2 post-settle continuation", () => { "exhaust", "stop", "stop-continued", + "stop-starting", + "started-first", + "completed-first", "steer", "start-fails", ] as const)("handles terminal Codex capacity failures when retries %s", (outcome) => @@ -2277,6 +2280,13 @@ describe("CodexAdapterV2 post-settle continuation", () => { nativeTurnId: nativeTurnId(attempt), prompt, }).slice(-3); + if ( + outcome === "started-first" || + outcome === "stop-starting" || + outcome === "completed-first" + ) { + start.splice(1, 0, ...start.splice(2, 1)); + } for (const entry of start) { if (entry.type === "expect_outbound") { entries.push({ @@ -2298,6 +2308,17 @@ describe("CodexAdapterV2 post-settle continuation", () => { }, }); } else if (entry.type === "emit_inbound" && entry.label === "turn/start") { + if (outcome === "completed-first") + entries.push({ + type: "emit_inbound", + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: nativeTurnId(attempt), status: "completed" }), + }, + }, + }); entries.push( outcome === "start-fails" ? { @@ -2310,18 +2331,43 @@ describe("CodexAdapterV2 post-settle continuation", () => { : withReplayRequestId(entry, 3 + attempt), ); } else if (outcome !== "start-fails") { - entries.push(entry); + entries.push( + outcome === "stop-starting" && entry.type === "emit_inbound" + ? { ...entry, afterMs: 1_000 } + : entry, + ); + if (outcome === "stop-starting") { + entries.push( + { + type: "expect_outbound", + frame: { + id: 5, + method: "turn/interrupt", + params: { threadId: nativeThreadId, turnId: nativeTurnId(attempt) }, + }, + }, + { type: "emit_inbound", frame: { id: 5, result: {} } }, + ); + } } } if (outcome === "exhaust") entries.push(...failureEntries(attempt)); - if (outcome === "recover" || outcome === "completion-only") { + if ( + outcome === "recover" || + outcome === "completion-only" || + outcome === "started-first" || + outcome === "stop-starting" + ) { entries.push({ type: "emit_inbound", frame: { method: "turn/completed", params: { threadId: nativeThreadId, - turn: makeCodexReplayTurn({ id: nativeTurnId(attempt), status: "completed" }), + turn: makeCodexReplayTurn({ + id: nativeTurnId(attempt), + status: outcome === "stop-starting" ? "interrupted" : "completed", + }), }, }, }); @@ -2365,6 +2411,7 @@ describe("CodexAdapterV2 post-settle continuation", () => { } const milestones = yield* Queue.unbounded(); const starts: Array = []; + const retryStarted = yield* Deferred.make(); const harness = yield* makeCodexReplayHarness( makeCodexReplayTranscript({ scenario, entries }), (event) => @@ -2375,8 +2422,11 @@ describe("CodexAdapterV2 post-settle continuation", () => { ? Queue.offer(milestones, event) : Effect.void, (method, params) => - Effect.sync(() => { - if (method === "turn/start") starts.push(params); + Effect.gen(function* () { + if (method === "turn/start") { + starts.push(params); + if (starts.length === 2) yield* Deferred.succeed(retryStarted, undefined); + } }), ); const turnInput = makeCodexTestTurnInput({ @@ -2417,6 +2467,14 @@ describe("CodexAdapterV2 post-settle continuation", () => { .pipe(Effect.forkScoped) : undefined; yield* TestClock.adjust(Math.min(2_000 * 2 ** (attempt - 1), 30_000)); + if (outcome === "stop-starting") { + yield* Deferred.await(retryStarted); + const stopping = yield* harness.runtime + .interruptTurn({ providerThread: harness.providerThread, providerTurnId }) + .pipe(Effect.forkScoped); + yield* TestClock.adjust("1 second"); + yield* Fiber.join(stopping); + } if (steering !== undefined) yield* Fiber.join(steering); if (outcome === "stop-continued") { yield* harness.until(() => @@ -2436,14 +2494,25 @@ describe("CodexAdapterV2 post-settle continuation", () => { const terminal = harness.terminalEvents()[0]; assert.equal( terminal?.status, - outcome === "recover" || outcome === "completion-only" || outcome === "steer" + outcome === "recover" || + outcome === "completion-only" || + outcome === "steer" || + outcome === "started-first" || + outcome === "completed-first" ? "completed" - : outcome === "stop" || outcome === "stop-continued" + : outcome === "stop" || outcome === "stop-continued" || outcome === "stop-starting" ? "interrupted" : "failed", ); assert.lengthOf(starts, 1 + retries); assert.lengthOf(harness.terminalEvents(), 1); + assert.lengthOf( + harness.events.filter( + (event) => + event.type === "provider_turn.updated" && event.providerTurn.status === "running", + ), + outcome === "start-fails" ? 1 : 1 + retries, + ); const retryItems = harness.events.flatMap((event) => event.type === "turn_item.updated" && event.turnItem.type === "error" ? [event.turnItem] @@ -5341,6 +5410,131 @@ describe("CodexAdapterV2 post-settle continuation", () => { }; }; + it.effect.each(["completed", "failed"] as const)( + "handles a late native child after the capacity retry %s", + (parentStatus) => + Effect.scoped( + Effect.gen(function* () { + const retriedTurnId = "native-capacity-recovered-parent"; + const retrying = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + makeCodexReplayTranscript({ + scenario: "capacity-late-child", + entries: [ + ...lateChildPrefix().spawned, + { + type: "emit_inbound", + frame: { + method: "turn/completed", + params: { + threadId: RESUME_NATIVE_THREAD, + turn: { + ...makeCodexReplayTurn({ id: RESUME_NATIVE_TURN, status: "failed" }), + error: { + message: "Selected model is at capacity.", + codexErrorInfo: "serverOverloaded", + }, + }, + }, + }, + }, + { + type: "expect_outbound", + frame: { + id: 4, + method: "turn/start", + params: { + threadId: RESUME_NATIVE_THREAD, + input: [], + cwd: "/workspace", + model: "gpt-5.4", + approvalPolicy: "never", + approvalsReviewer: "user", + sandboxPolicy: { type: "dangerFullAccess" }, + summary: "detailed", + collaborationMode: { mode: "default", settings: { model: "gpt-5.4" } }, + }, + }, + }, + { + type: "emit_inbound", + frame: { + id: 4, + result: { + turn: makeCodexReplayTurn({ id: retriedTurnId, status: "inProgress" }), + }, + }, + }, + { + type: "emit_inbound", + frame: { + method: "turn/started", + params: { + threadId: RESUME_NATIVE_THREAD, + turn: makeCodexReplayTurn({ id: retriedTurnId, status: "inProgress" }), + }, + }, + }, + { + type: "emit_inbound", + frame: { + method: "turn/completed", + params: { + threadId: RESUME_NATIVE_THREAD, + turn: { + ...makeCodexReplayTurn({ id: retriedTurnId, status: parentStatus }), + error: + parentStatus === "failed" + ? { message: "Recovery failed.", codexErrorInfo: "internalServerError" } + : null, + }, + }, + }, + }, + childAgentMessage({ + id: "child-after-capacity", + text: "CAPACITY_CHILD_DONE", + turnId: RESUME_CHILD_TURN_1, + completedAtMs: 1782622480000, + afterMs: 60_000, + }), + childTurnCompleted(RESUME_CHILD_TURN_1), + ], + }), + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "error" && + event.turnItem.status === "running" + ? Deferred.succeed(retrying, undefined) + : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("capacity-late-child"), + text: RESUME_PROMPT, + }), + ); + yield* Deferred.await(retrying); + yield* TestClock.adjust("2 seconds"); + yield* harness.awaitTerminal(1); + yield* TestClock.adjust("60 seconds"); + yield* harness.until( + () => harness.subagentUpdates().at(-1)?.subagent.status === "completed", + ); + assert.lengthOf(harness.continuationRequests, parentStatus === "completed" ? 1 : 0); + if (parentStatus === "completed") + assert.include(harness.continuationRequests[0]?.detail, "CAPACITY_CHILD_DONE"); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + it.effect("names a late child spawned without a prompt in its wake", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 2c9cdcb97fb50..0554c5cbd8957 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -5645,7 +5645,15 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( } : event; yield* emitProviderEvent(current); - for (const turn of continuedRuns.get(context.providerTurnId) ?? []) { + const continued = continuedRuns.get(context.providerTurnId) ?? [context]; + if (current.status !== "completed") { + yield* Ref.update( + terminalizedNonCompletedNativeTurns, + (nativeTurns) => + new Set([...nativeTurns, ...continued.map((turn) => turn.nativeTurnId)]), + ); + } + for (const turn of continued) { continuedRuns.delete(turn.providerTurnId); } const nativeThreadId = context.providerThread.nativeThreadRef?.nativeId; @@ -5872,7 +5880,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( ), }); } - if (input.status !== "completed") { + if (input.status !== "completed" && !input.continueRun) { yield* Ref.update(terminalizedNonCompletedNativeTurns, (current) => { const updated = new Set(current); updated.add(input.nativeTurnId); @@ -6194,57 +6202,70 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }); yield* Effect.sleep(retryDelayMs).pipe( Effect.andThen( - turnTerminalizationPermit.withPermits(1)( - Effect.gen(function* () { - if ( - (yield* Ref.get(activeTurns)).get(terminal.nativeTurnId) !== context || - (yield* Ref.get(interruptingNativeTurns)).has(terminal.nativeTurnId) - ) - return; - const nativeThreadId = yield* getNativeThreadId(context.providerThread); - capacityContinuations.set(nativeThreadId, { context, retry: state }); - yield* finalizeCodexTurn({ ...terminal, continueRun: true }); - yield* startNativeTurn( - { - ...context.input, - providerTurnOrdinal: context.providerTurnOrdinal + 1, - }, - [], - ).pipe( - Effect.catch((cause) => - Effect.gen(function* () { - yield* Effect.logWarning( - "orchestration-v2.codex-capacity-retry-start-failed", - { - providerTurnId: context.providerTurnId, - cause, - }, - ); - yield* emitProviderEvent({ - type: "turn_item.updated", - driver: CODEX_PROVIDER, - turnItem: codexRetryTurnItem( - context, - state, - "failed", - yield* DateTime.now, - ), - }); - if (context.capacityRetryPending !== undefined) { - yield* Deferred.succeed(context.capacityRetryPending, undefined); - } - yield* emitRootTerminal( + Effect.gen(function* () { + const nativeThreadId = yield* getNativeThreadId(context.providerThread); + const claimed = yield* turnTerminalizationPermit.withPermits(1)( + Effect.gen(function* () { + if ( + (yield* Ref.get(activeTurns)).get(terminal.nativeTurnId) !== context || + (yield* Ref.get(interruptingNativeTurns)).has(terminal.nativeTurnId) + ) + return false; + capacityContinuations.set(nativeThreadId, { context, retry: state }); + yield* finalizeCodexTurn({ ...terminal, continueRun: true }); + return true; + }), + ); + if (!claimed) return; + // Notifications before the RPC response also take the terminalization permit. + yield* startNativeTurn( + { + ...context.input, + providerTurnOrdinal: context.providerTurnOrdinal + 1, + }, + [], + ).pipe( + Effect.catch((cause) => + Effect.gen(function* () { + if ( + context.capacityRetryPending !== undefined && + (yield* Deferred.isDone(context.capacityRetryPending)) + ) + return; + yield* Effect.logWarning( + "orchestration-v2.codex-capacity-retry-start-failed", + { + providerTurnId: context.providerTurnId, + cause, + }, + ); + yield* emitProviderEvent({ + type: "turn_item.updated", + driver: CODEX_PROVIDER, + turnItem: codexRetryTurnItem( context, - yield* makeRootTerminalEvent({ ...terminal, providerRetry: state }), - ); - }), - ), - Effect.ensuring( - Effect.sync(() => capacityContinuations.delete(nativeThreadId)), - ), - ); - }), - ), + state, + "failed", + yield* DateTime.now, + ), + }); + if (context.capacityRetryPending !== undefined) { + yield* Deferred.succeed(context.capacityRetryPending, undefined); + } + yield* emitRootTerminal( + context, + yield* makeRootTerminalEvent({ ...terminal, providerRetry: state }), + ); + }), + ), + Effect.ensuring( + Effect.sync(() => { + if (capacityContinuations.get(nativeThreadId)?.context === context) + capacityContinuations.delete(nativeThreadId); + }), + ), + ); + }), ), Effect.forkIn(scope, { startImmediately: true }), ); @@ -6481,6 +6502,8 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( return next; }); const started = yield* client.request("turn/start", turnStartParams); + // A turn/started notification may have registered and completed this turn already. + if ((yield* Ref.get(pendingRootTurns)).get(threadId) !== turnInput) return; yield* registerRootTurn({ turnInput, nativeTurnId: started.turn.id, @@ -6493,6 +6516,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( Effect.ensuring( Effect.flatMap(getNativeThreadId(turnInput.providerThread), (threadId) => Ref.update(pendingRootTurns, (current) => { + if (current.get(threadId) !== turnInput) return current; const updated = new Map(current); updated.delete(threadId); return updated; @@ -6935,6 +6959,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( let activeTurn = Array.from((yield* Ref.get(activeTurns)).values()).find( (candidate) => candidate.providerTurnId === providerTurnId, ); + const startingRetry = capacityContinuations.get(threadId)?.context; + if (activeTurn === undefined && startingRetry?.providerTurnId === providerTurnId) { + activeTurn = startingRetry; + } while (activeTurn?.capacityRetryPending !== undefined) { activeTurn = yield* Deferred.await(activeTurn.capacityRetryPending); } @@ -6987,6 +7015,24 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( ), interruptTurn: Effect.fn("CodexAdapterV2.interruptTurn")( function* (requestedInput) { + const nativeThreadId = requestedInput.providerThread.nativeThreadRef?.nativeId; + const startingRetry = + nativeThreadId == null + ? undefined + : capacityContinuations.get(nativeThreadId)?.context; + if ( + startingRetry?.providerTurnId === latestRunTurnId(requestedInput.providerTurnId) && + startingRetry.capacityRetryPending !== undefined + ) { + const started = yield* Deferred.await(startingRetry.capacityRetryPending).pipe( + Effect.timeoutOption("10 seconds"), + ); + if (Option.isNone(started)) + return yield* toProtocolError( + "Codex did not start the capacity retry within 10 seconds; Stop could not be delivered.", + ); + if (started.value === undefined) return; + } const { turnInput, goalTurns } = yield* resolveGoalStopTarget(requestedInput); const [activeTurnContexts, settledTurnContexts] = yield* turnTerminalizationPermit.withPermits(1)( From 2f184c32f6937f7b3a08131599786149cf3c43ad Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Mon, 26 Oct 2026 17:08:57 -0500 Subject: [PATCH 3/6] fix(server): preserve child lineage and Stop across capacity retries --- .../Adapters/CodexAdapterV2.test.ts | 100 +++++++++++++++++- .../Adapters/CodexAdapterV2.ts | 39 +++++-- 2 files changed, 130 insertions(+), 9 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index 8b76955aeebe8..0ddedd6c88167 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -5410,7 +5410,7 @@ describe("CodexAdapterV2 post-settle continuation", () => { }; }; - it.effect.each(["completed", "failed"] as const)( + it.effect.each(["completed", "failed", "interrupted"] as const)( "handles a late native child after the capacity retry %s", (parentStatus) => Effect.scoped( @@ -5519,11 +5519,17 @@ describe("CodexAdapterV2 post-settle continuation", () => { ); yield* Deferred.await(retrying); yield* TestClock.adjust("2 seconds"); - yield* harness.awaitTerminal(1); + if (parentStatus === "interrupted") assert.isEmpty(harness.terminalEvents()); + else { + yield* harness.awaitTerminal(1); + const terminal = harness.terminalEvents()[0]; + if (terminal?.status === "failed") assert.isTrue(terminal.childTurnsContinue); + } yield* TestClock.adjust("60 seconds"); yield* harness.until( () => harness.subagentUpdates().at(-1)?.subagent.status === "completed", ); + yield* harness.awaitTerminal(1); assert.lengthOf(harness.continuationRequests, parentStatus === "completed" ? 1 : 0); if (parentStatus === "completed") assert.include(harness.continuationRequests[0]?.detail, "CAPACITY_CHILD_DONE"); @@ -6962,6 +6968,96 @@ describe("CodexAdapterV2 post-settle continuation", () => { ), ); + it.effect("Stop prevents a capacity retry while pausing its goal", () => + Effect.gen(function* () { + const nativeTurnId = "goal-capacity-stop"; + const prompt = "Keep going"; + const retrying = yield* Deferred.make(); + const pausing = yield* Deferred.make(); + const requests: Array = []; + const harness = yield* makeCodexReplayHarness( + makeCodexReplayTranscript({ + scenario: "goal-capacity-stop", + entries: [ + ...codexReplayPreamble({ nativeThreadId, nativeTurnId, prompt }), + notification("goal active", "thread/goal/updated", { + threadId: nativeThreadId, + turnId: nativeTurnId, + goal: codexGoal("active", 10), + }), + notification("capacity failure", "turn/completed", { + threadId: nativeThreadId, + turn: { + ...makeCodexReplayTurn({ id: nativeTurnId, status: "failed" }), + error: { + message: "Selected model is at capacity.", + codexErrorInfo: "serverOverloaded", + }, + }, + }), + { + type: "expect_outbound", + frame: { + id: 4, + method: "thread/goal/set", + params: { threadId: nativeThreadId, status: "paused" }, + }, + }, + { + type: "emit_inbound", + afterMs: 3_000, + frame: { id: 4, result: { goal: codexGoal("paused", 10) } }, + }, + ], + }), + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "error" && + event.turnItem.status === "running" + ? Deferred.succeed(retrying, undefined) + : Effect.void, + (method) => + Effect.gen(function* () { + requests.push(method); + if (method === "thread/goal/set") yield* Deferred.succeed(pausing, undefined); + }), + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("goal-capacity-stop"), + text: prompt, + }), + ); + yield* Deferred.await(retrying); + const stopping = yield* harness.runtime + .interruptTurn({ + providerThread: harness.providerThread, + providerTurnId: (yield* IdAllocator.IdAllocatorV2).derive.providerTurn({ + driver: CodexAdapterV2.CODEX_DRIVER_KIND, + nativeTurnId, + }), + }) + .pipe(Effect.forkScoped); + yield* Deferred.await(pausing); + yield* TestClock.adjust("3 seconds"); + assert.lengthOf( + requests.filter((method) => method === "turn/start"), + 1, + ); + yield* Fiber.join(stopping); + yield* harness.awaitTerminal(1); + assert.equal(harness.terminalEvents()[0]?.status, "interrupted"); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ); + it.effect("pauses an active goal before Stop interrupts its turn", () => Effect.gen(function* () { const prompt = "Keep going"; diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 0554c5cbd8957..e8305a2181fe2 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -1854,6 +1854,13 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( const goalStops = new Map(); const latestRunTurnId = (providerTurnId: ProviderTurnId) => continuedRuns.get(providerTurnId)?.at(-1)?.providerTurnId ?? providerTurnId; + const isContinuedRunDescendant = ( + candidate: ActiveCodexTurnContext, + context: ActiveCodexTurnContext, + ) => + (continuedRuns.get(context.providerTurnId) ?? [context]).some((ancestor) => + isDescendantCodexTurn(candidate, ancestor), + ); const emitProviderEvent = (event: ProviderAdapter.ProviderAdapterV2Event) => Queue.offer(events, event).pipe(Effect.asVoid); @@ -5692,7 +5699,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }) { const event = yield* makeRootTerminalEvent(input); const hasActiveDescendants = Array.from((yield* Ref.get(activeTurns)).values()).some( - (candidate) => isDescendantCodexTurn(candidate, input.context), + (candidate) => isContinuedRunDescendant(candidate, input.context), ); // Failed roots leave native children working and must settle without them. // Interrupted roots wait for descendants because Stop interrupts the whole lineage. @@ -5817,7 +5824,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( for (const [nativeTurnId, deferred] of current) { if ( !activeTurnContexts.some((candidate) => - isDescendantCodexTurn(candidate, deferred.context), + isContinuedRunDescendant(candidate, deferred.context), ) ) { updated.delete(nativeTurnId); @@ -5883,7 +5890,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( if (input.status !== "completed" && !input.continueRun) { yield* Ref.update(terminalizedNonCompletedNativeTurns, (current) => { const updated = new Set(current); - updated.add(input.nativeTurnId); + for (const turn of continuedRuns.get(input.context.providerTurnId) ?? [ + input.context, + ]) + updated.add(turn.nativeTurnId); return updated; }); } @@ -6208,6 +6218,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( Effect.gen(function* () { if ( (yield* Ref.get(activeTurns)).get(terminal.nativeTurnId) !== context || + goalStops.has(nativeThreadId) || (yield* Ref.get(interruptingNativeTurns)).has(terminal.nativeTurnId) ) return false; @@ -6252,9 +6263,8 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( if (context.capacityRetryPending !== undefined) { yield* Deferred.succeed(context.capacityRetryPending, undefined); } - yield* emitRootTerminal( - context, - yield* makeRootTerminalEvent({ ...terminal, providerRetry: state }), + yield* turnTerminalizationPermit.withPermits(1)( + emitOrDeferRootTerminal({ ...terminal, providerRetry: state }), ); }), ), @@ -7413,7 +7423,22 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( ); }, (effect, requestedInput) => - effect.pipe( + Effect.acquireUseRelease( + Effect.sync(() => { + const nativeThreadId = requestedInput.providerThread.nativeThreadRef?.nativeId; + if (nativeThreadId != null) + goalStops.set(nativeThreadId, (goalStops.get(nativeThreadId) ?? 0) + 1); + return nativeThreadId; + }), + () => effect, + (nativeThreadId) => + Effect.sync(() => { + if (nativeThreadId == null) return; + const stops = (goalStops.get(nativeThreadId) ?? 1) - 1; + if (stops > 0) goalStops.set(nativeThreadId, stops); + else goalStops.delete(nativeThreadId); + }), + ).pipe( Effect.mapError( (cause) => new ProviderAdapter.ProviderAdapterInterruptError({ From b09ea52ea64c265bfaa6d51f6906dd961bbae01d Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Tue, 27 Oct 2026 02:49:40 +0000 Subject: [PATCH 4/6] refactor(server): simplify capacity retry startup state --- .../orchestration-v2/Adapters/CodexAdapterV2.ts | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index e8305a2181fe2..cbf11a2c94909 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -6173,9 +6173,8 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( const attempt = (context.capacityRetryAttempt ?? 0) + 1; context.capacityRetryAttempt = attempt; - context.capacityRetryPending = yield* Deferred.make< - ActiveCodexTurnContext | undefined - >(); + const pending = yield* Deferred.make(); + context.capacityRetryPending = pending; const retryDelayMs = Math.min(2_000 * 2 ** (attempt - 1), 30_000); const now = yield* DateTime.now; const previous = (yield* Ref.get(providerRetries)).get(context.providerTurnId); @@ -6238,11 +6237,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( ).pipe( Effect.catch((cause) => Effect.gen(function* () { - if ( - context.capacityRetryPending !== undefined && - (yield* Deferred.isDone(context.capacityRetryPending)) - ) - return; + if (yield* Deferred.isDone(pending)) return; yield* Effect.logWarning( "orchestration-v2.codex-capacity-retry-start-failed", { @@ -6260,9 +6255,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( yield* DateTime.now, ), }); - if (context.capacityRetryPending !== undefined) { - yield* Deferred.succeed(context.capacityRetryPending, undefined); - } + yield* Deferred.succeed(pending, undefined); yield* turnTerminalizationPermit.withPermits(1)( emitOrDeferRootTerminal({ ...terminal, providerRetry: state }), ); From fa7f4a5a73b8270eaf06999cca7f187d8e35b070 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Tue, 27 Oct 2026 02:58:51 +0000 Subject: [PATCH 5/6] fix(server): preserve controls and child lineage during capacity recovery --- .../Adapters/CodexAdapterV2.test.ts | 125 +++++++++++++++++- .../Adapters/CodexAdapterV2.ts | 21 ++- .../ProviderTurnControlService.ts | 6 +- .../SteeringCompletion.integration.test.ts | 26 +++- 4 files changed, 167 insertions(+), 11 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index 0ddedd6c88167..076461bdd8019 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -5410,7 +5410,7 @@ describe("CodexAdapterV2 post-settle continuation", () => { }; }; - it.effect.each(["completed", "failed", "interrupted"] as const)( + it.effect.each(["completed", "failed", "interrupted", "starting"] as const)( "handles a late native child after the capacity retry %s", (parentStatus) => Effect.scoped( @@ -5456,6 +5456,7 @@ describe("CodexAdapterV2 post-settle continuation", () => { }, }, }, + ...(parentStatus === "starting" ? [childTurnCompleted(RESUME_CHILD_TURN_1)] : []), { type: "emit_inbound", frame: { @@ -5482,7 +5483,10 @@ describe("CodexAdapterV2 post-settle continuation", () => { params: { threadId: RESUME_NATIVE_THREAD, turn: { - ...makeCodexReplayTurn({ id: retriedTurnId, status: parentStatus }), + ...makeCodexReplayTurn({ + id: retriedTurnId, + status: parentStatus === "starting" ? "completed" : parentStatus, + }), error: parentStatus === "failed" ? { message: "Recovery failed.", codexErrorInfo: "internalServerError" } @@ -5541,6 +5545,123 @@ describe("CodexAdapterV2 post-settle continuation", () => { ), ); + it.effect("stops native children when capacity retry startup fails", () => + Effect.scoped( + Effect.gen(function* () { + const retrying = yield* Deferred.make(); + const starting = yield* Deferred.make(); + const release = yield* Deferred.make(); + const interrupts: unknown[] = []; + let starts = 0; + const harness = yield* makeCodexReplayHarness( + makeCodexReplayTranscript({ + scenario: "capacity-start-fails-stop-child", + entries: [ + ...lateChildPrefix().spawned, + { + type: "emit_inbound", + frame: { + method: "turn/completed", + params: { + threadId: RESUME_NATIVE_THREAD, + turn: { + ...makeCodexReplayTurn({ id: RESUME_NATIVE_TURN, status: "failed" }), + error: { message: "At capacity", codexErrorInfo: "serverOverloaded" }, + }, + }, + }, + }, + { + type: "expect_outbound", + frame: { + id: 4, + method: "turn/start", + params: { + threadId: RESUME_NATIVE_THREAD, + input: [], + cwd: "/workspace", + model: "gpt-5.4", + approvalPolicy: "never", + approvalsReviewer: "user", + sandboxPolicy: { type: "dangerFullAccess" }, + summary: "detailed", + collaborationMode: { mode: "default", settings: { model: "gpt-5.4" } }, + }, + }, + }, + { + type: "emit_inbound", + frame: { id: 4, error: { code: -32603, message: "Start failed" } }, + }, + { + type: "expect_outbound", + frame: { + id: 5, + method: "turn/interrupt", + params: { threadId: RESUME_CHILD_THREAD, turnId: RESUME_CHILD_TURN_1 }, + }, + }, + { type: "emit_inbound", frame: { id: 5, result: {} } }, + { + type: "emit_inbound", + frame: { + method: "turn/completed", + params: { + threadId: RESUME_CHILD_THREAD, + turn: makeCodexReplayTurn({ id: RESUME_CHILD_TURN_1, status: "interrupted" }), + }, + }, + }, + ], + }), + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "error" && + event.turnItem.status === "running" + ? Deferred.succeed(retrying, undefined) + : Effect.void, + (method, params) => + Effect.gen(function* () { + if (method === "turn/interrupt") interrupts.push(params); + if (method === "turn/start" && ++starts === 2) { + yield* Deferred.succeed(starting, undefined); + yield* Deferred.await(release); + } + }), + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("capacity-start-fails-stop-child"), + text: RESUME_PROMPT, + }), + ); + yield* Deferred.await(retrying); + const root = harness.events.find((event) => event.type === "provider_turn.updated"); + assert.equal(root?.type, "provider_turn.updated"); + if (root?.type !== "provider_turn.updated") return; + yield* TestClock.adjust("2 seconds"); + yield* Deferred.await(starting); + const stopping = yield* harness.runtime + .interruptTurn({ + providerThread: harness.providerThread, + providerTurnId: root.providerTurn.id, + }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(stopping); + assert.lengthOf(interrupts, 1); + assert.equal(harness.subagentUpdates().at(-1)?.subagent.status, "interrupted"); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + it.effect("names a late child spawned without a prompt in its wake", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index cbf11a2c94909..d904da9f1ca27 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -6014,11 +6014,14 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( // Only direct children wake an idle root turn; grandchildren report to their // own parent, and running root turns receive results natively. const parent = input.context.subagent.parentContext; + const parentNativeThreadId = parent.providerThread.nativeThreadRef?.nativeId; if ( continuationRequests !== undefined && parent.subagent === null && (input.status === "completed" || input.status === "failed") && !(yield* Ref.get(terminalizedNonCompletedNativeTurns)).has(parent.nativeTurnId) && + (parentNativeThreadId == null || + !capacityContinuations.has(parentNativeThreadId)) && ![...(yield* Ref.get(activeTurns)).values()].some( (active) => active.subagent === null && @@ -7019,14 +7022,15 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( interruptTurn: Effect.fn("CodexAdapterV2.interruptTurn")( function* (requestedInput) { const nativeThreadId = requestedInput.providerThread.nativeThreadRef?.nativeId; - const startingRetry = + const capacityRetry = nativeThreadId == null ? undefined : capacityContinuations.get(nativeThreadId)?.context; - if ( - startingRetry?.providerTurnId === latestRunTurnId(requestedInput.providerTurnId) && - startingRetry.capacityRetryPending !== undefined - ) { + const startingRetry = + capacityRetry?.providerTurnId === latestRunTurnId(requestedInput.providerTurnId) + ? capacityRetry + : undefined; + if (startingRetry?.capacityRetryPending !== undefined) { const started = yield* Deferred.await(startingRetry.capacityRetryPending).pipe( Effect.timeoutOption("10 seconds"), ); @@ -7034,7 +7038,6 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( return yield* toProtocolError( "Codex did not start the capacity retry within 10 seconds; Stop could not be delivered.", ); - if (started.value === undefined) return; } const { turnInput, goalTurns } = yield* resolveGoalStopTarget(requestedInput); const [activeTurnContexts, settledTurnContexts] = @@ -7056,7 +7059,11 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( ) : undefined); const lineageRoots = Array.from( - new Set([...(activeTurn === undefined ? [] : [activeTurn]), ...goalTurns]), + new Set([ + ...(activeTurn === undefined ? [] : [activeTurn]), + ...(startingRetry === undefined ? [] : [startingRetry]), + ...goalTurns, + ]), ); const inLineage = (context: ActiveCodexTurnContext) => lineageRoots.some( diff --git a/apps/server/src/orchestration-v2/ProviderTurnControlService.ts b/apps/server/src/orchestration-v2/ProviderTurnControlService.ts index 3a5bc04ecba2e..e1dbac21bc52e 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnControlService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnControlService.ts @@ -127,7 +127,11 @@ export const layer: Layer.Layer< const interruptProviderThread = targetsRecordedSession ? providerThread : { ...providerThread, providerSessionId: input.providerSessionId }; - if (providerTurn.status !== "running") { + const continuingRun = + input.operation === "steer" && + context.run?.status === "running" && + context.run.activeAttemptId === providerTurn.runAttemptId; + if (providerTurn.status !== "running" && !continuingRun) { if (input.operation === "steer") { return yield* new ProviderTurnControlError({ threadId: input.threadId, diff --git a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts index 2cc6ee6f825b2..2e4cdb7241358 100644 --- a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts +++ b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts @@ -42,6 +42,7 @@ it.effect.each( [true, "without native steering"], [true, "with interrupting native steering"], [true, "settled only"], + [false, "during continuation"], ] as const ).map(([mailbox, timing]) => ({ mailbox, @@ -135,7 +136,7 @@ it.effect.each( steerTurn: (turn) => Effect.gen(function* () { steerCalls += 1; - if (timing === "after delivery") return; + if (timing === "after delivery" || timing === "during continuation") return; yield* Deferred.succeed(steerEntered, undefined); yield* Deferred.await(rejectSteer); return yield* new ProviderAdapter.ProviderAdapterSteerRunError({ @@ -284,6 +285,29 @@ it.effect.each( : {}), }); if (timing !== "before dispatch") yield* dispatchSteer; + if (timing === "during continuation") { + const projection = yield* orchestrator.getThreadProjection(threadId); + const turn = projection.providerTurns[0]!; + const failed = yield* watch( + (event) => event.type === "provider-turn.updated" && event.payload.status === "failed", + ); + yield* Queue.offer(events, { + type: "provider_turn.updated", + driver, + providerTurn: { ...turn, status: "failed", completedAt: yield* DateTime.now }, + }); + yield* Fiber.join(failed); + yield* worker.drain(); + assert.equal(steerCalls, 1); + assert.lengthOf(started, 1); + const continued = yield* orchestrator.getThreadProjection(threadId); + assert.equal(continued.runs[0]?.status, "running"); + assert.equal( + continued.messages.find((message) => message.id === messageId)?.runId, + first.runId, + ); + return; + } if (timing === "after delivery") yield* worker.drain(); const delivery = timing === "during delivery" ? yield* worker.runOnce.pipe(Effect.forkScoped) : null; From e7d7f7f6238c34362fb9f15170a2cfc5ad993864 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Tue, 27 Oct 2026 03:06:56 +0000 Subject: [PATCH 6/6] fix(server): consume capacity continuation when its turn registers --- .../Adapters/CodexAdapterV2.test.ts | 202 ++++++++++-------- .../Adapters/CodexAdapterV2.ts | 3 + 2 files changed, 121 insertions(+), 84 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index 076461bdd8019..1e762543a13ae 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -2223,6 +2223,7 @@ describe("CodexAdapterV2 post-settle continuation", () => { "stop-starting", "started-first", "completed-first", + "new-run", "steer", "start-fails", ] as const)("handles terminal Codex capacity failures when retries %s", (outcome) => @@ -2283,7 +2284,8 @@ describe("CodexAdapterV2 post-settle continuation", () => { if ( outcome === "started-first" || outcome === "stop-starting" || - outcome === "completed-first" + outcome === "completed-first" || + outcome === "new-run" ) { start.splice(1, 0, ...start.splice(2, 1)); } @@ -2308,7 +2310,7 @@ describe("CodexAdapterV2 post-settle continuation", () => { }, }); } else if (entry.type === "emit_inbound" && entry.label === "turn/start") { - if (outcome === "completed-first") + if (outcome === "completed-first" || outcome === "new-run") entries.push({ type: "emit_inbound", frame: { @@ -2319,6 +2321,18 @@ describe("CodexAdapterV2 post-settle continuation", () => { }, }, }); + if (outcome === "new-run") { + entries.push( + ...codexReplayPreamble({ + nativeThreadId, + nativeTurnId: nativeTurnId(2), + prompt: "Next task.", + }) + .slice(-3) + .map((next, index) => (index < 2 ? withReplayRequestId(next, 5) : next)), + ...failureEntries(2), + ); + } entries.push( outcome === "start-fails" ? { @@ -2491,6 +2505,19 @@ describe("CodexAdapterV2 post-settle continuation", () => { } } yield* harness.awaitTerminal(1); + if (outcome === "new-run") { + assert.equal((yield* Queue.take(milestones)).type, "turn.terminal"); + yield* harness.runtime.startTurn({ + ...turnInput, + attemptId: RunAttemptId.make("capacity-next-task"), + message: { ...turnInput.message, text: "Next task." }, + }); + const nextRetry = yield* Queue.take(milestones); + assert.equal(nextRetry.type, "turn_item.updated"); + if (nextRetry.type !== "turn_item.updated" || nextRetry.turnItem.type !== "error") return; + assert.equal(nextRetry.turnItem.retry?.attempt, 1); + return; + } const terminal = harness.terminalEvents()[0]; assert.equal( terminal?.status, @@ -5410,101 +5437,105 @@ describe("CodexAdapterV2 post-settle continuation", () => { }; }; - it.effect.each(["completed", "failed", "interrupted", "starting"] as const)( + it.effect.each(["completed", "failed", "interrupted", "starting", "replied-last"] as const)( "handles a late native child after the capacity retry %s", (parentStatus) => Effect.scoped( Effect.gen(function* () { const retriedTurnId = "native-capacity-recovered-parent"; const retrying = yield* Deferred.make(); - const harness = yield* makeCodexReplayHarness( - makeCodexReplayTranscript({ - scenario: "capacity-late-child", - entries: [ - ...lateChildPrefix().spawned, - { - type: "emit_inbound", - frame: { - method: "turn/completed", - params: { - threadId: RESUME_NATIVE_THREAD, - turn: { - ...makeCodexReplayTurn({ id: RESUME_NATIVE_TURN, status: "failed" }), - error: { - message: "Selected model is at capacity.", - codexErrorInfo: "serverOverloaded", - }, - }, + const entries: Array = [ + ...lateChildPrefix().spawned, + { + type: "emit_inbound", + frame: { + method: "turn/completed", + params: { + threadId: RESUME_NATIVE_THREAD, + turn: { + ...makeCodexReplayTurn({ id: RESUME_NATIVE_TURN, status: "failed" }), + error: { + message: "Selected model is at capacity.", + codexErrorInfo: "serverOverloaded", }, }, }, - { - type: "expect_outbound", - frame: { - id: 4, - method: "turn/start", - params: { - threadId: RESUME_NATIVE_THREAD, - input: [], - cwd: "/workspace", - model: "gpt-5.4", - approvalPolicy: "never", - approvalsReviewer: "user", - sandboxPolicy: { type: "dangerFullAccess" }, - summary: "detailed", - collaborationMode: { mode: "default", settings: { model: "gpt-5.4" } }, - }, - }, + }, + }, + { + type: "expect_outbound", + frame: { + id: 4, + method: "turn/start", + params: { + threadId: RESUME_NATIVE_THREAD, + input: [], + cwd: "/workspace", + model: "gpt-5.4", + approvalPolicy: "never", + approvalsReviewer: "user", + sandboxPolicy: { type: "dangerFullAccess" }, + summary: "detailed", + collaborationMode: { mode: "default", settings: { model: "gpt-5.4" } }, }, - ...(parentStatus === "starting" ? [childTurnCompleted(RESUME_CHILD_TURN_1)] : []), - { - type: "emit_inbound", - frame: { - id: 4, - result: { - turn: makeCodexReplayTurn({ id: retriedTurnId, status: "inProgress" }), - }, - }, + }, + }, + ...(parentStatus === "starting" ? [childTurnCompleted(RESUME_CHILD_TURN_1)] : []), + { + type: "emit_inbound", + frame: { + id: 4, + result: { + turn: makeCodexReplayTurn({ id: retriedTurnId, status: "inProgress" }), }, - { - type: "emit_inbound", - frame: { - method: "turn/started", - params: { - threadId: RESUME_NATIVE_THREAD, - turn: makeCodexReplayTurn({ id: retriedTurnId, status: "inProgress" }), - }, - }, + }, + }, + { + type: "emit_inbound", + frame: { + method: "turn/started", + params: { + threadId: RESUME_NATIVE_THREAD, + turn: makeCodexReplayTurn({ id: retriedTurnId, status: "inProgress" }), }, - { - type: "emit_inbound", - frame: { - method: "turn/completed", - params: { - threadId: RESUME_NATIVE_THREAD, - turn: { - ...makeCodexReplayTurn({ - id: retriedTurnId, - status: parentStatus === "starting" ? "completed" : parentStatus, - }), - error: - parentStatus === "failed" - ? { message: "Recovery failed.", codexErrorInfo: "internalServerError" } - : null, - }, - }, + }, + }, + { + type: "emit_inbound", + frame: { + method: "turn/completed", + params: { + threadId: RESUME_NATIVE_THREAD, + turn: { + ...makeCodexReplayTurn({ + id: retriedTurnId, + status: + parentStatus === "starting" || parentStatus === "replied-last" + ? "completed" + : parentStatus, + }), + error: + parentStatus === "failed" + ? { message: "Recovery failed.", codexErrorInfo: "internalServerError" } + : null, }, }, - childAgentMessage({ - id: "child-after-capacity", - text: "CAPACITY_CHILD_DONE", - turnId: RESUME_CHILD_TURN_1, - completedAtMs: 1782622480000, - afterMs: 60_000, - }), - childTurnCompleted(RESUME_CHILD_TURN_1), - ], + }, + }, + childAgentMessage({ + id: "child-after-capacity", + text: "CAPACITY_CHILD_DONE", + turnId: RESUME_CHILD_TURN_1, + completedAtMs: 1782622480000, + afterMs: 60_000, }), + childTurnCompleted(RESUME_CHILD_TURN_1), + ]; + if (parentStatus === "replied-last") { + entries.push(...entries.splice(lateChildPrefix().spawned.length + 2, 1)); + } + const harness = yield* makeCodexReplayHarness( + makeCodexReplayTranscript({ scenario: "capacity-late-child", entries }), (event) => event.type === "turn_item.updated" && event.turnItem.type === "error" && @@ -5534,8 +5565,11 @@ describe("CodexAdapterV2 post-settle continuation", () => { () => harness.subagentUpdates().at(-1)?.subagent.status === "completed", ); yield* harness.awaitTerminal(1); - assert.lengthOf(harness.continuationRequests, parentStatus === "completed" ? 1 : 0); - if (parentStatus === "completed") + assert.lengthOf( + harness.continuationRequests, + parentStatus === "completed" || parentStatus === "replied-last" ? 1 : 0, + ); + if (parentStatus === "completed" || parentStatus === "replied-last") assert.include(harness.continuationRequests[0]?.detail, "CAPACITY_CHILD_DONE"); }).pipe( Effect.provide( diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index d904da9f1ca27..74c4b46646e3e 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -1991,6 +1991,9 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( updated.set(input.nativeTurnId, context); return updated; }); + if (continuation !== undefined && nativeThreadId != null) { + capacityContinuations.delete(nativeThreadId); + } yield* emitProviderEvent({ type: "provider_turn.updated", driver: CODEX_PROVIDER,