diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index cf533de93e4d6..b2b27327b66dc 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -15224,6 +15224,126 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + // regression: #37708 a gateway that holds the request open leaves Claude silent after "requesting" + it.effect.each([ + { case: "after the prompt starts", requesting: false, backgroundProgress: false }, + { case: "after a request goes out", requesting: true, backgroundProgress: false }, + { case: "while a skill it forked runs", requesting: false, forkedTask: true }, + { + case: "while an earlier background agent starts a nested task", + requesting: false, + nestedTaskStarted: true, + }, + { + case: "while an earlier background task reports progress", + requesting: false, + backgroundProgress: true, + }, + ])( + "does not keep a turn silently running when Claude goes quiet $case", + ({ requesting, backgroundProgress = false, forkedTask = false, nestedTaskStarted = false }) => + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-silent-after-started"), + text: "start again", + attachments: [], + }), + ); + // Captured from Claude Code 2.1.325 against a gateway that never answers /v1/messages. + for (const [index, state] of (["queued", "started"] as const).entries()) { + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "command_lifecycle", + command_uuid: harness.promptUuid(0), + state, + uuid: `00000000-0000-4000-8000-00000000079${index}`, + session_id: WAKE_NATIVE_SESSION, + }), + ); + } + if (requesting) { + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "status", + status: "requesting", + uuid: "00000000-0000-4000-8000-000000000792", + session_id: WAKE_NATIVE_SESSION, + }), + ); + } + if (backgroundProgress) { + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "task_progress", + task_id: "earlier-background-task", + description: "Earlier background task", + usage: { total_tokens: 1, tool_uses: 0, duration_ms: 1 }, + uuid: "00000000-0000-4000-8000-000000000793", + session_id: WAKE_NATIVE_SESSION, + }), + ); + } + if (nestedTaskStarted) { + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "task_started", + task_id: "earlier-nested-task", + description: "Nested task of an earlier background agent", + spawn_depth: 2, + uuid: "00000000-0000-4000-8000-000000000795", + session_id: WAKE_NATIVE_SESSION, + }), + ); + } + if (forkedTask) { + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "task_started", + task_id: "forked-skill-task", + description: "Forked skill", + uuid: "00000000-0000-4000-8000-000000000794", + session_id: WAKE_NATIVE_SESSION, + }), + ); + } + assert.lengthOf(harness.terminalEvents(), 0); + + yield* TestClock.adjust("1 hour"); + yield* yieldTicks(50); + const surfaced = + harness.terminalEvents().length > 0 || + harness.events.some( + (event) => + event.type === "turn_item.updated" && + (event.turnItem.type === "system_notice" || event.turnItem.type === "error"), + ); + if (forkedTask) { + assert.lengthOf(harness.terminalEvents(), 0, "a long forked skill was failed as silence"); + } else { + assert.isTrue(surfaced, "an hour of silence after the prompt started surfaced nothing"); + } + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + // regression: #17531 a mod that holds a tool behind a focused pane must fail the turn, not hang it it.effect.each([ { case: "fails a turn whose mod requests focus for a pane T3 cannot show", focus: true }, diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index fa8d3684f0f23..34d81c88d3559 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -4949,6 +4949,8 @@ function formatClaudeUsageLimitWait(waitMs: number): string { interface ActiveClaudeTurnContext { retryBudgetFailure?: OrchestrationV2ProviderFailure; + // Set when the prompt starts or a request goes out; cleared by the turn's next frame. + unansweredRequestSince?: DateTime.Utc | undefined; readonly queryPolicy: ClaudeRuntimeQueryPolicy; readonly input: ProviderAdapter.ProviderAdapterV2TurnInput; readonly autoCompactWindow: string; @@ -6397,40 +6399,73 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( : undefined; }); const retry = yield* isCurrentRetry; - // A user Stop already in flight keeps its interrupted outcome. - if ( - retry === undefined || - (yield* Ref.get(interruptedTurns)).has(context.providerTurnId) - ) - return; - context.retryBudgetFailure = { ...retry.failure, retryable: false }; - const liveQuery = yield* Ref.get(queryContext); - yield* interruptTurn({ - providerThread: context.input.providerThread, - providerTurnId: context.providerTurnId, - }).pipe( - Effect.timeoutOption(CLAUDE_STOP_CONFIRM_TIMEOUT_MS), - Effect.catch((cause) => - Effect.logWarning("orchestration-v2.claude-retry-budget-stop-failed", { cause }), - ), - ); - if (liveQuery === null || (yield* isCurrentRetry) === undefined) return; - // The CLI did not confirm the stop; settle the turn without it. - const ownedLiveQuery = yield* Ref.modify(queryContext, (current) => - current?.query === liveQuery.query ? [true, null] : [false, current], - ); - if (!ownedLiveQuery) return; - liveQuery.stopping = true; - yield* liveQuery.query.close.pipe( - Effect.timeoutOption(CLAUDE_STOP_CONFIRM_TIMEOUT_MS), - Effect.ignore, + if (retry === undefined) return; + yield* failStalledTurn( + context, + { ...retry.failure, retryable: false }, + Effect.map(isCurrentRetry, (current) => current !== undefined), ); - yield* finalizeActiveTurnAfterQueryExit(liveQuery); - yield* endSubagentCallsOfClosedQuery(liveQuery.nativeThreadId, "interrupted"); - yield* Deferred.succeed(liveQuery.closed, undefined); }, ); + // Partial messages are on, so a started prompt or a sent request streams within seconds. + // A gateway can hold the request open without ever letting the CLI time out and retry. + const failUnansweredRequest = Effect.fn("ClaudeAdapterV2.failUnansweredRequest")(function* ( + context: ActiveClaudeTurnContext, + since: DateTime.Utc, + ) { + const isUnanswered = Effect.map( + Ref.get(activeTurn), + (active) => active === context && context.unansweredRequestSince === since, + ); + if (!(yield* isUnanswered)) return; + yield* failStalledTurn( + context, + makeProviderFailure({ + class: "transport_error", + code: "claude_request_unanswered", + message: `Claude sent nothing for ${PROVIDER_RETRY_BUDGET_MS / 60_000} minutes. The API or a gateway in front of it is not responding.`, + retryable: false, + }), + // Once the stop is sent, a frame clearing the silence must not skip forced settlement. + Effect.map(Ref.get(activeTurn), (active) => active === context), + ); + }); + + const failStalledTurn = Effect.fn("ClaudeAdapterV2.failStalledTurn")(function* ( + context: ActiveClaudeTurnContext, + failure: OrchestrationV2ProviderFailure, + stillStalled: Effect.Effect, + ) { + // A user Stop already in flight keeps its interrupted outcome. + if ((yield* Ref.get(interruptedTurns)).has(context.providerTurnId)) return; + context.retryBudgetFailure = failure; + const liveQuery = yield* Ref.get(queryContext); + yield* interruptTurn({ + providerThread: context.input.providerThread, + providerTurnId: context.providerTurnId, + }).pipe( + Effect.timeoutOption(CLAUDE_STOP_CONFIRM_TIMEOUT_MS), + Effect.catch((cause) => + Effect.logWarning("orchestration-v2.claude-retry-budget-stop-failed", { cause }), + ), + ); + if (liveQuery === null || !(yield* stillStalled)) return; + // The CLI did not confirm the stop; settle the turn without it. + const ownedLiveQuery = yield* Ref.modify(queryContext, (current) => + current?.query === liveQuery.query ? [true, null] : [false, current], + ); + if (!ownedLiveQuery) return; + liveQuery.stopping = true; + yield* liveQuery.query.close.pipe( + Effect.timeoutOption(CLAUDE_STOP_CONFIRM_TIMEOUT_MS), + Effect.ignore, + ); + yield* finalizeActiveTurnAfterQueryExit(liveQuery); + yield* endSubagentCallsOfClosedQuery(liveQuery.nativeThreadId, "interrupted"); + yield* Deferred.succeed(liveQuery.closed, undefined); + }); + const rememberProviderThread = /* Provider events include streaming deltas; avoid a span for each update. */ Effect.fnUntraced(function* (providerThread: OrchestrationV2ProviderThread) { @@ -10556,6 +10591,33 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } return; } + if (context === (yield* Ref.get(activeTurn))) { + const lifecycle = claudePromptLifecycle(message); + if ( + (message.type === "system" && + message.subtype === "status" && + message.status === "requesting") || + (lifecycle?.state === "started" && lifecycle.uuid === context.promptUuid) + ) { + const since = yield* DateTime.now; + context.unansweredRequestSince = since; + yield* failUnansweredRequest(context, since).pipe( + Effect.delay(PROVIDER_RETRY_BUDGET_MS), + Effect.forkIn(sessionScope), + ); + } else if ( + // Background tasks from earlier turns keep streaming while the root request hangs. + // Only the root starts depth-1 tasks, so such a start (a forked skill) is this turn's. + Reflect.get(message, "parent_tool_use_id") == null && + !( + message.type === "system" && + message.subtype.startsWith("task_") && + !(message.subtype === "task_started" && (message.spawn_depth ?? 1) <= 1) + ) + ) { + context.unansweredRequestSince = undefined; + } + } const focusedPlugin = claudeFocusRequestedPanePlugin(message); if (focusedPlugin !== null && context === (yield* Ref.get(activeTurn))) { yield* Effect.logWarning("orchestration-v2.claude-mod-pane-focus-unsupported", {