diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index faddf5fa837b2..33bc37cf29eb4 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -9710,6 +9710,10 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + const gatewayLimitMessage = + "API Error: Server is temporarily limiting requests (not your usage limit) · All credentials for model claude-opus-5-5 are cooling down"; + const gatewayLimitWithCountdown = `${gatewayLimitMessage} (last error: rate_limit_error: This request would exceed your account's rate limit. Please try again later.) (reset after 2h26m17s)`; + it.effect.each([ { status: 503, cooldown: "4m 6s", seconds: 246 }, { status: 503, cooldown: "1h 2m 3s", seconds: 3723 }, @@ -9722,6 +9726,42 @@ describe("ClaudeAdapterV2 background wake turns", () => { { status: 429, cooldown: "1h2m3s", seconds: 3723, proxy: true, sdkResetSeconds: 60 }, { status: 429, cooldown: "1h2m3s", seconds: 7200, proxy: true, sdkResetSeconds: 7200 }, { status: 429, cooldown: "1h2m3s", seconds: 0, proxy: true, sdkResetSeconds: null }, + { + status: 429, + cooldown: "2h26m17s with an untimed assistant rejection", + seconds: 8777, + proxy: true, + assistantResetSeconds: null, + result: gatewayLimitWithCountdown, + }, + { + status: 429, + cooldown: "2h26m17s with a later assistant reset", + seconds: 9000, + proxy: true, + sdkResetSeconds: null, + sdkLimitType: null, + assistantResetSeconds: 9000, + result: gatewayLimitWithCountdown, + }, + { + status: 429, + cooldown: "a countdown only inside the last error", + seconds: 0, + proxy: true, + sdkResetSeconds: null, + sdkLimitType: null, + result: `${gatewayLimitMessage} (last error: rate_limit_error (reset after 1h))`, + }, + { + status: 429, + cooldown: "an outer countdown after a nested countdown", + seconds: 60, + proxy: true, + sdkResetSeconds: null, + sdkLimitType: null, + result: `${gatewayLimitMessage} (last error: rate_limit_error (reset after 1h)) (reset after 1m)`, + }, { status: 429, cooldown: "nested 1h", @@ -9745,8 +9785,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { proxy: true, sdkResetSeconds: null, sdkLimitType: null, - result: - "API Error: Server is temporarily limiting requests (not your usage limit) · All credentials for model claude-opus-5-5 are cooling down (last error: rate_limit_error: This request would exceed your account's rate limit. Please try again later.) (reset after 2h26m17s)", + result: gatewayLimitWithCountdown, }, ])("recovers wrapped gateway limits with status $status and cooldown $cooldown", (input) => Effect.gen(function* () { @@ -9786,6 +9825,22 @@ describe("ClaudeAdapterV2 background wake turns", () => { makeAssistantErrorFrame({ uuid: "00000000-0000-4000-8000-000000000652", error: input.proxy ? "rate_limit" : "server_error", + ...(input.assistantResetSeconds !== undefined + ? { + apiErrorParams: { + rate_limit_info: { + status: "rejected", + isUsingOverage: false, + ...(input.assistantResetSeconds === null + ? {} + : { + resetsAt: + DateTime.toEpochMillis(now) / 1000 + input.assistantResetSeconds, + }), + }, + }, + } + : {}), }), makeResultFrame({ uuid: "00000000-0000-4000-8000-000000000653", diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 46441a539c0ad..18b21d185479c 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -4827,6 +4827,25 @@ function gatewayCooldownResetAt( : undefined; } +function claude429Cooldown(result: string): RegExpExecArray | null { + const pooledCooldown = result.replace( + /^API Error: (?:Request rejected \(429\)|Server is temporarily limiting requests \(not your usage limit\)) · All credentials for model /, + "", + ); + if (pooledCooldown === result) { + return /\(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec(result); + } + // The gateway can include a separate upstream cooldown in its nested last error. + return ( + /^[^\s()]+ are cooling down \(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec( + pooledCooldown, + ) ?? + /^[^\s()]+ are cooling down \(last error: [\s\S]*\) \(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)$/.exec( + pooledCooldown, + ) + ); +} + const CLAUDE_UNSUPPORTED_VERSION_ERROR = /^API Error: 400 Claude Code (\d+\.\d+\.\d+) does not support this model; version (\d+\.\d+\.\d+) or newer is required\./; @@ -4904,20 +4923,7 @@ function providerFailureFromResult( // API-key sessions get no rate_limit_info, so a gateway's 429 carries its reset only as text. const resetAt = apiErrorStatus === 429 && message.result.startsWith("API Error:") - ? gatewayCooldownResetAt( - // CLIProxyAPI nests upstream errors in its message; only its own cooldown counts. - message.result.startsWith( - "API Error: Request rejected (429) · All credentials for model ", - ) - ? (/^API Error: Request rejected \(429\) · All credentials for model [^\s()]+ are cooling down \(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec( - message.result, - ) ?? - /^API Error: Request rejected \(429\) · All credentials for model [^\s()]+ are cooling down \(last error: [\s\S]*\) \(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)$/.exec( - message.result, - )) - : /\(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec(message.result), - receivedAt, - ) + ? gatewayCooldownResetAt(claude429Cooldown(message.result), receivedAt) : undefined; return makeProviderFailure({ message: listedError ?? structuredError ?? failureHint ?? message.result, @@ -12127,16 +12133,21 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ); const resultResetAt = resultFailure?.class === "usage_limit" ? resultFailure.resetAt : null; - // An untyped rejection without a reset is the gateway's own 429; its text names the reset. + // An untimed, typeless SDK rejection can describe the same gateway 429. + const countsTowardReset = (type: string, resetAt: string | null | undefined) => + resetAt !== undefined && + (resultResetAt == null || type !== "unknown" || resetAt !== null); const resetTimes = Array.from(context.rateLimitResetTimes) - .filter( - ([type, time]) => !(type === "unknown" && time === null && resultResetAt != null), - ) + .filter(([type, time]) => countsTowardReset(type, time)) .map(([, time]) => time); const assistantLimit = context.latestAssistantRateLimit; if ( assistantLimit?.type != null && - !context.rateLimitResetTimes.has(assistantLimit.type) + !countsTowardReset( + assistantLimit.type, + context.rateLimitResetTimes.get(assistantLimit.type), + ) && + countsTowardReset(assistantLimit.type, assistantLimit.resetAt) ) { resetTimes.push(assistantLimit.resetAt); }