From 256567b106e1927f9acbb0217866a50cf02477f5 Mon Sep 17 00:00:00 2001 From: Rodrigo Brechard Date: Fri, 20 Nov 2026 13:17:30 +0100 Subject: [PATCH 1/2] fix(server): keep Claude gateway cooldowns after untimed rejections Claude can report an untimed SDK rejection for a 429 that has a gateway countdown. Keep the countdown and honor other SDK reset times. Implemented with Codex gpt-6-luna and gpt-6.2-sol through T3 Code. --- .../Adapters/ClaudeAdapterV2.test.ts | 50 ++++++++++++++++++- .../Adapters/ClaudeAdapterV2.ts | 20 ++++++-- 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 0077d99f03f4e..c351b1a07196e 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -9710,6 +9710,9 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + const gatewayLimitWithCountdown = + "API Error: Server is temporarily limiting requests (not your usage limit) · All credentials for model claude-opus-5-5 are cooling down (last error: rate_limit_error: This request would exceed your account's rate limit. Please try again later.) (reset after 2h26m17s)"; + it.effect.each([ { status: 503, cooldown: "4m 6s", seconds: 246 }, { status: 503, cooldown: "1h 2m 3s", seconds: 3723 }, @@ -9722,6 +9725,33 @@ describe("ClaudeAdapterV2 background wake turns", () => { { status: 429, cooldown: "1h2m3s", seconds: 3723, proxy: true, sdkResetSeconds: 60 }, { status: 429, cooldown: "1h2m3s", seconds: 7200, proxy: true, sdkResetSeconds: 7200 }, { status: 429, cooldown: "1h2m3s", seconds: 0, proxy: true, sdkResetSeconds: null }, + { + status: 429, + cooldown: "2h26m17s with an untimed SDK event", + seconds: 8777, + proxy: true, + sdkResetSeconds: null, + sdkRateLimitUntyped: true, + result: gatewayLimitWithCountdown, + }, + { + status: 429, + cooldown: "2h26m17s with an untimed assistant rejection", + seconds: 8777, + proxy: true, + assistantResetSeconds: null, + result: gatewayLimitWithCountdown, + }, + { + status: 429, + cooldown: "2h26m17s with a later assistant reset", + seconds: 9000, + proxy: true, + sdkResetSeconds: null, + sdkRateLimitUntyped: true, + assistantResetSeconds: 9000, + result: gatewayLimitWithCountdown, + }, { status: 429, cooldown: "nested 1h", @@ -9759,7 +9789,9 @@ describe("ClaudeAdapterV2 background wake turns", () => { type: "rate_limit_event", rate_limit_info: { status: "rejected", - rateLimitType: "five_hour", + ...(input.sdkRateLimitUntyped + ? { isUsingOverage: false } + : { rateLimitType: "five_hour" }), ...(input.sdkResetSeconds === null ? {} : { @@ -9774,6 +9806,22 @@ describe("ClaudeAdapterV2 background wake turns", () => { makeAssistantErrorFrame({ uuid: "00000000-0000-4000-8000-000000000652", error: input.proxy ? "rate_limit" : "server_error", + ...(input.assistantResetSeconds !== undefined + ? { + apiErrorParams: { + rate_limit_info: { + status: "rejected", + isUsingOverage: false, + ...(input.assistantResetSeconds === null + ? {} + : { + resetsAt: + DateTime.toEpochMillis(now) / 1000 + input.assistantResetSeconds, + }), + }, + }, + } + : {}), }), makeResultFrame({ uuid: "00000000-0000-4000-8000-000000000653", diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 7527b590cf162..e68cb560486cb 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -12125,16 +12125,28 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( failureHint, usageLimited, ); - const resetTimes = Array.from(context.rateLimitResetTimes.values()); + const gatewayResetAt = + resultFailure?.class === "usage_limit" ? resultFailure.resetAt : undefined; + // An untimed, typeless SDK rejection can describe the same gateway 429. + const countsTowardReset = (type: string, resetAt: string | null | undefined) => + resetAt !== undefined && + (gatewayResetAt == null || type !== "unknown" || resetAt !== null); + const resetTimes = Array.from(context.rateLimitResetTimes.entries()) + .filter(([type, resetAt]) => countsTowardReset(type, resetAt)) + .map(([, resetAt]) => resetAt); const assistantLimit = context.latestAssistantRateLimit; if ( assistantLimit?.type != null && - !context.rateLimitResetTimes.has(assistantLimit.type) + !countsTowardReset( + assistantLimit.type, + context.rateLimitResetTimes.get(assistantLimit.type), + ) && + countsTowardReset(assistantLimit.type, assistantLimit.resetAt) ) { resetTimes.push(assistantLimit.resetAt); } - if (resultFailure?.class === "usage_limit" && resultFailure.resetAt != null) { - resetTimes.push(resultFailure.resetAt); + if (gatewayResetAt != null) { + resetTimes.push(gatewayResetAt); } const resetAt = resetTimes.length > 0 && resetTimes.every((time) => time !== null) From c9a0ad43761b21474d39a4546f912956de755dab Mon Sep 17 00:00:00 2001 From: Rodrigo Brechard Date: Fri, 20 Nov 2026 13:36:48 +0100 Subject: [PATCH 2/2] fix(server): read the outer Claude gateway cooldown Apply the same gateway cooldown parser to both CLI 429 prefixes. A countdown inside the nested upstream error can describe a different limit. Implemented and reviewed with Codex gpt-6.2-sol through T3 Code. --- .../Adapters/ClaudeAdapterV2.test.ts | 23 +++++++++++-- .../Adapters/ClaudeAdapterV2.ts | 34 +++++++++++-------- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index c351b1a07196e..a086d1039fb19 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -9710,8 +9710,9 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); - const gatewayLimitWithCountdown = - "API Error: Server is temporarily limiting requests (not your usage limit) · All credentials for model claude-opus-5-5 are cooling down (last error: rate_limit_error: This request would exceed your account's rate limit. Please try again later.) (reset after 2h26m17s)"; + const gatewayLimitMessage = + "API Error: Server is temporarily limiting requests (not your usage limit) · All credentials for model claude-opus-5-5 are cooling down"; + const gatewayLimitWithCountdown = `${gatewayLimitMessage} (last error: rate_limit_error: This request would exceed your account's rate limit. Please try again later.) (reset after 2h26m17s)`; it.effect.each([ { status: 503, cooldown: "4m 6s", seconds: 246 }, @@ -9752,6 +9753,24 @@ describe("ClaudeAdapterV2 background wake turns", () => { assistantResetSeconds: 9000, result: gatewayLimitWithCountdown, }, + { + status: 429, + cooldown: "a countdown only inside the last error", + seconds: 0, + proxy: true, + sdkResetSeconds: null, + sdkRateLimitUntyped: true, + result: `${gatewayLimitMessage} (last error: rate_limit_error (reset after 1h))`, + }, + { + status: 429, + cooldown: "an outer countdown after a nested countdown", + seconds: 60, + proxy: true, + sdkResetSeconds: null, + sdkRateLimitUntyped: true, + result: `${gatewayLimitMessage} (last error: rate_limit_error (reset after 1h)) (reset after 1m)`, + }, { status: 429, cooldown: "nested 1h", diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index e68cb560486cb..8ad51f551dd4e 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -4827,6 +4827,25 @@ function gatewayCooldownResetAt( : undefined; } +function claude429Cooldown(result: string): RegExpExecArray | null { + const pooledCooldown = result.replace( + /^API Error: (?:Request rejected \(429\)|Server is temporarily limiting requests \(not your usage limit\)) · All credentials for model /, + "", + ); + if (pooledCooldown === result) { + return /\(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec(result); + } + // The gateway can include a separate upstream cooldown in its nested last error. + return ( + /^[^\s()]+ are cooling down \(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec( + pooledCooldown, + ) ?? + /^[^\s()]+ are cooling down \(last error: [\s\S]*\) \(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)$/.exec( + pooledCooldown, + ) + ); +} + const CLAUDE_UNSUPPORTED_VERSION_ERROR = /^API Error: 400 Claude Code (\d+\.\d+\.\d+) does not support this model; version (\d+\.\d+\.\d+) or newer is required\./; @@ -4904,20 +4923,7 @@ function providerFailureFromResult( // API-key sessions get no rate_limit_info, so a gateway's 429 carries its reset only as text. const resetAt = apiErrorStatus === 429 && message.result.startsWith("API Error:") - ? gatewayCooldownResetAt( - // CLIProxyAPI nests upstream errors in its message; only its own cooldown counts. - message.result.startsWith( - "API Error: Request rejected (429) · All credentials for model ", - ) - ? (/^API Error: Request rejected \(429\) · All credentials for model [^\s()]+ are cooling down \(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec( - message.result, - ) ?? - /^API Error: Request rejected \(429\) · All credentials for model [^\s()]+ are cooling down \(last error: [\s\S]*\) \(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)$/.exec( - message.result, - )) - : /\(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec(message.result), - receivedAt, - ) + ? gatewayCooldownResetAt(claude429Cooldown(message.result), receivedAt) : undefined; return makeProviderFailure({ message: listedError ?? structuredError ?? failureHint ?? message.result,