diff --git a/apps/server/src/provider/Drivers/ClaudeDriver.ts b/apps/server/src/provider/Drivers/ClaudeDriver.ts index 5bedffe9242b4..432c60b2d5092 100644 --- a/apps/server/src/provider/Drivers/ClaudeDriver.ts +++ b/apps/server/src/provider/Drivers/ClaudeDriver.ts @@ -210,6 +210,7 @@ export const ClaudeDriver: ProviderDriver(undefined); const capabilitiesProbeCache = yield* Cache.makeWith( () => @@ -282,11 +283,13 @@ export const ClaudeDriver: ProviderDriver - ClaudeResetCredits.readClaudeResetCredits(configDir, version).pipe( - Effect.provideService(HttpClient.HttpClient, httpClient), - Effect.provideService(FileSystem.FileSystem, fileSystem), - Effect.provideService(Path.Path, path), - ), + resetCreditsReader + .read({ configDir, accountConfigPath, version }) + .pipe( + Effect.provideService(HttpClient.HttpClient, httpClient), + Effect.provideService(FileSystem.FileSystem, fileSystem), + Effect.provideService(Path.Path, path), + ), ), ), Effect.tap((provider) => @@ -390,6 +393,7 @@ export const ClaudeDriver: ProviderDriver { }), ); + it.live.skipIf(windowsHost)("keeps Claude reset credits when a later read is rate limited", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const fixtures = yield* makeTildeProviderFixtures(); + yield* fs.writeFileString( + path.join(fixtures.claudeHomePath, ".credentials.json"), + '{"claudeAiOauth":{"accessToken":"fake-token"}}', + ); + let rateLimited = false; + const client = HttpClient.make((request) => + Effect.succeed( + request.url.endsWith("/api/oauth/usage") + ? HttpClientResponse.fromWeb( + request, + rateLimited + ? Response.json({}, { status: 429, headers: { "retry-after": "60" } }) + : Response.json({ + cedar_ember: { + eligible: true, + next_grant_id: "grant_a", + grants: [{ id: "grant_a", resets_left: 1, usable_now: true }], + }, + }), + ) + : HttpClientResponse.fromWeb(request, Response.json({ version: "0.0.0" })), + ), + ); + const instanceId = ProviderInstanceId.make("claude_reset_429"); + const { registry } = yield* makeProviderInstanceRegistry({ + drivers: [ClaudeDriver], + configMap: { + [instanceId]: { + driver: ProviderDriverKind.make("claudeAgent"), + enabled: true, + config: makeClaudeConfig({ + enabled: true, + binaryPath: fixtures.claudeBinaryPath, + homePath: fixtures.claudeHomePath, + }), + }, + }, + }).pipe(Effect.provideService(HttpClient.HttpClient, client)); + const instance = yield* registry.getInstance(instanceId); + const before = yield* instance!.snapshot.refresh; + expect(before.usageLimits?.resetCredits?.nextCreditId).toBe("grant_a"); + + rateLimited = true; + const after = yield* instance!.snapshot.refresh; + expect(after.usageLimits?.windows.length).toBeGreaterThan(0); + expect(after.usageLimits?.resetCredits?.nextCreditId).toBe("grant_a"); + }).pipe(Effect.provideService(HostProcess.Platform, "linux"), Effect.provide(layerTest)), + ); + const makeClaudeCacheFixture = Effect.fnUntraced(function* ( state: "ready" | "signed-out" | "usage-failed", ) { diff --git a/apps/server/src/provider/claudeResetCredits.test.ts b/apps/server/src/provider/claudeResetCredits.test.ts index a3a1110d87d34..bce5c59a3eb90 100644 --- a/apps/server/src/provider/claudeResetCredits.test.ts +++ b/apps/server/src/provider/claudeResetCredits.test.ts @@ -5,6 +5,7 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; +import * as TestClock from "effect/testing/TestClock"; import { HttpClient, HttpClientResponse, UrlParams } from "effect/http"; import { describe, expect, it } from "vite-plus/test"; @@ -153,22 +154,83 @@ effectIt.layer(NodeServices.layer)("readClaudeResetCredits", (it) => { Effect.provideService(HostProcess.Platform, "linux"), Effect.provideService(HttpClient.HttpClient, client), ); - expect(credits).toMatchObject({ availableCount: 1, nextCreditId: "grant_a" }); + expect(credits).toMatchObject({ + _tag: "read", + credits: { availableCount: 1, nextCreditId: "grant_a" }, + }); }), ); - it.effect("reads nothing from keychain logins or failed requests", () => + it.effect("reads nothing from keychain logins or failed requests, and reports rate limits", () => Effect.gen(function* () { const { configDir } = yield* writeLogin; const darwin = yield* ClaudeResetCredits.readClaudeResetCredits(configDir, "2.1.0").pipe( Effect.provideService(HostProcess.Platform, "darwin"), Effect.provideService(HttpClient.HttpClient, refuseRequests), ); + const failed = yield* ClaudeResetCredits.readClaudeResetCredits(configDir, "2.1.0").pipe( + Effect.provideService(HostProcess.Platform, "linux"), + Effect.provideService(HttpClient.HttpClient, respond(500, {})), + ); const limited = yield* ClaudeResetCredits.readClaudeResetCredits(configDir, "2.1.0").pipe( Effect.provideService(HostProcess.Platform, "linux"), Effect.provideService(HttpClient.HttpClient, respond(429, {})), ); - expect([darwin, limited]).toEqual([undefined, undefined]); + expect([darwin, failed, limited]).toEqual([ + { _tag: "read", credits: undefined }, + { _tag: "read", credits: undefined }, + { _tag: "rateLimited", retryAfterMillis: 0 }, + ]); + }), + ); + + it.effect("keeps the last credits through a rate limit and waits out Retry-After", () => + Effect.gen(function* () { + const login = yield* writeLogin; + let limited = false; + let requests = 0; + const client = HttpClient.make((request) => { + requests++; + return Effect.succeed( + HttpClientResponse.fromWeb( + request, + limited + ? Response.json({}, { status: 429, headers: { "retry-after": "60" } }) + : Response.json({ + cedar_ember: { eligible: true, next_grant_id: "grant_a", grants: [grant({})] }, + }), + ), + ); + }); + const reader = yield* ClaudeResetCredits.makeClaudeResetCreditsReader(); + const read = reader + .read({ ...login, version: "2.1.0" }) + .pipe( + Effect.provideService(HostProcess.Platform, "linux"), + Effect.provideService(HttpClient.HttpClient, client), + ); + expect((yield* read)?.nextCreditId).toBe("grant_a"); + limited = true; + expect((yield* read)?.nextCreditId).toBe("grant_a"); + yield* TestClock.adjust("59 seconds"); + expect((yield* read)?.nextCreditId).toBe("grant_a"); + expect(requests).toBe(2); + yield* TestClock.adjust("1 second"); + yield* read; + expect(requests).toBe(3); + yield* reader.forget; + expect(yield* read).toBeUndefined(); + + // Another organization is not held to the previous one's Retry-After. + expect(requests).toBe(3); + const fs = yield* FileSystem.FileSystem; + yield* fs.writeFileString( + login.accountConfigPath, + '{"oauthAccount":{"organizationUuid":"org-2"}}', + ); + limited = false; + expect((yield* read)?.nextCreditId).toBe("grant_a"); + expect(requests).toBe(4); }), ); }); diff --git a/apps/server/src/provider/claudeResetCredits.ts b/apps/server/src/provider/claudeResetCredits.ts index d2000804cf4f6..b2caaeee2eab8 100644 --- a/apps/server/src/provider/claudeResetCredits.ts +++ b/apps/server/src/provider/claudeResetCredits.ts @@ -4,11 +4,13 @@ import type { ServerProviderResetCredits, } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Clock from "effect/Clock"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http"; @@ -207,28 +209,80 @@ const withClaudeHeaders = (token: string, version: string) => "user-agent": `claude-cli/${version} (external, cli)`, }); +// Retry-After is delay-seconds or an HTTP date (RFC 9110 §10.2.3). +const MAX_RETRY_AFTER_MILLIS = 60 * 60 * 1000; +const retryAfterMillis = (value: string | undefined, nowMs: number) => { + const trimmed = value?.trim() ?? ""; + const millis = /^\d+$/.test(trimmed) ? Number(trimmed) * 1000 : Date.parse(trimmed) - nowMs; + return Number.isFinite(millis) && millis > 0 ? Math.min(millis, MAX_RETRY_AFTER_MILLIS) : 0; +}; + +export type ClaudeResetCreditsRead = + | { readonly _tag: "read"; readonly credits: ServerProviderResetCredits | undefined } + | { readonly _tag: "rateLimited"; readonly retryAfterMillis: number }; + +const unread: ClaudeResetCreditsRead = { _tag: "read", credits: undefined }; + export const readClaudeResetCredits = Effect.fn("readClaudeResetCredits")( function* (configDir: string, version: string) { const token = yield* readAccessToken(configDir); - if (!token) return undefined; + if (!token) return unread; const client = yield* HttpClient.HttpClient; const response = yield* client.execute( HttpClientRequest.get(`${API_BASE}/api/oauth/usage`, { urlParams: { cedar_ember: "1", skip_spend: "1" }, }).pipe(withClaudeHeaders(token, version)), ); + const nowMs = yield* Clock.currentTimeMillis; + if (response.status === 429) { + return { + _tag: "rateLimited", + retryAfterMillis: retryAfterMillis(response.headers["retry-after"], nowMs), + } satisfies ClaudeResetCreditsRead; + } const body = yield* HttpClientResponse.schemaBodyJson(UsageResponse)( yield* HttpClientResponse.filterStatusOk(response), ); - return claudeResetCreditsToContract( - body.cedar_ember, - DateTime.toEpochMillis(yield* DateTime.now), - ); + return { + _tag: "read", + credits: claudeResetCreditsToContract(body.cedar_ember, nowMs), + } satisfies ClaudeResetCreditsRead; }, Effect.timeout("10 seconds"), - Effect.orElseSucceed(() => undefined), + Effect.orElseSucceed(() => unread), ); +/** Keeps an account's last credits through a rate limit and skips reads until Retry-After. */ +export const makeClaudeResetCreditsReader = Effect.fn("makeClaudeResetCreditsReader")(function* () { + const known = yield* Ref.make<{ + readonly account?: string | undefined; + readonly credits?: ServerProviderResetCredits | undefined; + readonly retryAtMs: number; + }>({ retryAtMs: 0 }); + const read = Effect.fn("ClaudeResetCreditsReader.read")(function* (input: { + readonly configDir: string; + readonly accountConfigPath: string; + readonly version: string; + }) { + const account = yield* readClaudeOrganizationId(input.accountConfigPath); + const state = yield* Ref.get(known); + const sameAccount = state.account === account; + const kept = sameAccount ? state.credits : undefined; + if (sameAccount && (yield* Clock.currentTimeMillis) < state.retryAtMs) return kept; + const result = yield* readClaudeResetCredits(input.configDir, input.version); + if (result._tag === "read") { + yield* Ref.set(known, { account, credits: result.credits, retryAtMs: 0 }); + return result.credits; + } + const retryAtMs = (yield* Clock.currentTimeMillis) + result.retryAfterMillis; + yield* Ref.set(known, { account, credits: kept, retryAtMs }); + return kept; + }); + /** A redeemed credit is gone; never show it again from memory. */ + const forget = Ref.update(known, ({ account, retryAtMs }) => ({ account, retryAtMs })); + return { read, forget }; +}); + export const readClaudeOauthAccount = (accountConfigPath: string) => readJson(Config, accountConfigPath).pipe( Effect.map((config) => config.oauthAccount),