diff --git a/apps/server/src/orchestration-v2/CommandPolicy.ts b/apps/server/src/orchestration-v2/CommandPolicy.ts index c49d229949a77..0961331fcbd3e 100644 --- a/apps/server/src/orchestration-v2/CommandPolicy.ts +++ b/apps/server/src/orchestration-v2/CommandPolicy.ts @@ -1,5 +1,6 @@ import { type ChatAttachment, + type ModelSelection, CommandId, type OrchestrationV2Command, OrchestrationV2ProviderCapabilities, @@ -16,6 +17,16 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; +// Account choice is preflighted outside the dispatch lock; a raced user action wins. +export const AccountFallback = Context.Reference< + | { + readonly commandId: CommandId; + readonly threadSelection: ModelSelection; + readonly requestedSelection: ModelSelection | undefined; + } + | undefined +>("t3/orchestration-v2/AccountFallback", { defaultValue: () => undefined }); + export const SteeringExecutionPolicyV2 = Schema.Literals(["active_steering", "interrupt_restart"]); export type SteeringExecutionPolicyV2 = typeof SteeringExecutionPolicyV2.Type; diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 6b1fe8567b8e6..7790eff03d8a6 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -1,3 +1,4 @@ +import { AccountFallback } from "./CommandPolicy.ts"; import type { OrchestrationV2ThreadLinkMetadata, GuestShareId, @@ -6878,6 +6879,25 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }); return; } + const fallback = yield* AccountFallback; + const automaticFallback = fallback?.commandId === command.commandId ? fallback : undefined; + if (automaticFallback && command.manualContinuationOfRunId === undefined) { + const mode = resolveMessageDispatchIntent( + projection, + command.dispatchMode, + command.deliveryIntent, + ); + if ( + ((mode.type === "steer_active" || mode.type === "restart_active") && + command.freshContext !== true) || + !modelSelectionsEqual(projection.thread.modelSelection, automaticFallback.threadSelection) + ) { + command = { + ...command, + modelSelection: automaticFallback.requestedSelection ?? projection.thread.modelSelection, + }; + } + } const activeProviderInstanceId = projection.providerThreads.find( (candidate) => candidate.id === projection.thread.activeProviderThreadId, )?.providerInstanceId; @@ -6901,7 +6921,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio latestExecutedRun(projection.runs)?.id !== source.id || projection.thread.archivedAt !== null || projection.thread.deletedAt !== null || - projection.runtimeRequests.some((request) => request.status === "pending") + projection.runtimeRequests.some((request) => request.status === "pending") || + (automaticFallback !== undefined && + (projection.thread.settledOverride === "settled" || + !modelSelectionsEqual(source.modelSelection, automaticFallback.threadSelection) || + !modelSelectionsEqual( + projection.thread.modelSelection, + automaticFallback.threadSelection, + ) || + (projection.thread.snoozedUntil != null && + DateTime.toEpochMillis(projection.thread.snoozedUntil) > + DateTime.toEpochMillis(yield* DateTime.now)) || + (yield* stopReachedRun(command, command.threadId, source.id)))) ) { return yield* new OrchestratorDispatchError({ commandId: command.commandId, diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index 2c667bf43679f..0911bad316b3c 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -8413,6 +8413,8 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { if (lastErrorClass === "usage_limit") { assert.deepEqual(candidate, { id: sqlShell.id, + projectId: sqlShell.projectId, + modelSelection: sqlShell.modelSelection, status: sqlShell.status, lastErrorClass: sqlShell.lastErrorClass, usageLimitResetAt: sqlShell.usageLimitResetAt, @@ -8592,6 +8594,33 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { (row) => row.id === threadId, ), ); + for (const [since, found] of [ + [DateTime.subtract(now, { minutes: 1 }), true], + [DateTime.add(now, { minutes: 1 }), false], + ] as const) { + assert.strictEqual( + (yield* store.getLimitRecoveryCandidates({ + ...recoveryOptions, + switchAccountsSince: since, + })).some((row) => row.id === threadId), + found, + ); + } + yield* sql`UPDATE orchestration_v2_projection_turn_items + SET payload_json = json_remove(payload_json, '$.failure.resetAt') WHERE turn_item_id = ${limitItem.id}`; + assert.isTrue( + (yield* store.getLimitRecoveryCandidates({ + ...recoveryOptions, + switchAccountsSince: DateTime.subtract(now, { minutes: 1 }), + })).some((row) => row.id === threadId), + ); + assert.isFalse( + (yield* store.getLimitRecoveryCandidates({ ...recoveryOptions, autoResume: true })).some( + (row) => row.id === threadId, + ), + ); + yield* sql`UPDATE orchestration_v2_projection_turn_items + SET payload_json = json_set(payload_json, '$.failure.resetAt', ${limitItem.failure.resetAt}) WHERE turn_item_id = ${limitItem.id}`; const reset = DateTime.makeUnsafe(limitItem.failure.resetAt); const recovery = { runId: original.id, @@ -8856,6 +8885,17 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { deletedParentChild, activeParentChild, ]; + assert.deepEqual( + (yield* store.getLimitRecoveryCandidates({ + now, + autoResume: false, + snooze: false, + switchAccountsSince: DateTime.subtract(now, { minutes: 1 }), + })) + .filter((row) => children.includes(row.id)) + .map((row) => row.id), + [activeParentChild], + ); for (const options of [ { now, autoResume: true, snooze: false }, { now, autoResume: false, snooze: true }, diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 5b8a636cc24f1..10fd0661dbcc5 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -184,6 +184,8 @@ export type ProjectionRecoveryKind = export type ProjectionLimitRecoveryCandidate = Pick< OrchestrationV2ThreadShell, | "id" + | "projectId" + | "modelSelection" | "status" | "lastErrorClass" | "latestRunId" @@ -636,6 +638,8 @@ export interface ProjectionStoreV2Shape { readonly now: DateTime.Utc; readonly autoResume: boolean; readonly snooze: boolean; + /** Also returns limits hit since then whose window has not reset, to move to another account. */ + readonly switchAccountsSince?: DateTime.Utc; }) => Effect.Effect, ProjectionStoreV2Error>; /** Epoch ms of the next armed reset or snooze end after `now`, when a waiting thread becomes a candidate. */ readonly getLimitRecoveryWakeAt: ( @@ -4764,11 +4768,28 @@ export const layer: Layer.Layer = AND t.archived_at IS NULL AND json_extract(t.payload_json, '$.settledOverride') IS NOT 'settled' AND json_extract(item.payload_json, '$.failure.class') = 'usage_limit' - AND json_extract(item.payload_json, '$.failure.resetAt') IS NOT NULL - AND julianday(json_extract(item.payload_json, '$.failure.resetAt')) > julianday(COALESCE(r.completed_at, json_extract(t.payload_json, '$.updatedAt'))) AND ( ( - json_extract(t.payload_json, '$.limitRecovery.runId') IS r.run_id + ${booleanInt(options.switchAccountsSince !== undefined)} + AND julianday(COALESCE(r.completed_at, json_extract(t.payload_json, '$.updatedAt'))) >= julianday(${DateTime.formatIso(options.switchAccountsSince ?? options.now)}) + AND ( + json_extract(item.payload_json, '$.failure.resetAt') IS NULL + OR julianday(json_extract(item.payload_json, '$.failure.resetAt')) > julianday(${DateTime.formatIso(options.now)}) + ) + AND NOT EXISTS ( + SELECT 1 FROM orchestration_v2_projection_threads parent + WHERE json_extract(t.payload_json, '$.lineage.relationshipToParent') = 'subagent' + AND parent.thread_id = json_extract(t.payload_json, '$.lineage.parentThreadId') + AND ( + json_extract(parent.payload_json, '$.settledOverride') = 'settled' + OR json_extract(parent.payload_json, '$.archivedAt') IS NOT NULL + OR parent.deleted_at IS NOT NULL + ) + ) + ) + OR ( + julianday(json_extract(item.payload_json, '$.failure.resetAt')) > julianday(COALESCE(r.completed_at, json_extract(t.payload_json, '$.updatedAt'))) + AND json_extract(t.payload_json, '$.limitRecovery.runId') IS r.run_id AND json_extract(t.payload_json, '$.limitRecovery.resetAt') IS json_extract(item.payload_json, '$.failure.resetAt') AND json_extract(t.payload_json, '$.limitRecovery.autoResume') = 1 AND julianday(json_extract(item.payload_json, '$.failure.resetAt')) <= julianday(${DateTime.formatIso(options.now)}) @@ -4778,6 +4799,8 @@ export const layer: Layer.Layer = ) ) OR ( + julianday(json_extract(item.payload_json, '$.failure.resetAt')) > julianday(COALESCE(r.completed_at, json_extract(t.payload_json, '$.updatedAt'))) + AND ( json_extract(t.payload_json, '$.limitRecovery.runId') IS NOT r.run_id OR json_extract(t.payload_json, '$.limitRecovery.resetAt') IS NOT json_extract(item.payload_json, '$.failure.resetAt') @@ -4816,6 +4839,8 @@ export const layer: Layer.Layer = if (summary.lastErrorClass !== "usage_limit") continue; candidates.push({ id: thread.id, + projectId: thread.projectId, + modelSelection: thread.modelSelection, status: "failed", lastErrorClass: summary.lastErrorClass, usageLimitResetAt: summary.usageLimitResetAt, diff --git a/apps/server/src/orchestration-v2/ThreadMessageIntake.test.ts b/apps/server/src/orchestration-v2/ThreadMessageIntake.test.ts index a92cdd22aa515..7c72e4e0eb7d2 100644 --- a/apps/server/src/orchestration-v2/ThreadMessageIntake.test.ts +++ b/apps/server/src/orchestration-v2/ThreadMessageIntake.test.ts @@ -1,4 +1,5 @@ import * as DocumentText from "../provider/DocumentText.ts"; +import { AccountFallback } from "./CommandPolicy.ts"; // @effect-diagnostics nodeBuiltinImport:off import * as NodeFS from "node:fs"; import * as NodePath from "node:path"; @@ -10,6 +11,9 @@ import { EventId, GuestMessageId, MessageId, + ProjectId, + ProviderDriverKind, + ProviderInstanceId, RunId, RuntimeRequestId, ThreadId, @@ -17,6 +21,7 @@ import { type OrchestrationV2Command, type OrchestrationV2ServerCommand, type OrchestrationV2StoredEvent, + type ServerProvider, type UserInputAttachments, } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; @@ -28,6 +33,7 @@ import { resolveAttachmentPath, } from "../attachmentStore.ts"; import * as ServerConfig from "../config.ts"; +import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; import * as ServerSettings from "../serverSettings.ts"; import { OrchestratorCommandPreviouslyRejectedError, @@ -41,6 +47,9 @@ const layerIntakeTest = ServerConfig.layerTest(process.cwd(), { }).pipe( Layer.provideMerge(DocumentText.layer.pipe(Layer.provideMerge(NodeServices.layer))), Layer.provideMerge(ServerSettings.layerTest()), + Layer.provideMerge( + Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }), + ), ); const layerFailingDispatchHandler = @@ -428,6 +437,112 @@ it.effect("sends a guest message with the guest's files, never ones the client n }).pipe(Effect.provide(layerIntakeTest)), ); +it.effect.each([false, true])( + "routes an exhausted account while preserving manual resume: %s", + (manual) => + Effect.gen(function* () { + const claude = (id: string, usedPercent: number): ServerProvider => ({ + instanceId: ProviderInstanceId.make(id), + driver: ProviderDriverKind.make("claudeAgent"), + enabled: true, + installed: true, + version: null, + status: "ready", + auth: { status: "authenticated" }, + checkedAt: "2026-09-13T00:00:00.000Z", + models: [ + { slug: "claude-opus-5-5", name: "Opus 5.5", isCustom: false, capabilities: null }, + ], + slashCommands: [], + skills: [], + usageLimits: { + checkedAt: "2026-09-13T00:00:00.000Z", + windows: [ + { + id: "five_hour", + kind: "session", + label: "Session", + usedPercent, + resetsAt: "2099-01-01T00:00:00.000Z", + }, + ], + }, + }); + const captured: OrchestrationV2ServerCommand[] = []; + const selection = { + instanceId: ProviderInstanceId.make("claude_work"), + model: "claude-opus-5-5", + }; + yield* dispatchCommand({ + type: "message.dispatch", + commandId: CommandId.make("send-limited"), + threadId: ThreadId.make("thread-limited"), + messageId: MessageId.make("message-limited"), + ...(manual ? { manualContinuationOfRunId: RunId.make("limited-run") } : {}), + text: "go", + attachments: [], + dispatchMode: { type: "start_immediately" }, + createdBy: "user", + creationSource: "web", + }).pipe( + Effect.provide( + Layer.mergeAll( + Layer.mock(ThreadManagementService.ThreadManagementService)({ + dispatch: (command) => + AccountFallback.pipe( + Effect.flatMap((fallback) => { + expect(fallback).toEqual( + manual + ? undefined + : { + commandId: command.commandId, + threadSelection: selection, + requestedSelection: undefined, + }, + ); + return layerFailingDispatchHandler(captured)(command); + }), + ), + getThreadRecords: () => + Effect.succeed({ + thread: { + projectId: ProjectId.make("project-limited"), + modelSelection: selection, + }, + } as never), + }), + Layer.mock(ProviderRegistry.ProviderRegistry)({ + getProviders: Effect.succeed([ + claude("claude_work", 100), + claude("claude_personal", 30), + ]), + }), + ServerSettings.layerTest({ + fallbackToAvailableAccounts: true, + providerInstances: Object.fromEntries( + ["claude_work", "claude_personal"].map((id) => [ + ProviderInstanceId.make(id), + { driver: ProviderDriverKind.make("claudeAgent"), enabled: true }, + ]), + ), + }), + ), + ), + Effect.result, + ); + if (manual) { + expect(captured[0]).not.toHaveProperty("modelSelection"); + return; + } + expect(captured[0]).toMatchObject({ + modelSelection: { + instanceId: "claude_personal", + model: "claude-opus-5-5", + }, + }); + }).pipe(Effect.provide(layerIntakeTest)), +); + it.effect("refuses a queued edit that keeps a file another edit already deleted", () => Effect.gen(function* () { const config = yield* ServerConfig.ServerConfig; diff --git a/apps/server/src/orchestration-v2/ThreadMessageIntake.ts b/apps/server/src/orchestration-v2/ThreadMessageIntake.ts index 6e9fd503baec5..2f1c87d64e41e 100644 --- a/apps/server/src/orchestration-v2/ThreadMessageIntake.ts +++ b/apps/server/src/orchestration-v2/ThreadMessageIntake.ts @@ -1,14 +1,24 @@ import { remapComposerContextAttachments } from "@t3tools/shared/composerContextReferences"; import { appendUserInputAttachmentPaths } from "../provider/userInputAttachments.ts"; -import type { ChatAttachment, OrchestrationV2Command } from "@t3tools/contracts"; +import type { + ChatAttachment, + ModelSelection, + OrchestrationV2Command, + ProjectId, +} from "@t3tools/contracts"; +import { availableAccountSelection } from "@t3tools/shared/usageLimits"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Schema from "effect/Schema"; import { resolveAttachmentPath } from "../attachmentStore.ts"; import * as ServerConfig from "../config.ts"; +import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; +import * as ServerSettings from "../serverSettings.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as AttachmentClaims from "./AttachmentClaims.ts"; +import { AccountFallback } from "./CommandPolicy.ts"; import * as ThreadLaunch from "./ThreadLaunchService.ts"; import * as ThreadManagement from "./ThreadManagementService.ts"; @@ -85,11 +95,65 @@ const withGuestAttachments = Effect.fn("ThreadMessageIntake.withGuestAttachments return { ...command, attachments: message?.attachments ?? [] }; }); +/** Null unless the user lets sends move off an account that is out of usage. */ +const readAccountChoice = Effect.fn("ThreadMessageIntake.readAccountChoice")(function* () { + const settings = yield* ServerSettings.ServerSettingsService.pipe( + Effect.flatMap((service) => service.getSettings), + Effect.orElseSucceed(() => null), + ); + if (!settings?.fallbackToAvailableAccounts) return null; + const providers = yield* ProviderRegistry.ProviderRegistry.pipe( + Effect.flatMap((registry) => registry.getProviders), + ); + const now = DateTime.toEpochMillis(yield* DateTime.now); + return (selection: ModelSelection, projectId: ProjectId) => + availableAccountSelection({ selection, providers, now, settings, projectId }) ?? selection; +}); + +const withAvailableAccount = Effect.fn("ThreadMessageIntake.withAvailableAccount")(function* ( + command: Extract, +) { + // Steering stays on the active run's account; server continuations validate their own. + if ( + command.dispatchMode.type === "steer_active" || + command.dispatchMode.type === "restart_active" || + command.manualContinuationOfRunId !== undefined || + command.usageLimitContinuationOfRunId !== undefined || + command.restartContinuationOfRunId !== undefined + ) { + return { command, fallback: undefined }; + } + const choose = yield* readAccountChoice(); + if (choose === null) return { command, fallback: undefined }; + const threads = yield* ThreadManagement.ThreadManagementService; + const records = yield* threads + .getThreadRecords(command.threadId, []) + .pipe(Effect.orElseSucceed(() => null)); + if (records === null) return { command, fallback: undefined }; + const { thread } = records; + const selection = command.modelSelection ?? thread.modelSelection; + const next = choose(selection, thread.projectId); + return { + command: next === selection ? command : { ...command, modelSelection: next }, + fallback: + next === selection + ? undefined + : { + commandId: command.commandId, + threadSelection: thread.modelSelection, + requestedSelection: command.modelSelection, + }, + }; +}); + export const dispatchCommand = Effect.fn("ThreadMessageIntake.dispatchCommand")(function* ( incoming: OrchestrationV2Command, ) { - const command = - incoming.type === "message.dispatch" ? yield* withGuestAttachments(incoming) : incoming; + const prepared = + incoming.type === "message.dispatch" + ? yield* withAvailableAccount(yield* withGuestAttachments(incoming)) + : { command: incoming, fallback: undefined }; + const { command } = prepared; const threads = yield* ThreadManagement.ThreadManagementService; if (command.type === "runtime-request.respond" && command.attachmentsByQuestionId) { const config = yield* ServerConfig.ServerConfig; @@ -197,6 +261,7 @@ export const dispatchCommand = Effect.fn("ThreadMessageIntake.dispatchCommand")( : {}), }) .pipe( + Effect.provideService(AccountFallback, prepared.fallback), Effect.tap((result) => releaseUnusedClaims( claimed.claimedPaths, @@ -229,9 +294,14 @@ export const sendToThread = Effect.fn("ThreadMessageIntake.sendToThread")(functi }); export const launchThread = Effect.fn("ThreadMessageIntake.launchThread")(function* ( - input: ThreadLaunch.ThreadLaunchInput, + requested: ThreadLaunch.ThreadLaunchInput, ) { const launches = yield* ThreadLaunch.ThreadLaunchService; + const choose = yield* readAccountChoice(); + const input = + choose === null + ? requested + : { ...requested, modelSelection: choose(requested.modelSelection, requested.projectId) }; yield* AttachmentClaims.validateAttachmentLimits(input.initialMessage?.attachments ?? []); if (!input.initialMessage?.attachments.some(AttachmentClaims.attachmentIsPendingUpload)) { return yield* launches.launch(input); diff --git a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts index 430018adcb06a..c4f40435edcf5 100644 --- a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts +++ b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts @@ -1,5 +1,14 @@ import { assert, describe, it } from "@effect/vitest"; -import { RunId, ThreadId, type OrchestrationV2DomainEvent } from "@t3tools/contracts"; +import { + DEFAULT_SERVER_SETTINGS, + ProjectId, + ProviderDriverKind, + ProviderInstanceId, + RunId, + ThreadId, + type OrchestrationV2DomainEvent, + type ServerProvider, +} from "@t3tools/contracts"; import * as Clock from "effect/Clock"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -8,13 +17,14 @@ import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; +import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; import * as Scheduler from "../scheduling/Scheduler.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ThreadManagement from "./ThreadManagementService.ts"; import * as UsageLimitRecoveryWorker from "./UsageLimitRecoveryWorker.ts"; -import { limitRecoveryCommand } from "./UsageLimitRecoveryWorker.ts"; +import { limitRecoveryCommand, limitSwitchCommand } from "./UsageLimitRecoveryWorker.ts"; it.effect("idle sweeps skip SQL until the next wake time or a relevant event", () => Effect.gen(function* () { @@ -22,6 +32,7 @@ it.effect("idle sweeps skip SQL until the next wake time or a relevant event", ( const startMs = Date.parse("2026-07-01T09:00:00.000Z"); const queries = yield* Queue.unbounded(); const events = yield* Queue.unbounded(); + const providerChanges = yield* Queue.unbounded>(); const layer = UsageLimitRecoveryWorker.layer.pipe( Layer.provide( Layer.mergeAll( @@ -39,6 +50,10 @@ it.effect("idle sweeps skip SQL until the next wake time or a relevant event", ( ), }), Layer.mock(ThreadManagement.ThreadManagementService)({}), + Layer.mock(ProviderRegistry.ProviderRegistry)({ + getProviders: Effect.succeed([]), + streamChanges: Stream.fromQueue(providerChanges), + }), Layer.mock(Orchestrator.OrchestratorV2)({ streamDomainEvents: Stream.fromQueue(events), }), @@ -66,6 +81,9 @@ it.effect("idle sweeps skip SQL until the next wake time or a relevant event", ( yield* TestClock.adjust("5 seconds"); // A streaming item at 90s does not wake it; the failed run does on the 95s tick. assert.equal(yield* Queue.take(queries), 95_000); + yield* Queue.offer(providerChanges, []); + yield* TestClock.adjust("5 seconds"); + assert.equal(yield* Queue.take(queries), 100_000); }).pipe(Effect.provide(layer)); }), ); @@ -77,6 +95,11 @@ const failedThread = (failedAgoMs: number, resetAgoMs: number) => { const failedAt = DateTime.makeUnsafe(nowMs - failedAgoMs); const candidate: ProjectionStore.ProjectionLimitRecoveryCandidate = { id: ThreadId.make("thread:limited"), + projectId: ProjectId.make("project:limited"), + modelSelection: { + instanceId: ProviderInstanceId.make("claude_work"), + model: "claude-opus-5-5", + }, status: "failed", lastErrorClass: "usage_limit", usageLimitResetAt: DateTime.formatIso(DateTime.makeUnsafe(nowMs - resetAgoMs)), @@ -116,3 +139,170 @@ describe("limitRecoveryCommand arming", () => { assert.isNull(limitRecoveryCommand(thread, true, nowMs)); }); }); + +describe("limitSwitchCommand", () => { + const claude = (id: string, usedPercent: number): ServerProvider => ({ + instanceId: ProviderInstanceId.make(id), + driver: ProviderDriverKind.make("claudeAgent"), + enabled: true, + installed: true, + version: null, + status: "ready", + auth: { status: "authenticated" }, + checkedAt: "2026-10-19T11:00:00.000Z", + models: [{ slug: "claude-opus-5-5", name: "Opus 5.5", isCustom: false, capabilities: null }], + slashCommands: [], + skills: [], + usageLimits: { + checkedAt: "2026-10-19T11:00:00.000Z", + windows: [{ id: "five_hour", kind: "session", label: "Session", usedPercent }], + }, + }); + const settings = { + ...DEFAULT_SERVER_SETTINGS, + fallbackToAvailableAccounts: true, + providerInstances: { + [ProviderInstanceId.make("claude_work")]: { + driver: ProviderDriverKind.make("claudeAgent"), + enabled: true, + }, + [ProviderInstanceId.make("claude_personal")]: { + driver: ProviderDriverKind.make("claudeAgent"), + enabled: true, + }, + }, + }; + // The work account's bars are stale: the turn that just failed is the proof it is spent. + const providers = [claude("claude_work", 40), claude("claude_personal", 20)]; + const input = { providers, settings, nowMs, limitedUntil: new Map() }; + const justLimited = failedThread(60_000, -HOUR_MS); + + it("continues the same model on another account with usage left", () => { + assert.deepInclude(limitSwitchCommand(justLimited, input), { + type: "message.dispatch", + manualContinuationOfRunId: RunId.make("run:limited"), + modelSelection: { + instanceId: ProviderInstanceId.make("claude_personal"), + model: "claude-opus-5-5", + }, + }); + }); + + it.each([ + ["the setting is off", justLimited, { ...input, settings: DEFAULT_SERVER_SETTINGS }], + ["the limit was hit long ago", failedThread(HOUR_MS, -HOUR_MS), input], + [ + "the other account hit its own limit here and has not reset", + justLimited, + { + ...input, + limitedUntil: new Map([[ProviderInstanceId.make("claude_personal"), nowMs + HOUR_MS]]), + }, + ], + ])("waits when %s", (_name, thread, options) => { + assert.isNull(limitSwitchCommand(thread, options)); + }); +}); + +it.effect("wakes for account exclusion expiry and keeps expired resets excluded", () => + Effect.gen(function* () { + const start = Date.parse("2026-10-19T12:00:00.000Z"); + yield* TestClock.setTime(start); + const dispatched = yield* Queue.unbounded(); + const queried = yield* Queue.unbounded(); + const first = failedThread(60_000, -30_000); + const second = { + ...failedThread(60_000, 1), + id: ThreadId.make("thread:second"), + latestRunId: RunId.make("run:second"), + modelSelection: { + ...first.modelSelection, + instanceId: ProviderInstanceId.make("claude_personal"), + }, + }; + const provider = (id: string): ServerProvider => ({ + instanceId: ProviderInstanceId.make(id), + driver: ProviderDriverKind.make("claudeAgent"), + enabled: true, + installed: true, + version: null, + status: "ready", + auth: { status: "authenticated" }, + checkedAt: "2026-10-19T12:00:00.000Z", + models: [{ slug: "claude-opus-5-5", name: "Opus", isCustom: false, capabilities: null }], + slashCommands: [], + skills: [], + usageLimits: { + checkedAt: "2026-10-19T12:00:00.000Z", + windows: [{ id: "five_hour", kind: "session", label: "Session", usedPercent: 20 }], + }, + }); + const layer = UsageLimitRecoveryWorker.layer.pipe( + Layer.provide( + Layer.mergeAll( + Scheduler.layer, + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getLimitRecoveryCandidates: () => + Clock.currentTimeMillis.pipe( + Effect.flatMap((now) => + Queue.offer(queried, now).pipe( + Effect.as([ + now === start + ? first + : now < start + 35_000 + ? second + : { + ...first, + id: ThreadId.make("thread:third"), + latestRunId: RunId.make("run:third"), + }, + ]), + ), + ), + ), + getLimitRecoveryWakeAt: () => Effect.succeed(null), + }), + Layer.mock(ThreadManagement.ThreadManagementService)({ + dispatch: (command) => + command.type === "message.dispatch" + ? Queue.offer(dispatched, command.threadId).pipe(Effect.as({} as never)) + : Effect.die("Expected an account-switch message dispatch"), + }), + Layer.mock(ProviderRegistry.ProviderRegistry)({ + getProviders: Effect.succeed([provider("claude_work"), provider("claude_personal")]), + streamChanges: Stream.never, + }), + Layer.mock(Orchestrator.OrchestratorV2)({ streamDomainEvents: Stream.never }), + Layer.mock(ServerSettings.ServerSettingsService)({ + getSettings: Effect.succeed({ + ...DEFAULT_SERVER_SETTINGS, + providerInstances: Object.fromEntries( + ["claude_work", "claude_personal"].map((id) => [ + id, + { driver: ProviderDriverKind.make("claudeAgent"), enabled: true }, + ]), + ), + fallbackToAvailableAccounts: true, + autoResumeLimitedThreads: false, + snoozeLimitedThreads: false, + }), + streamChanges: Stream.never, + }), + ), + ), + ); + yield* Effect.gen(function* () { + assert.equal(yield* Queue.take(dispatched), first.id); + assert.equal(yield* Queue.take(queried), start); + yield* TestClock.adjust("10 seconds"); + assert.deepEqual(yield* Queue.takeAll(queried), [start + 5_000]); + assert.equal(yield* Queue.size(dispatched), 0); + yield* TestClock.adjust("20 seconds"); + assert.deepEqual(yield* Queue.takeAll(dispatched), [second.id]); + assert.deepEqual(yield* Queue.takeAll(queried), [start + 30_000]); + yield* TestClock.adjust("5 seconds"); + assert.deepEqual(yield* Queue.takeAll(queried), [start + 35_000]); + assert.equal(yield* Queue.size(dispatched), 0); + }).pipe(Effect.provide(layer)); + }), +); diff --git a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts index 3992d46e6bf9a..f95e63ebdcaed 100644 --- a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts +++ b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts @@ -1,4 +1,12 @@ -import { CommandId, MessageId, type OrchestrationV2Command } from "@t3tools/contracts"; +import { + CommandId, + MessageId, + type OrchestrationV2Command, + type ProviderInstanceId, + type ServerProvider, + type ServerSettings as ServerSettingsValue, +} from "@t3tools/contracts"; +import { availableAccountSelection } from "@t3tools/shared/usageLimits"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -6,13 +14,70 @@ import * as Stream from "effect/Stream"; import type { OrchestrationV2DomainEvent } from "@t3tools/contracts"; import * as DueHint from "../scheduling/DueHint.ts"; import * as Scheduler from "../scheduling/Scheduler.ts"; +import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; import { forkParked } from "../serverActivation.ts"; import * as ServerSettings from "../serverSettings.ts"; +import { AccountFallback } from "./CommandPolicy.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ThreadManagement from "./ThreadManagementService.ts"; export const MAX_OVERDUE_ARM_MS = 24 * 60 * 60 * 1000; +// Only limits hit this recently move accounts, so turning the setting on never wakes old threads. +const ACCOUNT_SWITCH_WINDOW_MS = 10 * 60 * 1000; + +/** Continues a thread that just hit a usage limit on another account with usage left. */ +export function limitSwitchCommand( + thread: ProjectionStore.ProjectionLimitRecoveryCandidate, + input: { + readonly providers: ReadonlyArray; + readonly settings: ServerSettingsValue; + readonly nowMs: number; + /** Accounts that hit a limit here until the given reset, whatever their stale bars say. */ + readonly limitedUntil: ReadonlyMap; + }, +): OrchestrationV2Command | null { + if ( + thread.status !== "failed" || + thread.lastErrorClass !== "usage_limit" || + !thread.latestRunId || + thread.archivedAt !== null || + thread.settledOverride === "settled" || + thread.pendingRuntimeRequest !== null || + input.nowMs - DateTime.toEpochMillis(thread.latestRunCompletedAt ?? thread.updatedAt) > + ACCOUNT_SWITCH_WINDOW_MS || + (thread.snoozedUntil != null && DateTime.toEpochMillis(thread.snoozedUntil) > input.nowMs) + ) + return null; + const selection = thread.modelSelection; + const next = availableAccountSelection({ + selection, + providers: input.providers.filter( + (provider) => + provider.instanceId === selection.instanceId || + (input.limitedUntil.get(provider.instanceId) ?? 0) <= input.nowMs, + ), + now: input.nowMs, + settings: input.settings, + projectId: thread.projectId, + exhausted: true, + }); + if (next === null) return null; + const identity = `${thread.id}:${thread.latestRunId}`; + return { + type: "message.dispatch", + commandId: CommandId.make(`limit-switch:${identity}`), + messageId: MessageId.make(`limit-switch:${identity}`), + threadId: thread.id, + manualContinuationOfRunId: thread.latestRunId, + modelSelection: next, + text: "Continue where you left off.", + attachments: [], + dispatchMode: { type: "start_immediately" }, + createdBy: "user", + creationSource: "server", + }; +} export function limitRecoveryCommand( thread: ProjectionStore.ProjectionLimitRecoveryCandidate, @@ -92,6 +157,10 @@ const makeSweep = Effect.gen(function* () { const threads = yield* ThreadManagement.ThreadManagementService; const orchestrator = yield* Orchestrator.OrchestratorV2; const settings = yield* ServerSettings.ServerSettingsService; + const registry = yield* ProviderRegistry.ProviderRegistry; + const limitedUntil = new Map(); + // A rejected switch is not retried every tick; the thread falls back to waiting for the reset. + const attemptedSwitches = new Set(); const due = DueHint.make(); yield* forkParked( Stream.runForEach(orchestrator.streamDomainEvents, (event) => @@ -99,6 +168,7 @@ const makeSweep = Effect.gen(function* () { ), ); yield* forkParked(Stream.runForEach(settings.streamChanges, () => due.invalidate)); + yield* forkParked(Stream.runForEach(registry.streamChanges, () => due.invalidate)); return Effect.fn("UsageLimitRecoveryWorker.sweep")(function* () { const now = yield* DateTime.now; const nowMs = DateTime.toEpochMillis(now); @@ -109,9 +179,45 @@ const makeSweep = Effect.gen(function* () { now, autoResume: preferences.autoResumeLimitedThreads, snooze: preferences.snoozeLimitedThreads, + ...(preferences.fallbackToAvailableAccounts + ? { switchAccountsSince: DateTime.makeUnsafe(nowMs - ACCOUNT_SWITCH_WINDOW_MS) } + : {}), }); + const providers = preferences.fallbackToAvailableAccounts ? yield* registry.getProviders : []; let acted = false; for (const thread of candidates) { + const switchCommand = preferences.fallbackToAvailableAccounts + ? limitSwitchCommand(thread, { providers, settings: preferences, nowMs, limitedUntil }) + : null; + if (switchCommand !== null && !attemptedSwitches.has(switchCommand.commandId)) { + attemptedSwitches.add(switchCommand.commandId); + acted = true; + const resetMs = Date.parse(thread.usageLimitResetAt ?? ""); + yield* threads.dispatch(switchCommand).pipe( + Effect.provideService(AccountFallback, { + commandId: switchCommand.commandId, + threadSelection: thread.modelSelection, + requestedSelection: undefined, + }), + Effect.tap(() => + Effect.sync(() => + limitedUntil.set( + thread.modelSelection.instanceId, + Number.isFinite(resetMs) && resetMs > nowMs + ? resetMs + : nowMs + ACCOUNT_SWITCH_WINDOW_MS, + ), + ), + ), + Effect.catchCause((cause) => + Effect.logWarning("orchestration-v2.limit-recovery.account-switch-failed", { + threadId: thread.id, + cause, + }), + ), + ); + continue; + } const command = limitRecoveryCommand( thread, preferences.autoResumeLimitedThreads, @@ -142,7 +248,10 @@ const makeSweep = Effect.gen(function* () { // A sweep that acted, or failed to, re-checks on the next tick. if (acted) return; const wakeAt = yield* projections.getLimitRecoveryWakeAt(now); - yield* due.record(readVersion, wakeAt ?? Infinity, nowMs); + const accountWakeAt = preferences.fallbackToAvailableAccounts + ? Math.min(...Array.from(limitedUntil.values()).filter((expiry) => expiry > nowMs)) + : Infinity; + yield* due.record(readVersion, Math.min(wakeAt ?? Infinity, accountWakeAt), nowMs); }); }); diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index 82fd3381e20d9..0313e2de1a456 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -8,6 +8,7 @@ import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; import { vi } from "vite-plus/test"; +import { AccountFallback } from "./CommandPolicy.ts"; import { type ApplicationStoredEvent, CheckpointId, @@ -1876,179 +1877,200 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { }).pipe(Effect.provide(Layer.fresh(layerTest))), ); - it.effect("resolves delivery intent against the active run and starts after it completes", () => - Effect.gen(function* () { - const orchestrator = yield* Orchestrator.OrchestratorV2; - const eventSink = yield* EventSink.EventSinkV2; - const outbox = yield* EffectOutbox.EffectOutboxV2; - const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; - const threadId = ThreadId.make("runtime-delivery-intent"); - yield* orchestrator.dispatch({ - type: "thread.create", - createdBy: "user", - creationSource: "web", - commandId: CommandId.make("runtime-delivery-intent-create"), - threadId, - projectId: ProjectId.make("runtime-delivery-intent-project"), - title: "Delivery intent", - modelSelection, - runtimeMode: "full-access", - interactionMode: "default", - branch: null, - worktreePath: process.cwd(), - }); - yield* orchestrator.dispatch({ - type: "message.dispatch", - createdBy: "user", - creationSource: "web", - commandId: CommandId.make("runtime-delivery-intent-first"), - threadId, - messageId: MessageId.make("runtime-delivery-intent-first"), - text: "Start work.", - attachments: [], - dispatchMode: { type: "start_immediately" }, - }); - const initial = yield* orchestrator.getThreadProjection(threadId); - const run = initial.runs[0]!; - const providerThread = initial.providerThreads[0]!; - const now = yield* DateTime.now; - const providerSession = { - id: providerThread.providerSessionId!, - driver, - providerInstanceId: modelSelection.instanceId, - status: "running" as const, - cwd: process.cwd(), - model: modelSelection.model, - capabilities: CodexProviderCapabilitiesV2, - createdAt: now, - updatedAt: now, - lastError: null, - }; - const providerTurn = { - id: ProviderTurnId.make("runtime-delivery-intent-turn"), - providerThreadId: providerThread.id, - nodeId: run.rootNodeId!, - runAttemptId: run.activeAttemptId, - nativeTurnRef: null, - ordinal: 1, - status: "running" as const, - startedAt: now, - completedAt: null, - }; - yield* eventSink.write({ - commandId: CommandId.make("runtime-delivery-intent-running"), - events: [ - { - id: EventId.make("runtime-delivery-intent-run-event"), - type: "run.updated", - threadId, - runId: run.id, - occurredAt: now, - payload: { ...run, status: "running", startedAt: now }, - }, - { - id: EventId.make("runtime-delivery-intent-session-event"), - type: "provider-session.attached", - threadId, - occurredAt: now, - payload: providerSession, - }, - { - id: EventId.make("runtime-delivery-intent-turn-event"), - type: "provider-turn.updated", - threadId, - runId: run.id, - occurredAt: now, - payload: providerTurn, - }, - ], - }); - const sessionSpy = vi.spyOn(sessions, "get").mockReturnValue( - Effect.succeedSome({ - providerSession, - } as ProviderAdapter.ProviderAdapterV2SessionRuntime), - ); - yield* Effect.addFinalizer(() => Effect.sync(() => sessionSpy.mockRestore())); + it.effect.each([false, true])( + "resolves delivery intent with fresh context %s against the active run", + (freshContext) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; + const threadId = ThreadId.make(`runtime-delivery-intent-${freshContext}`); + yield* orchestrator.dispatch({ + type: "thread.create", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(`runtime-delivery-intent-create-${freshContext}`), + threadId, + projectId: ProjectId.make(`runtime-delivery-intent-project-${freshContext}`), + title: "Delivery intent", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: process.cwd(), + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(`runtime-delivery-intent-first-${freshContext}`), + threadId, + messageId: MessageId.make(`runtime-delivery-intent-first-${freshContext}`), + text: "Start work.", + attachments: [], + dispatchMode: { type: "start_immediately" }, + }); + const initial = yield* orchestrator.getThreadProjection(threadId); + const run = initial.runs[0]!; + const providerThread = initial.providerThreads[0]!; + const now = yield* DateTime.now; + const providerSession = { + id: providerThread.providerSessionId!, + driver, + providerInstanceId: modelSelection.instanceId, + status: "running" as const, + cwd: process.cwd(), + model: modelSelection.model, + capabilities: CodexProviderCapabilitiesV2, + createdAt: now, + updatedAt: now, + lastError: null, + }; + const providerTurn = { + id: ProviderTurnId.make(`runtime-delivery-intent-turn-${freshContext}`), + providerThreadId: providerThread.id, + nodeId: run.rootNodeId!, + runAttemptId: run.activeAttemptId, + nativeTurnRef: null, + ordinal: 1, + status: "running" as const, + startedAt: now, + completedAt: null, + }; + yield* eventSink.write({ + commandId: CommandId.make(`runtime-delivery-intent-running-${freshContext}`), + events: [ + { + id: EventId.make(`runtime-delivery-intent-run-event-${freshContext}`), + type: "run.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: { ...run, status: "running", startedAt: now }, + }, + { + id: EventId.make(`runtime-delivery-intent-session-event-${freshContext}`), + type: "provider-session.attached", + threadId, + occurredAt: now, + payload: providerSession, + }, + { + id: EventId.make(`runtime-delivery-intent-turn-event-${freshContext}`), + type: "provider-turn.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: providerTurn, + }, + ], + }); + const sessionSpy = vi.spyOn(sessions, "get").mockReturnValue( + Effect.succeedSome({ + providerSession, + } as ProviderAdapter.ProviderAdapterV2SessionRuntime), + ); + yield* Effect.addFinalizer(() => Effect.sync(() => sessionSpy.mockRestore())); - const steerCommandId = CommandId.make("runtime-delivery-intent-auto"); - const steerMessageId = MessageId.make("runtime-delivery-intent-auto"); - yield* orchestrator.dispatch({ - type: "message.dispatch", - createdBy: "user", - creationSource: "web", - commandId: steerCommandId, - threadId, - messageId: steerMessageId, - text: "Include this in the active work.", - attachments: [], - dispatchMode: { type: "start_immediately" }, - deliveryIntent: "auto", - }); - const steered = yield* orchestrator.getThreadProjection(threadId); - assert.lengthOf(steered.runs, 1); - assert.equal( - steered.messages.find((message) => message.id === steerMessageId)?.runId, - run.id, - ); - assert.deepEqual( - (yield* outbox.listByCommandId(steerCommandId)).map((effect) => effect.request), - [ - { - type: "provider-turn.steer", - providerSessionId: providerSession.id, - providerThreadId: providerThread.id, - providerTurnId: providerTurn.id, + const steerCommandId = CommandId.make(`runtime-delivery-intent-auto-${freshContext}`); + const steerMessageId = MessageId.make(`runtime-delivery-intent-auto-${freshContext}`); + yield* orchestrator + .dispatch({ + type: "message.dispatch", + createdBy: "user", + creationSource: "web", + commandId: steerCommandId, + threadId, messageId: steerMessageId, - modelSelection: run.modelSelection, - steerFollowUpAfterRunOrdinal: run.ordinal, - }, - ], - ); + text: "Include this in the active work.", + freshContext, + attachments: [], + dispatchMode: { type: "start_immediately" }, + deliveryIntent: "auto", + modelSelection: { + ...modelSelection, + instanceId: alternateInstanceId, + }, + }) + .pipe( + Effect.provideService(AccountFallback, { + commandId: steerCommandId, + threadSelection: modelSelection, + requestedSelection: undefined, + }), + ); + const steered = yield* orchestrator.getThreadProjection(threadId); + if (freshContext) { + const queued = steered.runs.find((run) => run.status === "queued"); + assert.equal(queued?.modelSelection.instanceId, alternateInstanceId); + assert.equal(queued?.freshContext, true); + return; + } + assert.lengthOf(steered.runs, 1); + assert.equal( + steered.messages.find((message) => message.id === steerMessageId)?.runId, + run.id, + ); + assert.deepEqual( + (yield* outbox.listByCommandId(steerCommandId)).map((effect) => effect.request), + [ + { + type: "provider-turn.steer", + providerSessionId: providerSession.id, + providerThreadId: providerThread.id, + providerTurnId: providerTurn.id, + messageId: steerMessageId, + modelSelection: run.modelSelection, + steerFollowUpAfterRunOrdinal: run.ordinal, + }, + ], + ); - yield* eventSink.write({ - commandId: CommandId.make("runtime-delivery-intent-completed"), - events: [ - { - id: EventId.make("runtime-delivery-intent-run-completed"), - type: "run.updated", - threadId, - runId: run.id, - occurredAt: now, - payload: { ...run, status: "completed", startedAt: now, completedAt: now }, - }, - { - id: EventId.make("runtime-delivery-intent-turn-completed"), - type: "provider-turn.updated", - threadId, - runId: run.id, - occurredAt: now, - payload: { ...providerTurn, status: "completed", completedAt: now }, - }, - ], - }); - const nextCommandId = CommandId.make("runtime-delivery-intent-next"); - yield* orchestrator.dispatch({ - type: "message.dispatch", - createdBy: "user", - creationSource: "web", - commandId: nextCommandId, - threadId, - messageId: MessageId.make("runtime-delivery-intent-next"), - text: "The previous run finished before this arrived.", - attachments: [], - dispatchMode: { type: "start_immediately" }, - deliveryIntent: "restart", - }); - const restarted = yield* orchestrator.getThreadProjection(threadId); - assert.deepEqual( - restarted.runs.map((candidate) => candidate.status), - ["completed", "starting"], - ); - assert.deepEqual( - (yield* outbox.listByCommandId(nextCommandId)).map((effect) => effect.request.type), - ["provider-turn.start"], - ); - }), + yield* eventSink.write({ + commandId: CommandId.make(`runtime-delivery-intent-completed-${freshContext}`), + events: [ + { + id: EventId.make(`runtime-delivery-intent-run-completed-${freshContext}`), + type: "run.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: { ...run, status: "completed", startedAt: now, completedAt: now }, + }, + { + id: EventId.make(`runtime-delivery-intent-turn-completed-${freshContext}`), + type: "provider-turn.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: { ...providerTurn, status: "completed", completedAt: now }, + }, + ], + }); + const nextCommandId = CommandId.make(`runtime-delivery-intent-next-${freshContext}`); + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "user", + creationSource: "web", + commandId: nextCommandId, + threadId, + messageId: MessageId.make(`runtime-delivery-intent-next-${freshContext}`), + text: "The previous run finished before this arrived.", + attachments: [], + dispatchMode: { type: "start_immediately" }, + deliveryIntent: "restart", + }); + const restarted = yield* orchestrator.getThreadProjection(threadId); + assert.deepEqual( + restarted.runs.map((candidate) => candidate.status), + ["completed", "starting"], + ); + assert.deepEqual( + (yield* outbox.listByCommandId(nextCommandId)).map((effect) => effect.request.type), + ["provider-turn.start"], + ); + }), ); // regression: #37412 one thread's busy lock held up every other thread's run-end reaction @@ -12708,6 +12730,32 @@ it.layer(layerTest)("usage-limit recovery", (it) => { yield* orchestrator.dispatch(freshResume!); assert.lengthOf((yield* orchestrator.getThreadProjection(threadId)).runs, 2); } + if (["stop", "settled", "archive", "replacement"].includes(scenario)) { + const switchId = CommandId.make(`account-switch:${scenario}`); + const result = yield* orchestrator + .dispatch({ + type: "message.dispatch", + commandId: switchId, + threadId, + messageId: MessageId.make(switchId), + manualContinuationOfRunId: run.id, + text: "Continue where you left off.", + attachments: [], + dispatchMode: { type: "start_immediately" }, + createdBy: "user", + creationSource: "server", + }) + .pipe( + Effect.provideService(AccountFallback, { + commandId: switchId, + threadSelection: modelSelection, + requestedSelection: undefined, + }), + Effect.exit, + ); + assert.equal(result._tag, "Failure"); + assert.lengthOf((yield* orchestrator.getThreadProjection(threadId)).runs, 1); + } const before = yield* orchestrator.getThreadProjection(threadId); if (resume !== null) { yield* orchestrator.dispatch(resume); diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index bbe80f28e9681..08511ad68469d 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -1445,6 +1445,7 @@ export const layerWsRpc = ( | FileSystem.FileSystem | ServerConfig.ServerConfig | ServerSettings.ServerSettingsService + | ProviderRegistry.ProviderRegistry | DocumentText.DocumentText >(); const applicationEvents = yield* OrchestrationEventStore.OrchestrationEventStore; diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index e69de418cd3ee..db6a2d7a6cb7a 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -2651,6 +2651,9 @@ export default function ChatView(props: ChatViewProps) { const fallbackWindow = fallback ? `${fallback.provider.instanceId}:${fallback.resetsAt ?? ""}` : null; + const fallbackProvider = providerStatuses.find( + (provider) => provider.instanceId === fallbackSelection?.instanceId, + ); const fallbackItems: ComposerBannerStackItem[] = fallback && fallbackSelection && @@ -2660,10 +2663,9 @@ export default function ChatView(props: ChatViewProps) { modelFallbackBannerItem( { fallbackModelName: - providerStatuses - .find((provider) => provider.instanceId === fallbackSelection.instanceId) - ?.models.find((model) => model.slug === fallbackSelection.model)?.name ?? - fallbackSelection.model, + fallbackProvider?.models.find((model) => model.slug === fallbackSelection.model) + ?.name ?? fallbackSelection.model, + fallbackProviderName: `${fallbackProvider?.displayName?.trim() || fallbackProvider?.driver || "Account"} [${fallbackSelection.instanceId}]`, defaultProviderName: fallback.provider.displayName ?? fallback.provider.driver, defaultModelName: fallback.provider.models.find((model) => model.slug === defaultSelection.model) diff --git a/apps/web/src/components/chat/ComposerUsageLimits.tsx b/apps/web/src/components/chat/ComposerUsageLimits.tsx index 5e2ed683225b9..bbd650a5a4c9e 100644 --- a/apps/web/src/components/chat/ComposerUsageLimits.tsx +++ b/apps/web/src/components/chat/ComposerUsageLimits.tsx @@ -80,6 +80,7 @@ export function usageLimitsBannerItem( export function modelFallbackBannerItem( { fallbackModelName, + fallbackProviderName, defaultProviderName, defaultModelName, resetsAt, @@ -88,6 +89,7 @@ export function modelFallbackBannerItem( onDismiss, }: { readonly fallbackModelName: string; + readonly fallbackProviderName: string; readonly defaultProviderName: string; readonly defaultModelName: string; readonly resetsAt: string | null; @@ -101,7 +103,12 @@ export function modelFallbackBannerItem( id: "fallback-model", variant: "info", icon: , - title: resolve(msg("Using {model}", { model: fallbackModelName })), + title: resolve( + msg("Using {model} on {provider}", { + model: fallbackModelName, + provider: fallbackProviderName, + }), + ), description: resetsAt ? resolve( msg("{provider} is out of usage · resets {time}", { diff --git a/apps/web/src/components/settings/ProjectDefaultsSettings.tsx b/apps/web/src/components/settings/ProjectDefaultsSettings.tsx index 49d9774553822..65e27583e4613 100644 --- a/apps/web/src/components/settings/ProjectDefaultsSettings.tsx +++ b/apps/web/src/components/settings/ProjectDefaultsSettings.tsx @@ -110,6 +110,7 @@ export function ProjectDefaultsSettings({ category }: { category: ProjectSetting const mixedWorkspace = useScopedSettingsMixed(["defaultThreadEnvMode"]); const mixedAutoPull = useScopedSettingsMixed(["defaultAutoPull"]); const mixedAgentCredits = useScopedSettingsMixed(["removeAgentCreditsOnMerge"]); + const mixedAvailableAccounts = useScopedSettingsMixed(["fallbackToAvailableAccounts"]); const mixedMergeMethod = useScopedSettingsMixed(["pullRequestMergeMethod"]); const modelSource = useScopedSettingSource(["defaultModelSelection"]); const isProjectScope = scope.kind === "project" || scope.kind === "checkout"; @@ -349,6 +350,33 @@ export function ProjectDefaultsSettings({ category }: { category: ProjectSetting } /> ) : null; + const availableAccountsRow = !isProjectScope ? ( + updateSettings({ fallbackToAvailableAccounts: false })} + /> + ) : null + } + control={ + updateSettings({ fallbackToAvailableAccounts: enabled })} + /> + } + /> + ) : null; return ( {modelRow} {fallbackRow} + {availableAccountsRow} {workspaceRow} ) : category === "general" ? ( <> {modelRow} {fallbackRow} + {availableAccountsRow} { const selection = { instanceId: ProviderInstanceId.make("codex"), model: "gpt-6.1-sol" }; const fallback = { instanceId: ProviderInstanceId.make("claude"), model: "claude-fable-5-1" }; const limited = provider({ + auth: { status: "authenticated", email: "quota@example.com", workspaceId: "workspace_a" }, usageLimits: { checkedAt: "2026-09-03T11:00:00.000Z", windows: [{ ...window, usedPercent: 100 }], @@ -112,6 +113,148 @@ describe("new thread model fallback", () => { }).selection, ).toEqual(selection); }); + describe("without a usable configured fallback", () => { + const limits = (usedPercent: number) => ({ + checkedAt: limited.checkedAt, + windows: [{ ...window, usedPercent }], + }); + const model = (slug: string, isDefault = false) => ({ + slug, + name: slug, + isCustom: false, + isDefault, + capabilities: null, + }); + const codexAccount = (id: string, usedPercent: number) => + provider({ + instanceId: ProviderInstanceId.make(id), + models: [model("gpt-6.1-luna", true), model(selection.model)], + usageLimits: limits(usedPercent), + }); + const claude = provider({ + instanceId: fallback.instanceId, + driver: ProviderDriverKind.make("claudeAgent"), + models: [model("claude-sonnet-5-5"), model("claude-opus-5-5", true)], + usageLimits: limits(0), + }); + const resolve = (providers: ServerProvider[], fallbackToAvailableAccounts = true) => + resolveNewThreadModelSelection({ + defaultSelection: { ...selection, options: [{ id: "reasoningEffort", value: "high" }] }, + fallbackSelection: null, + providers: [limited, ...providers], + settings: { + ...DEFAULT_SERVER_SETTINGS, + fallbackToAvailableAccounts, + providerInstances: Object.fromEntries( + providers.map((entry) => [ + entry.instanceId, + { driver: entry.driver, enabled: true, config: {} }, + ]), + ), + }, + now, + }).selection; + + it.each([ + [ + "another account of the same provider with the most headroom, keeping model and options", + [claude, codexAccount("codex_a", 90), codexAccount("codex_b", 20)], + { + instanceId: ProviderInstanceId.make("codex_b"), + model: selection.model, + options: [{ id: "reasoningEffort", value: "high" }], + }, + ], + [ + "a distinct login instead of another instance sharing the exhausted account", + [ + { + ...codexAccount("codex_duplicate", 0), + auth: { + status: "authenticated", + email: "QUOTA@example.com", + workspaceId: "workspace_a", + }, + }, + codexAccount("codex_other", 20), + ], + { instanceId: ProviderInstanceId.make("codex_other"), model: selection.model }, + ], + [ + "a different workspace on the same login", + [ + { + ...codexAccount("codex_workspace", 0), + auth: { + status: "authenticated", + email: "quota@example.com", + workspaceId: "workspace_b", + }, + }, + codexAccount("codex_other", 20), + ], + { instanceId: ProviderInstanceId.make("codex_workspace"), model: selection.model }, + ], + [ + "the requested model before a same-provider default with more headroom", + [ + { ...codexAccount("codex_other", 0), models: [model("gpt-6.1-luna", true)] }, + codexAccount("codex_same", 80), + ], + { + instanceId: ProviderInstanceId.make("codex_same"), + model: selection.model, + options: [{ id: "reasoningEffort", value: "high" }], + }, + ], + [ + "an account whose stored window already reset", + [ + { + ...codexAccount("codex_reset", 100), + usageLimits: { + ...limits(100), + windows: [{ ...window, usedPercent: 100, resetsAt: "2026-09-03T11:00:00.000Z" }], + }, + }, + codexAccount("codex_partial", 20), + ], + { instanceId: ProviderInstanceId.make("codex_reset"), model: selection.model }, + ], + [ + "headroom for the candidate model instead of unrelated pools", + [ + { + ...codexAccount("codex_model", 0), + usageLimits: { + ...limits(0), + windows: [ + { ...window, usedPercent: 0, modelIds: [selection.model] }, + { ...window, id: "other", usedPercent: 100, modelIds: ["gpt-6.1-luna"] }, + ], + }, + }, + codexAccount("codex_partial", 20), + ], + { instanceId: ProviderInstanceId.make("codex_model"), model: selection.model }, + ], + [ + "another provider's default model when every same-provider account is spent", + [codexAccount("codex_a", 100), claude], + { instanceId: claude.instanceId, model: "claude-opus-5-5" }, + ], + [ + "nothing when no other account reports limits", + [{ ...claude, usageLimits: undefined }], + selection, + ], + ] as const)("switches to %s", (_name, providers, expected) => { + expect(resolve([...providers])).toMatchObject(expected); + }); + it("stays on the default while the setting is off", () => { + expect(resolve([codexAccount("codex_a", 0), claude], false)).toMatchObject(selection); + }); + }); it.each([undefined, "invalid"])("treats an unknown reset as exhausted: %s", (resetsAt) => { expect( exhaustedUsageWindow( diff --git a/packages/shared/src/usageLimits.ts b/packages/shared/src/usageLimits.ts index 693d82b956971..40471060f4688 100644 --- a/packages/shared/src/usageLimits.ts +++ b/packages/shared/src/usageLimits.ts @@ -23,97 +23,192 @@ import { isLocalLoopbackHost, isWildcardHost } from "./hostClassification.ts"; const MINUTE = 60_000; +function modelUsageWindows(provider: ServerProvider | undefined, model: string) { + const windows = provider?.usageLimits?.windows ?? []; + return windows.filter((window) => { + if (window.modelIds && !window.modelIds.includes(model)) return false; + if (window.providerIds && !window.providerIds.includes(model.split("/")[0] ?? "")) return false; + switch (provider?.driver) { + case "cursor": + return ( + window.id === "apiPercentUsed" || + (/^(auto|default|grok|composer)/.test(model) && window.id === "autoPercentUsed") + ); + case "antigravity": { + const family = model.match(/^(gemini|claude|gpt)-/)?.[1]; + const pool = window.id.match(/^(gemini|claude|gpt|3p)-/)?.[1]; + return ( + pool === undefined || + family === undefined || + pool === family || + (pool === "3p" && (family === "claude" || family === "gpt")) + ); + } + case "codex": + // Seat-based plans spend credits only past their plan windows. + return ( + window.id !== "individual" || + !windows.some((entry) => entry.id === "primary" || entry.id === "secondary") + ); + case "opencode": + return !window.id.startsWith("go_") || model.startsWith("opencode-go/"); + default: + return true; + } + }); +} + export function exhaustedUsageWindow( provider: ServerProvider | undefined, now: number, model?: string, ) { - const windows = provider?.usageLimits?.windows ?? []; + const windows = + model === undefined + ? (provider?.usageLimits?.windows ?? []) + : modelUsageWindows(provider, model); const exhausted = windows.filter( (window) => (window.usedPercent ?? 0) >= 100 && - (model === undefined || window.modelIds === undefined || window.modelIds.includes(model)) && - (model === undefined || - window.providerIds === undefined || - window.providerIds.includes(model.split("/")[0] ?? "")) && (!window.resetsAt || !Number.isFinite(Date.parse(window.resetsAt)) || Date.parse(window.resetsAt) > now), ); - if (model === undefined) return exhausted[0]; - switch (provider?.driver) { - case "cursor": { - const api = exhausted.find((window) => window.id === "apiPercentUsed"); - const auto = exhausted.find((window) => window.id === "autoPercentUsed"); - return /^(auto|default|grok|composer)/.test(model) ? auto && api : api; - } - case "antigravity": { - const family = model.match(/^(gemini|claude|gpt)-/)?.[1]; - return exhausted.find((window) => { - const pool = window.id.match(/^(gemini|claude|gpt|3p)-/)?.[1]; - return ( - pool === undefined || - family === undefined || - pool === family || - (pool === "3p" && (family === "claude" || family === "gpt")) - ); - }); - } - case "codex": { - // Seat-based plans spend credits only past their plan windows, so the credit cap blocks only plans without them. - const hasPlanWindows = windows.some( - (window) => window.id === "primary" || window.id === "secondary", - ); - return exhausted.find((window) => window.id !== "individual" || !hasPlanWindows); - } - case "opencode": - return exhausted.find( - (window) => !window.id.startsWith("go_") || model.startsWith("opencode-go/"), - ); - default: - return exhausted[0]; + if ( + provider?.driver === "cursor" && + model !== undefined && + /^(auto|default|grok|composer)/.test(model) + ) { + return ( + exhausted.find((window) => window.id === "autoPercentUsed") && + exhausted.find((window) => window.id === "apiPercentUsed") + ); } + return exhausted[0]; } -export function resolveNewThreadModelSelection(input: { - readonly defaultSelection: ModelSelection | null; - readonly explicitSelection?: ModelSelection | null; - readonly fallbackSelection: ModelSelection | null; +interface AccountChoiceInput { readonly providers: readonly ServerProvider[]; readonly now: number; readonly settings?: ServerSettings; readonly projectId?: ProjectId | null; -}) { +} + +function isUsableSelection(candidate: ModelSelection, input: AccountChoiceInput) { + const entry = input.providers.find((item) => item.instanceId === candidate.instanceId); + return ( + entry !== undefined && + entry.enabled && + entry.installed && + entry.auth.status !== "unauthenticated" && + isProviderAvailable(entry) && + !exhaustedUsageWindow(entry, input.now, candidate.model) && + (!input.settings || + resolveProjectProviderInstanceEnabled( + input.settings, + input.projectId ?? null, + entry.instanceId, + )) + ); +} + +/** + * With `fallbackToAvailableAccounts` on and the selection's account out of usage (`exhausted` + * when a turn just said so), the same model on another account with usage left, else another + * provider's default model. + */ +export function availableAccountSelection( + input: AccountChoiceInput & { readonly selection: ModelSelection; readonly exhausted?: boolean }, +): ModelSelection | null { + if (!input.settings?.fallbackToAvailableAccounts) return null; + const provider = input.providers.find((entry) => entry.instanceId === input.selection.instanceId); + if (!provider) return null; + if (!input.exhausted && !exhaustedUsageWindow(provider, input.now, input.selection.model)) { + return null; + } + return ( + automaticFallbackSelections(input.selection, provider, input.providers, input.now).find( + (candidate) => isUsableSelection(candidate, input), + ) ?? null + ); +} + +export function resolveNewThreadModelSelection( + input: AccountChoiceInput & { + readonly defaultSelection: ModelSelection | null; + readonly explicitSelection?: ModelSelection | null; + readonly fallbackSelection: ModelSelection | null; + }, +) { const selection = input.explicitSelection ?? input.defaultSelection; const unchanged = { selection, fallback: null }; - if (input.explicitSelection || !selection || !input.fallbackSelection) return unchanged; + if (input.explicitSelection || !selection) return unchanged; const provider = input.providers.find((entry) => entry.instanceId === selection.instanceId); const window = exhaustedUsageWindow(provider, input.now, selection.model); - const fallback = input.providers.find( - (entry) => entry.instanceId === input.fallbackSelection?.instanceId, - ); - if ( - !provider || - !window || - !fallback?.enabled || - !fallback.installed || - fallback.auth.status === "unauthenticated" || - !isProviderAvailable(fallback) || - exhaustedUsageWindow(fallback, input.now, input.fallbackSelection.model) || - (input.settings && - !resolveProjectProviderInstanceEnabled( - input.settings, - input.projectId ?? null, - fallback.instanceId, - )) - ) - return unchanged; + if (!provider || !window) return unchanged; + const replacement = + (input.fallbackSelection && isUsableSelection(input.fallbackSelection, input) + ? input.fallbackSelection + : null) ?? availableAccountSelection({ ...input, selection, exhausted: true }); + if (!replacement) return unchanged; return { - selection: input.fallbackSelection, + selection: replacement, fallback: { provider, resetsAt: window.resetsAt ?? null }, }; } +/** Other accounts of the exhausted provider first, then other providers, most headroom first. */ +function automaticFallbackSelections( + selection: ModelSelection, + exhausted: ServerProvider, + providers: readonly ServerProvider[], + now: number, +): ModelSelection[] { + // Only accounts that report limits: an unknown balance could be pay-per-use or already spent. + const headroom = (entry: ServerProvider, model: string) => { + const windows = entry.usageLimits?.unavailable ? [] : modelUsageWindows(entry, model); + if (windows.length === 0) return null; + const left = windows.map((window) => + window.resetsAt && Date.parse(window.resetsAt) <= now ? 100 : remainingPercent(window), + ); + return entry.driver === "cursor" && /^(auto|default|grok|composer)/.test(model) + ? Math.max(...left) + : Math.min(...left); + }; + const resolveAccountKey = accountKeyResolver(providers.map(nativeReport)); + const exhaustedLogin = resolveAccountKey(nativeReport(exhausted)); + return providers + .flatMap((entry) => { + if ( + entry.instanceId === exhausted.instanceId || + // Two instances signed in to the same login share one quota. + (exhaustedLogin !== null && resolveAccountKey(nativeReport(entry)) === exhaustedLogin) + ) { + return []; + } + const sameDriver = entry.driver === exhausted.driver; + const model = + (sameDriver ? entry.models.find((item) => item.slug === selection.model) : undefined) ?? + entry.models.find((item) => item.isDefault) ?? + entry.models[0]; + if (!model) return []; + const left = headroom(entry, model.slug); + if (left === null) return []; + const keepsModel = sameDriver && model.slug === selection.model; + const candidate: ModelSelection = keepsModel + ? { ...selection, instanceId: entry.instanceId } + : { instanceId: entry.instanceId, model: model.slug }; + return [{ candidate, sameDriver, keepsModel, left }]; + }) + .sort( + (a, b) => + Number(b.sameDriver) - Number(a.sameDriver) || + Number(b.keepsModel) - Number(a.keepsModel) || + b.left - a.left, + ) + .map(({ candidate }) => candidate); +} + export const CHATGPT_USAGE_URL = "https://chatgpt.com/#settings/Usage"; export const CHATGPT_USAGE_LIMIT_CODE = "subscription_sharing_usage_limit_exceeded"; /** A ChatGPT Business admin caps plan use in other apps; only the admin can raise it. */