diff --git a/apps/mobile/src/lib/modelOptions.test.ts b/apps/mobile/src/lib/modelOptions.test.ts index 9703998e6d1ca..4a6a7331862ae 100644 --- a/apps/mobile/src/lib/modelOptions.test.ts +++ b/apps/mobile/src/lib/modelOptions.test.ts @@ -427,6 +427,28 @@ describe("mobile model options", () => { expect(isModelSelectionUnavailable(config, disabled, null)).toBe(true); expect(isModelSelectionUnavailable(config, removed, null)).toBe(true); expect(resolveSelectableModelSelection(null, disabled, null)).toBe(disabled); + const cooling = { + ...config, + providers: config.providers.map((provider) => ({ + ...provider, + models: [ + { + slug: usable.model, + name: "GPT-5.6-Sol", + isUnavailable: true, + isCustom: false, + capabilities: null, + }, + ], + })), + }; + expect(isModelSelectionUnavailable(cooling, usable, null)).toBe(true); + expect(resolveSelectableModelSelection(cooling, usable, null)).toBe(usable); + expect(buildModelOptions(cooling, usable, undefined, null)[0]).toMatchObject({ + selection: usable, + label: "GPT-5.6-Sol", + isUnavailable: true, + }); }); describe("Antigravity selections", () => { diff --git a/apps/mobile/src/lib/modelOptions.ts b/apps/mobile/src/lib/modelOptions.ts index 44b36478cd972..854f3eb0326e4 100644 --- a/apps/mobile/src/lib/modelOptions.ts +++ b/apps/mobile/src/lib/modelOptions.ts @@ -23,7 +23,10 @@ import { buildExplicitProviderOptionSelectionsFromDescriptors, getProviderOptionDescriptors, } from "@t3tools/shared/model"; -import { resolveModelShortName } from "@t3tools/client-runtime/provider-clients"; +import { + resolveModelShortName, + unavailableModelReason, +} from "@t3tools/client-runtime/provider-clients"; import { resolveProvidersForCwd } from "@t3tools/client-runtime/providerSkills"; export type ModelOption = { @@ -223,13 +226,25 @@ export function isModelSelectionUnavailable( config: T3ServerConfig | null | undefined, selection: ModelSelection | null | undefined, projectId: ProjectId | null, +): boolean { + const provider = config?.providers.find( + (candidate) => candidate.instanceId === selection?.instanceId, + ); + return ( + isProviderSelectionUnavailable(config, selection, provider, projectId) || + (selection != null && unavailableModelReason(provider, selection.model) !== null) + ); +} + +function isProviderSelectionUnavailable( + config: T3ServerConfig | null | undefined, + selection: ModelSelection | null | undefined, + provider: T3ServerConfig["providers"][number] | undefined, + projectId: ProjectId | null, ): boolean { if (!config || !selection) { return false; } - const provider = config.providers.find( - (candidate) => candidate.instanceId === selection.instanceId, - ); if (isDisabledForProject(config, provider, selection.instanceId, projectId)) { return true; } @@ -263,6 +278,8 @@ export function describeUnavailableModelSelection( (candidate) => candidate.instanceId === selection.instanceId, ); const driver = selectionDriver(config, selection.instanceId); + const modelReason = unavailableModelReason(provider, selection.model); + if (modelReason) return { title: "Model unavailable", detail: modelReason }; if (isDriverDisallowed(config, driver)) { const label = providerDisplayLabel({ driver: driver ?? selection.instanceId, @@ -310,7 +327,7 @@ export function resolveSelectableModelSelection( isProviderEnabledForProject(config, provider, projectId) && provider.installed && provider.auth.status !== "unauthenticated" && - !isModelSelectionUnavailable(config, selection, projectId) + !isProviderSelectionUnavailable(config, selection, provider, projectId) ? selection : null; } @@ -455,7 +472,7 @@ export function buildModelOptions( ...(provider.iconUrl ? { providerIconUrl: provider.iconUrl } : {}), isHidden: !model.isCustom && hiddenModels.has(model.slug), canHide: !model.isCustom, - ...(providerUnavailable ? { isUnavailable: true } : {}), + ...(providerUnavailable || model.isUnavailable ? { isUnavailable: true } : {}), isDefault: model.isDefault === true, isLegacy: model.isLegacy === true, ...(updateRequired ? { providerUpdateRequired: updateRequired } : {}), diff --git a/apps/mobile/src/state/thread-task-permissions.test.ts b/apps/mobile/src/state/thread-task-permissions.test.ts index 47adfc0cfa26a..ba5447a3b2940 100644 --- a/apps/mobile/src/state/thread-task-permissions.test.ts +++ b/apps/mobile/src/state/thread-task-permissions.test.ts @@ -230,7 +230,7 @@ beforeEach(() => { state.thread.groupDiscussionOpen = false; state.projection = null; state.serverConfig = { - providers: [{ instanceId: "codex", driver: "codex", enabled: true }], + providers: [{ instanceId: "codex", driver: "codex", enabled: true, models: [] }], environment: { capabilities: {} }, } as unknown as ServerConfig; state.pendingRequests = { @@ -482,7 +482,7 @@ describe("mobile task permissions", () => { vi.clearAllMocks(); state.serverConfig = { ...state.serverConfig, - providers: [{ instanceId: "codex", driver: "codex", enabled: false }], + providers: [{ instanceId: "codex", driver: "codex", enabled: false, models: [] }], } as unknown as ServerConfig; state.draft = { ...state.draft, attachments: [] }; expect(await useThreadComposerState({ startSideChat }).onSendMessage()).toBeNull(); diff --git a/apps/server/src/inference/InferenceProxy.test.ts b/apps/server/src/inference/InferenceProxy.test.ts index 1a62a344754ba..bae69bcb5f140 100644 --- a/apps/server/src/inference/InferenceProxy.test.ts +++ b/apps/server/src/inference/InferenceProxy.test.ts @@ -1171,6 +1171,34 @@ it.effect( expect(activeCodexConfig).toMatchObject({ customModels: expect.arrayContaining(["gpt-local-new"]), }); + const astra = "gpt-local-new"; + catalog = initialCatalog; + poolCooldowns = [ + { + scope: "model", + model_key: astra, + retry_at: DateTime.formatIso( + DateTime.makeUnsafe((yield* Clock.currentTimeMillis) + 320_000), + ), + }, + ]; + yield* proxy.refreshModelCatalog(true, ProviderInstanceId.make("codex")); + expect(activeCodexConfig).toMatchObject({ + customModels: expect.arrayContaining([astra]), + }); + expect(yield* proxy.routeHealth(configured.config, local.key, "codex")).toMatchObject({ + unavailableModels: [astra], + }); + expect(yield* proxy.routeCandidates(local.key, hostId, "codex", astra)).toEqual([]); + poolCooldowns = undefined; + catalog = [...initialCatalog, { id: astra, owned_by: "openai" }]; + yield* proxy.refreshModelCatalog(true, ProviderInstanceId.make("codex")); + expect(yield* proxy.routeCandidates(local.key, hostId, "codex", astra)).toHaveLength(1); + catalog = initialCatalog; + yield* proxy.refreshModelCatalog(true, ProviderInstanceId.make("codex")); + expect(activeCodexConfig).toMatchObject({ + customModels: expect.not.arrayContaining([astra]), + }); const piId = ProviderInstanceId.make("pi"); yield* proxy.connect({ ...connect, @@ -1453,7 +1481,7 @@ it.effect( ).toHaveLength(2); peerId = hostId; yield* proxy.connect({ ...connect, grant: issued }); - expect(yield* proxy.hasFallback(issued.key, hostId, "codex")).toBe(true); + expect(yield* proxy.hasRoutePool(issued.key, hostId, "codex")).toBe(true); expect(yield* fs.readFileString(config.settingsPath)).not.toContain(issued.key); const setCodexConfigEnabled = (enabled: boolean) => settings.reviseProviderInstance(ProviderInstanceId.make("codex"), (instance) => { @@ -1469,7 +1497,7 @@ it.effect( }); }); yield* setCodexConfigEnabled(false); - expect(yield* proxy.hasFallback(issued.key, hostId, "codex")).toBe(false); + expect(yield* proxy.hasRoutePool(issued.key, hostId, "codex")).toBe(false); expect(yield* proxy.routeCandidates(issued.key, hostId, "codex")).toBeNull(); expect(yield* proxy.authorize(issued.key, hostId, "codex")).toBeNull(); expect((yield* proxy.status).routes.map((route) => route.instanceId)).not.toContain( @@ -1482,7 +1510,7 @@ it.effect( [inferenceProject]: { providerInstances: { [codexId]: { enabled: true } } }, }, }); - expect(yield* proxy.hasFallback(issued.key, hostId, "codex")).toBe(true); + expect(yield* proxy.hasRoutePool(issued.key, hostId, "codex")).toBe(true); expect(yield* proxy.routeCandidates(issued.key, hostId, "codex")).toHaveLength(1); expect(yield* proxy.authorize(issued.key, hostId, "codex")).not.toBeNull(); expect((yield* proxy.status).routes.map((route) => route.instanceId)).toContain( @@ -1512,13 +1540,13 @@ it.effect( const resumed = (yield* settings.getSettings).providerInstances[codexId]!; expect(resumed.enabled).toBe(false); expect(resumed.config).toMatchObject({ inferenceEnabled: true }); - expect(yield* proxy.hasFallback(issued.key, hostId, "codex")).toBe(true); + expect(yield* proxy.hasRoutePool(issued.key, hostId, "codex")).toBe(true); yield* settings.updateSettings({ projectSettingsOverrides: { [inferenceProject]: null }, }); yield* setCodexConfigEnabled(true); yield* proxy.connect({ ...connect, grant: issued }); - expect(yield* proxy.hasFallback(issued.key, hostId, "codex")).toBe(true); + expect(yield* proxy.hasRoutePool(issued.key, hostId, "codex")).toBe(true); expect( (yield* settings.getSettings).providerInstances[ProviderInstanceId.make("codex")] ?.config, @@ -1540,6 +1568,9 @@ it.effect( (candidate) => candidate.hostId, ), ).toEqual([EnvironmentId.make("other-host")]); + const fallbackHealth = yield* proxy.health(issued.key, hostId, "codex"); + expect(fallbackHealth?.models).toContain("codex-auto-review"); + expect(fallbackHealth?.unavailableModels ?? []).not.toContain("codex-auto-review"); yield* proxy.recordRouteOutcome({ driver: "codex", hostId, @@ -1632,7 +1663,25 @@ it.effect( ).toHaveLength(2); const reissued = { ...issued, key: "reissued-primary-key" }; yield* proxy.connect({ ...connect, grant: reissued }); - expect(yield* proxy.hasFallback(reissued.key, hostId, "codex")).toBe(false); + expect(yield* proxy.hasRoutePool(reissued.key, hostId, "codex")).toBe(true); + expect( + yield* proxy.routeCandidates(reissued.key, hostId, "codex", "codex-auto-review"), + ).toHaveLength(1); + yield* proxy.recordRouteOutcome({ + driver: "codex", + hostId, + model: "codex-auto-review", + status: 503, + retryAfter: "320", + }); + yield* TestClock.adjust("319 seconds"); + expect( + yield* proxy.routeCandidates(reissued.key, hostId, "codex", "codex-auto-review"), + ).toEqual([]); + yield* TestClock.adjust("1 second"); + expect( + yield* proxy.routeCandidates(reissued.key, hostId, "codex", "codex-auto-review"), + ).toHaveLength(1); expect(yield* proxy.authorize(issued.key, hostId, "codex")).toBeNull(); peerId = EnvironmentId.make("other-host"); yield* proxy.connect({ diff --git a/apps/server/src/inference/InferenceProxy.ts b/apps/server/src/inference/InferenceProxy.ts index 9a71272bbd108..2654cc39251c5 100644 --- a/apps/server/src/inference/InferenceProxy.ts +++ b/apps/server/src/inference/InferenceProxy.ts @@ -73,6 +73,7 @@ import { claudeKeychainEntry } from "./claudeKeychain.ts"; import { install, proxyVersion, resolveRelease, Release } from "./install.ts"; import * as PeerLinks from "../peer/PeerLinks.ts"; import { + inferenceRetryDelayMs, isRetryableInferenceStatus, modelsForDriver, needsAccountModelLookup, @@ -1514,19 +1515,22 @@ const make = Effect.gen(function* () { const baseUrl = yield* candidateAddress(input.candidate, ownId); const health = input.candidate.hostId === ownId - ? yield* mutex.withPermit(localHealth()) + ? yield* mutex.withPermit(localHealth(input.candidate.key)) : yield* jsonRequest(baseUrl + "/health", input.candidate.key, InferenceRouteHealth); - const models = input.candidate.models.filter((model) => health.models.includes(model)); - return health.availableDrivers.includes(input.driver) && models.length - ? { baseUrl, key: input.candidate.key, models } - : undefined; + const models = health.availableDrivers.includes(input.driver) + ? input.candidate.models.filter((model) => health.models.includes(model)) + : []; + const unavailableModels = input.candidate.models.filter((model) => + health.unavailableModels?.includes(model), + ); + return { baseUrl, key: input.candidate.key, models, unavailableModels }; }, Effect.timeout("3 seconds"), Effect.orElseSucceed(() => undefined), ), }); - const hasFallback = Effect.fn("InferenceProxy.hasFallback")( + const hasRoutePool = Effect.fn("InferenceProxy.hasRoutePool")( function* (key: string, hostId: EnvironmentId | undefined, driver: string | undefined) { if (!hostId || !driver) return false; const instanceId = defaultInstanceIdForDriver(ProviderDriverKind.make(driver)); @@ -1544,16 +1548,16 @@ const make = Effect.gen(function* () { return ( (yield* read).routePools?.some( (entry) => - entry.driver === driver && - entry.primary.hostId === hostId && - entry.primary.key === key && - entry.alternates.length > 0, + entry.driver === driver && entry.primary.hostId === hostId && entry.primary.key === key, ) ?? false ); }, Effect.mapError(safeFailure("Could not inspect inference routes.")), ); + const routeCoolingDown = (driver: string, hostId: EnvironmentId, model: string, now: number) => + (routeFailures.get(`${driver}\0${hostId}\0${model}`) ?? 0) > now; + const routeCandidates = Effect.fn("InferenceProxy.routeCandidates")( function* ( key: string, @@ -1561,10 +1565,11 @@ const make = Effect.gen(function* () { driver?: string, model?: string, includeAll = false, + includeUnavailable = false, ) { if (hostId === undefined) { const target = yield* authorize(key); - return target ? [{ ...target, models: [] }] : null; + return target ? [{ ...target, models: [], unavailableModels: [] }] : null; } const current = yield* settings.getSettings; const active = Arr.findFirst(INFERENCE_DRIVERS, (candidateDriver) => { @@ -1584,7 +1589,9 @@ const make = Effect.gen(function* () { const pool = (yield* read).routePools?.find((entry) => entry.driver === activeDriver); if (!pool || pool.primary.hostId !== hostId || pool.primary.key !== key) { const target = yield* authorize(key, hostId, driver); - return target ? [{ ...target, models: route.inferenceModels }] : null; + return target + ? [{ ...target, models: route.inferenceModels, unavailableModels: [] }] + : null; } const matching = model ? [pool.primary, ...pool.alternates].filter((candidate) => candidate.models.includes(model)) @@ -1601,11 +1608,14 @@ const make = Effect.gen(function* () { const available = candidates.flatMap((candidate, index) => { if (!candidate) return []; const host = matching[index]!; - const models = candidate.models.filter( - (slug) => (routeFailures.get(`${activeDriver}\0${host.hostId}\0${slug}`) ?? 0) <= now, - ); - return models.length && (model === undefined || models.includes(model)) - ? [{ ...candidate, hostId: host.hostId, models }] + const cooling = (slug: string) => routeCoolingDown(activeDriver, host.hostId, slug, now); + const models = candidate.models.filter((slug) => !cooling(slug)); + const unavailableModels = includeUnavailable + ? [...new Set([...candidate.unavailableModels, ...host.models.filter(cooling)])] + : []; + return (models.length && (model === undefined || models.includes(model))) || + unavailableModels.length + ? [{ ...candidate, hostId: host.hostId, models, unavailableModels }] : []; }); return available; @@ -1628,26 +1638,13 @@ const make = Effect.gen(function* () { } if (!isRetryableInferenceStatus(input.status)) return; const now = yield* Clock.currentTimeMillis; - const numericDelay = (value: string | undefined, scale: number) => { - if (!value?.trim()) return undefined; - const amount = Number(value); - if (!Number.isFinite(amount) || amount < 0) return undefined; - const delay = amount * scale; - return Number.isSafeInteger(Math.ceil(delay)) ? Math.ceil(delay) : undefined; - }; - const millis = numericDelay(input.retryAfterMs, 1); - const seconds = input.retryAfter?.trim() ? Number(input.retryAfter) : Number.NaN; - const standard = Number.isFinite(seconds) - ? numericDelay(input.retryAfter, 1000) - : input.retryAfter - ? Date.parse(input.retryAfter) - now - : undefined; - const retryDelay = - millis ?? - (standard !== undefined && Number.isFinite(standard) && standard >= 0 ? standard : undefined); + const retryDelay = inferenceRetryDelayMs(input.retryAfter, input.retryAfterMs, now); const duration = - input.status === 429 && retryDelay !== undefined - ? Math.max(5_000, Math.min(retryDelay, Number.MAX_SAFE_INTEGER - now)) + (input.status === 429 || input.status === 503) && retryDelay !== undefined + ? Math.max( + input.status === 429 ? 5_000 : 0, + Math.min(retryDelay, Number.MAX_SAFE_INTEGER - now), + ) : input.status === 502 || input.status === 503 || input.status === 529 ? 5_000 : 30_000; @@ -1710,10 +1707,12 @@ const make = Effect.gen(function* () { Effect.mapError(safeFailure("Inference host unavailable.")), ); - const localHealth = Effect.fn("InferenceProxy.localHealth")(function* () { + const localHealth = Effect.fn("InferenceProxy.localHealth")(function* (expectedKey?: string) { const state = yield* read; if (!state.enabled) return yield* failure("Inference hosting is disabled."); const runtime = yield* ensureWatched(state); + if (expectedKey !== undefined && runtime.key !== expectedKey) + return yield* failure("Inference route credentials have changed. Connect the route again."); const [{ files }, { data }] = yield* Effect.all( [ jsonRequest(runtime.baseUrl + "/v0/management/auth-files", runtime.management, Files), @@ -1729,10 +1728,12 @@ const make = Effect.gen(function* () { const eligible = files .filter( (file) => - file.status === "active" && !file.disabled && - !file.unavailable && - !excludedNames.has(file.name), + ((file.status === "active" && !file.unavailable && !excludedNames.has(file.name)) || + file.cooldowns?.some( + (cooldown) => + cooldown.scope === "credential" && Date.parse(cooldown.retry_at) > now, + )), ) .slice(0, 128); const keys: AccountModelsCacheKey[] = eligible.map((file) => ({ @@ -1750,14 +1751,45 @@ const make = Effect.gen(function* () { ); } } + const models = modelsForDriver(data, "opencode", { + accounts: files, + excludedNames, + registeredModels, + now, + }); + const unavailable = new Set(); + for (const file of files) { + if (file.disabled) continue; + for (const cooldown of file.cooldowns ?? []) { + if (!(Date.parse(cooldown.retry_at) > now)) continue; + if (cooldown.scope === "model" && cooldown.model_key) unavailable.add(cooldown.model_key); + if (cooldown.scope === "credential") + for (const model of registeredModels.get(file.id || file.name) ?? []) + unavailable.add(model); + } + } + for (const model of models) unavailable.delete(model); return { availableDrivers: availableDrivers(files.filter((file) => !evidence.has(file.name))), - models: modelsForDriver(data, "opencode", { - accounts: files, - excludedNames, - registeredModels, - now, - }), + models, + ...(unavailable.size ? { unavailableModels: [...unavailable] } : {}), + }; + }); + + const poolHealth = Effect.fn("InferenceProxy.poolHealth")(function* ( + key: string, + hostId: EnvironmentId, + driver: InferenceDriver, + ) { + const candidates = yield* routeCandidates(key, hostId, driver, undefined, true, true); + if (candidates === null) return null; + const models = new Set(candidates.flatMap((candidate) => candidate.models)); + const unavailable = new Set(candidates.flatMap((candidate) => candidate.unavailableModels)); + for (const model of models) unavailable.delete(model); + return { + availableDrivers: models.size ? [driver] : [], + models: [...models], + ...(unavailable.size ? { unavailableModels: [...unavailable] } : {}), }; }); @@ -1771,14 +1803,7 @@ const make = Effect.gen(function* () { entry.primary.key === key, ) : undefined; - if (pool) { - const candidates = yield* routeCandidates(key, hostId, pool.driver, undefined, true); - if (candidates === null) return null; - return { - availableDrivers: candidates.length ? [pool.driver] : [], - models: [...new Set(candidates.flatMap((candidate) => candidate.models))], - }; - } + if (pool) return yield* poolHealth(key, pool.primary.hostId, pool.driver); const target = yield* authorize(key, hostId, driver); if (!target) return null; return hostId === undefined @@ -1796,39 +1821,15 @@ const make = Effect.gen(function* () { entry.primary.hostId === route.inferenceHostId && entry.primary.key === key, ); - if (pool?.alternates.length) { - const candidates = yield* routeCandidates( - key, - route.inferenceHostId, - pool.driver, - undefined, - true, - ); - if (!candidates?.length) - return yield* failure("No authorized inference host is available."); - return { - availableDrivers: [pool.driver], - models: [...new Set(candidates.flatMap((candidate) => candidate.models))], - }; + if (pool) { + const health = yield* poolHealth(key, route.inferenceHostId, pool.driver); + if (health === null) return yield* failure("No authorized inference host is available."); + return health; } const ownId = yield* environment.getEnvironmentId; - if (route.inferenceHostId === ownId) { - return yield* mutex.withPermit( - Effect.gen(function* () { - const state = yield* read; - if (!state.enabled) return yield* failure("Inference hosting is disabled."); - const runtime = yield* ensureWatched(state); - if (runtime.key !== key) - return yield* failure( - "Inference route credentials have changed. Connect the route again.", - ); - return yield* localHealth(); - }), - ); - } - const address = pool - ? yield* candidateAddress(pool.primary, ownId) - : route.inferenceConnection || route.inferenceBaseUrl.includes("/api/inference/linked/") + if (route.inferenceHostId === ownId) return yield* mutex.withPermit(localHealth(key)); + const address = + route.inferenceConnection || route.inferenceBaseUrl.includes("/api/inference/linked/") ? new URL( "/api/inference", yield* peerAddress(route.inferenceHostId, route.inferenceConnection), @@ -2096,6 +2097,33 @@ const make = Effect.gen(function* () { const catalog = yield* Cache.get(catalogs, connection); if (catalog === undefined) return candidate; const models = modelsForDriver(catalog, driver); + const discovered = new Set(models); + const missing = candidate.models.filter((model) => !discovered.has(model)); + const probeKey = { candidate, driver }; + if (!missing.length) { + yield* Cache.invalidate(routeProbes, probeKey); + } else { + // CLIProxyAPI omits a model from /v1/models during a cooldown. + const now = yield* Clock.currentTimeMillis; + const health = yield* Cache.refresh(routeProbes, probeKey); + if (!health) return candidate; + const retained = missing.filter( + (model) => + routeCoolingDown(driver, candidate.hostId, model, now) || + health.unavailableModels.includes(model), + ); + if (retained.length) { + const previous = new Set(candidate.models); + for (const model of retained) discovered.add(model); + const persisted = [ + ...candidate.models.filter((model) => discovered.has(model)), + ...models.filter((model) => !previous.has(model)), + ]; + return Equal.equals(persisted, candidate.models) + ? candidate + : { ...candidate, models: persisted }; + } + } return Equal.equals(models, candidate.models) ? candidate : { ...candidate, models }; }); let changed = false; @@ -2204,7 +2232,7 @@ const make = Effect.gen(function* () { status, manage, authorize, - hasFallback, + hasRoutePool, routeCandidates, recordRouteOutcome, health, @@ -2266,7 +2294,7 @@ export class InferenceProxy extends Context.Service< }> | null, InferenceProxyError >; - readonly hasFallback: ( + readonly hasRoutePool: ( key: string, hostId: EnvironmentId | undefined, driver: string | undefined, diff --git a/apps/server/src/inference/http.test.ts b/apps/server/src/inference/http.test.ts index 09c91158eb522..2fc170c981f44 100644 --- a/apps/server/src/inference/http.test.ts +++ b/apps/server/src/inference/http.test.ts @@ -11,6 +11,7 @@ import * as InferenceHttp from "./http.ts"; it("streams an authorized request with a host key, then denies the revoked consumer", async () => { let active = true; let throttled = false; + let unavailable = false; let closed = false; let upstreamCalls = 0; let linkedHost: string | undefined; @@ -29,6 +30,17 @@ it("streams an authorized request with a host key, then denies the revoked consu signal.addEventListener("abort", () => { closed = true; }); + if (unavailable) + return HttpClientResponse.fromWeb( + request, + Response.json( + { error: { code: "model_cooldown", message: "All credentials are cooling down." } }, + { + status: 503, + headers: { "retry-after": "320" }, + }, + ), + ); if (throttled) return HttpClientResponse.fromWeb( request, @@ -83,7 +95,7 @@ it("streams an authorized request with a host key, then denies the revoked consu : null, ); }, - hasFallback: () => Effect.succeed(false), + hasRoutePool: () => Effect.succeed(false), recordRouteOutcome: () => Effect.void, }), ), @@ -134,6 +146,15 @@ it("streams an authorized request with a host key, then denies the revoked consu expect(limited.headers.get("retry-after")).toBe("60"); expect(limited.headers.get("retry-after-ms")).toBe("1200000"); expect(await limited.text()).toBe("quota"); + unavailable = true; + const cooling = await handler(request("linked/codex/host/v1/messages?beta=true", "consumer")); + expect(cooling.headers.get("retry-after")).toBe("320"); + expect(await cooling.json()).toEqual({ + error: { + code: "model_cooldown", + message: "All credentials are cooling down. Retry after 320 seconds.", + }, + }); const health = await handler( new Request("http://t3.test/api/inference/linked/codex/host/health", { headers: { authorization: "Bearer consumer" }, @@ -158,7 +179,7 @@ it("streams an authorized request with a host key, then denies the revoked consu ) ).status, ).toBe(403); - expect(upstreamCalls).toBe(4); + expect(upstreamCalls).toBe(5); } finally { await dispose(); } @@ -169,6 +190,7 @@ it("tries another authorized host for the same model after an upstream auth fail let recordedRetry: unknown; let primaryStatus = 403; let backupStatus = 200; + let available = true; const client = HttpClient.make((request) => Effect.sync(() => { calls.push(request.url); @@ -207,21 +229,25 @@ it("tries another authorized host for the same model after an upstream auth fail Layer.mock(InferenceProxy)({ authorize: () => Effect.succeed({ baseUrl: "http://primary.test", key: "primary-key" }), routeCandidates: () => - Effect.succeed([ - { - baseUrl: "http://primary.test", - key: "primary-key", - models: ["claude-fable-5-2", "claude-alpha"], - hostId: EnvironmentId.make("primary"), - }, - { - baseUrl: "http://backup.test", - key: "backup-key", - models: ["claude-fable-5-2", "claude-beta"], - hostId: EnvironmentId.make("backup"), - }, - ]), - hasFallback: () => Effect.succeed(true), + Effect.succeed( + available + ? [ + { + baseUrl: "http://primary.test", + key: "primary-key", + models: ["claude-fable-5-2", "claude-alpha"], + hostId: EnvironmentId.make("primary"), + }, + { + baseUrl: "http://backup.test", + key: "backup-key", + models: ["claude-fable-5-2", "claude-beta"], + hostId: EnvironmentId.make("backup"), + }, + ] + : [], + ), + hasRoutePool: () => Effect.succeed(true), recordRouteOutcome: (input) => Effect.sync(() => { if (input.status === 429) recordedRetry = input; @@ -327,6 +353,13 @@ it("tries another authorized host for the same model after an upstream auth fail const invalid = await send(); expect(invalid.status).toBe(400); expect(calls.slice(12)).toEqual(["http://backup.test/v1/messages"]); + available = false; + const unavailable = await send(); + expect(unavailable.status).toBe(503); + expect(await unavailable.json()).toMatchObject({ + error: { message: expect.stringContaining("claude-fable-5-2") }, + }); + expect(calls).toHaveLength(13); } finally { await dispose(); } @@ -356,7 +389,7 @@ it("streams an oversized chunked request once without losing its buffered prefix { baseUrl: "http://primary.test", key: "primary-key", models: ["claude-fable-5-2"] }, { baseUrl: "http://backup.test", key: "backup-key", models: ["claude-fable-5-2"] }, ]), - hasFallback: () => Effect.succeed(true), + hasRoutePool: () => Effect.succeed(true), recordRouteOutcome: () => Effect.void, }), ), diff --git a/apps/server/src/inference/http.ts b/apps/server/src/inference/http.ts index 2dec1739c193b..a5b862c9ad585 100644 --- a/apps/server/src/inference/http.ts +++ b/apps/server/src/inference/http.ts @@ -2,6 +2,7 @@ import * as Layer from "effect/Layer"; import * as Effect from "effect/Effect"; import * as Clock from "effect/Clock"; import * as Schema from "effect/Schema"; +import * as Option from "effect/Option"; import * as Result from "effect/Result"; import * as Cause from "effect/Cause"; import * as Stream from "effect/Stream"; @@ -15,9 +16,14 @@ import { HttpServerResponse, } from "effect/http"; import { InferenceProxy } from "./InferenceProxy.ts"; -import { inferenceTarget, isRetryableInferenceStatus } from "./routes.ts"; +import { inferenceRetryDelayMs, inferenceTarget, isRetryableInferenceStatus } from "./routes.ts"; const MAX_REPLAY_BODY_BYTES = 8 * 1024 * 1024; +const MAX_ERROR_BODY_BYTES = 64 * 1024; +const decodeErrorJson = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Unknown)); +const isProviderError = Schema.is( + Schema.Struct({ error: Schema.Struct({ message: Schema.String }) }), +); const MAX_CATALOG_BODY_BYTES = 1024 * 1024; const MAX_CATALOG_UNION_BYTES = 8 * 1024 * 1024; const MAX_CATALOG_MODELS = 4096; @@ -142,7 +148,7 @@ const handler = Effect.gen(function* () { ? grantDenied() : HttpServerResponse.jsonUnsafe(health, { headers: { "cache-control": "no-store" } }); } - const canReplay = yield* host.hasFallback(key, targetPath.environmentId, targetPath.driver); + const canReplay = yield* host.hasRoutePool(key, targetPath.environmentId, targetPath.driver); const declaredLength = request.headers["content-length"]; const length = Number(declaredLength); const buffered = @@ -173,7 +179,17 @@ const handler = Effect.gen(function* () { targetPath.path === "/v1/models", ); if (candidates === null) return grantDenied(); - if (!candidates.length) return HttpServerResponse.empty({ status: 503 }); + if (!candidates.length) + return HttpServerResponse.jsonUnsafe( + { + type: "error", + error: { + type: "server_error", + message: `No inference host is currently available${model ? ` for model ${model}` : ""}. Check Inference accounts or choose another model.`, + }, + }, + { status: 503, headers: { "cache-control": "no-store" } }, + ); const eligible = model ? candidates.filter((candidate) => candidate.models.includes(model)) : candidates.slice(0, 1); @@ -307,7 +323,7 @@ const handler = Effect.gen(function* () { break; } if (!result) return HttpServerResponse.empty({ status: 502 }); - return HttpServerResponse.stream(result.stream, { + const responseOptions = { status: result.status, contentType: result.headers["content-type"] ?? "application/json", headers: { @@ -320,7 +336,39 @@ const handler = Effect.gen(function* () { ? { "retry-after-ms": result.headers["retry-after-ms"] } : {}), }, - }); + }; + if ( + targetPath.driver === "codex" && + result.status === 503 && + result.headers["content-type"]?.startsWith("application/json") + ) { + const delay = inferenceRetryDelayMs( + result.headers["retry-after"], + result.headers["retry-after-ms"], + yield* Clock.currentTimeMillis, + ); + if (delay !== undefined && delay > 0) { + const buffered = yield* replayBody(result.stream, MAX_ERROR_BODY_BYTES); + const decoded = buffered.body && decodeErrorJson(new TextDecoder().decode(buffered.body)); + if (decoded && Option.isSome(decoded) && isProviderError(decoded.value)) { + const payload = decoded.value; + const seconds = Math.ceil(delay / 1000); + // Codex displays the error message but omits Retry-After from retry notifications. + return HttpServerResponse.jsonUnsafe( + { + ...payload, + error: { + ...payload.error, + message: `${payload.error.message} Retry after ${seconds} ${seconds === 1 ? "second" : "seconds"}.`, + }, + }, + responseOptions, + ); + } + return HttpServerResponse.stream(buffered.stream, responseOptions); + } + } + return HttpServerResponse.stream(result.stream, responseOptions); }).pipe( Effect.orElseSucceed(() => HttpServerResponse.empty({ status: 502 })), Effect.withTracerEnabled(false), diff --git a/apps/server/src/inference/providerHealth.ts b/apps/server/src/inference/providerHealth.ts index daa3a58e77705..9745a70eecc6d 100644 --- a/apps/server/src/inference/providerHealth.ts +++ b/apps/server/src/inference/providerHealth.ts @@ -16,15 +16,22 @@ export function filterInferenceWorkspaceModels( snapshots: NonNullable, ): NonNullable { if (provider.driver !== "pi" && provider.driver !== "opencode") return snapshots; - const routedModels = new Set( - provider.models.filter((model) => model.slug.startsWith("t3proxy/")).map((model) => model.slug), - ); + const routedModels = new Map(provider.models.map((model) => [model.slug, model])); const filtered = snapshots.map((snapshot) => { if (snapshot.models === undefined) return snapshot; - const models = snapshot.models.filter( - (model) => !model.slug.startsWith("t3proxy/") || routedModels.has(model.slug), - ); - return models.length === snapshot.models.length ? snapshot : { ...snapshot, models }; + const models = snapshot.models.flatMap((model) => { + if (!model.slug.startsWith("t3proxy/")) return [model]; + const routed = routedModels.get(model.slug); + if (!routed) return []; + if (routed.isUnavailable === model.isUnavailable) return [model]; + if (routed.isUnavailable) return [{ ...model, isUnavailable: true }]; + const { isUnavailable: _unavailable, ...available } = model; + return [available]; + }); + return models.length === snapshot.models.length && + models.every((model, i) => model === snapshot.models?.[i]) + ? snapshot + : { ...snapshot, models }; }); return filtered.every((snapshot, index) => snapshot === snapshots[index]) ? snapshots : filtered; } @@ -70,11 +77,13 @@ export const makeInferenceProviderHealth = Effect.fn("makeInferenceProviderHealt catalog.health.availableDrivers.some((driver) => driver === provider.driver) ? new Set(catalog.health.models) : new Set(); - const models = provider.models.filter((model) => { - if (!mixedCatalog) return servedModels.has(model.slug); - if (!model.slug.startsWith("t3proxy/")) return true; - const id = model.slug.slice("t3proxy/".length); - return route.inferenceModels.includes(id) && servedModels.has(id); + const unavailable = new Set("health" in catalog ? catalog.health.unavailableModels : []); + const models = provider.models.flatMap((model) => { + if (mixedCatalog && !model.slug.startsWith("t3proxy/")) return [model]; + const id = mixedCatalog ? model.slug.slice("t3proxy/".length) : model.slug; + if (mixedCatalog && !route.inferenceModels.includes(id)) return []; + if (servedModels.has(id)) return [model]; + return unavailable.has(id) ? [{ ...model, isUnavailable: true }] : []; }); const problem = "problem" in catalog @@ -103,15 +112,21 @@ export const makeInferenceProviderHealth = Effect.fn("makeInferenceProviderHealt message: problem, }; } - return problem === undefined - ? { ...provider, models } - : { - ...provider, - availability: "unavailable" as const, - unavailableReason: problem, - status: "error" as const, - message: problem, - }; + if (problem === undefined) return { ...provider, models }; + if (models.some((model) => model.isUnavailable)) + return { + ...provider, + models, + status: provider.status === "ready" ? ("warning" as const) : provider.status, + message: problem, + }; + return { + ...provider, + availability: "unavailable" as const, + unavailableReason: problem, + status: "error" as const, + message: problem, + }; }, (effect, provider) => effect.pipe( diff --git a/apps/server/src/inference/routes.test.ts b/apps/server/src/inference/routes.test.ts index 57f4dfde52ac8..c24065cceb47e 100644 --- a/apps/server/src/inference/routes.test.ts +++ b/apps/server/src/inference/routes.test.ts @@ -95,6 +95,7 @@ describe("inference routes", () => { }), ); expect(codex.config.launchArgs).toContain(baseUrl + "/v1"); + expect(codex.config.launchArgs).toContain("request_max_retries=0"); expect(codex.config.customModels).toEqual(["gpt-fixture"]); expect(codex.config.launchArgs).not.toContain("private"); expect(codex.environment).toContainEqual({ diff --git a/apps/server/src/inference/routes.ts b/apps/server/src/inference/routes.ts index a4eb94e45f0ca..782dd462bc6bf 100644 --- a/apps/server/src/inference/routes.ts +++ b/apps/server/src/inference/routes.ts @@ -8,6 +8,7 @@ import { type SettingsSecretValue, type ProviderInstanceConfig, } from "@t3tools/contracts"; +import { formatModelSlugName } from "@t3tools/shared/model"; import { fromLenientJson } from "@t3tools/shared/schemaJson"; import * as Equal from "effect/Equal"; import * as Option from "effect/Option"; @@ -37,6 +38,31 @@ export function inferencePath(method: string, rawUrl: string): string | null { export const isRetryableInferenceStatus = (status: number): boolean => [401, 403, 429, 502, 503, 529].includes(status); +export function inferenceRetryDelayMs( + retryAfter: string | undefined, + retryAfterMs: string | undefined, + now: number, +) { + const numericDelay = (value: string | undefined, scale: number) => { + if (!value?.trim()) return undefined; + const amount = Number(value); + if (!Number.isFinite(amount) || amount < 0) return undefined; + const delay = amount * scale; + return Number.isSafeInteger(Math.ceil(delay)) ? Math.ceil(delay) : undefined; + }; + const millis = numericDelay(retryAfterMs, 1); + const seconds = retryAfter?.trim() ? Number(retryAfter) : Number.NaN; + const standard = Number.isFinite(seconds) + ? numericDelay(retryAfter, 1000) + : retryAfter + ? Date.parse(retryAfter) - now + : undefined; + return ( + millis ?? + (standard !== undefined && Number.isFinite(standard) && standard >= 0 ? standard : undefined) + ); +} + export function inferenceTarget(method: string, rawUrl: string) { const linked = /^\/api\/inference\/linked\/([^/]+)\/([^/]+)(\/.*)$/.exec(rawUrl); const driver = linked && isInferenceDriver(linked[1]) ? linked[1] : undefined; @@ -133,8 +159,9 @@ const ownedBy = (vendor: string, pattern: RegExp) => (model: ProxyModel) => { const owner = model.owned_by?.toLowerCase(); return owner !== undefined && VENDORS.includes(owner) ? owner === vendor : pattern.test(model.id); }; +// Codex reports stream retries to the thread. Its inner HTTP retries can wait silently for minutes. const CODEX_PROVIDER = (baseUrl: string) => - `model_providers.t3proxy={name="T3 inference",base_url="${baseUrl}/v1",env_key="T3_INFERENCE_KEY",wire_api="responses",supports_websockets=false,requires_openai_auth=false}`; + `model_providers.t3proxy={name="T3 inference",base_url="${baseUrl}/v1",env_key="T3_INFERENCE_KEY",wire_api="responses",request_max_retries=0,supports_websockets=false,requires_openai_auth=false}`; const configRecord = Schema.Record(Schema.String, Schema.Unknown); const isConfigRecord = Schema.is(configRecord); @@ -366,5 +393,13 @@ export function withInferenceModelCatalog( // Codex and Claude ingest host discoveries through their custom model config. return models .filter((model) => hostedSlugs.has(model.slug)) - .map((model) => (model.isCustom ? { ...model, isCustom: false } : model)); + .map((model) => + model.isCustom + ? { + ...model, + name: model.name === model.slug ? formatModelSlugName(model.slug) : model.name, + isCustom: false, + } + : model, + ); } diff --git a/apps/server/src/mcp/OrchestratorMcpService.test.ts b/apps/server/src/mcp/OrchestratorMcpService.test.ts index b91705c4c162c..ff966c551e7ca 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.test.ts @@ -2893,6 +2893,7 @@ describe("OrchestratorMcpService provider resolution", () => { ...codex, models: [ { slug: "claude-opus-4-1", isLegacy: true }, + { slug: "claude-opus-cooling-down", isUnavailable: true }, ...Array.from({ length: 9 }, (_, index) => ({ slug: `claude-opus-5-${index + 1}` })), ].map((model) => ({ ...model, name: model.slug, isCustom: false, capabilities: null })), version: "2.1.302", @@ -2941,6 +2942,9 @@ describe("OrchestratorMcpService provider resolution", () => { assert.equal(updateRequired.code, "model_unavailable"); assert.include(updateRequired.message, "v2.1.302"); assert.include(updateRequired.message, "2.1.321"); + + const coolingDown = yield* refuse("claude-opus-cooling-down"); + assert.equal(coolingDown.code, "model_unavailable"); }).pipe(Effect.provide(layerOrchestratorService.pipe(Layer.provide(layerDependencies)))); }), ); diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 4a22fa9cf1066..d9dbcbb0aef8c 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -1482,7 +1482,15 @@ const make = Effect.gen(function* () { return { parent, target, shell, runCount } as const; }); - const loadProviders = providerRegistry.getProviders; + const loadProviders = providerRegistry.getProviders.pipe( + Effect.map((providers) => + providers.map((provider) => + provider.models.some((model) => model.isUnavailable) + ? { ...provider, models: provider.models.filter((model) => !model.isUnavailable) } + : provider, + ), + ), + ); const loadOrchestrationCapableInstanceIds = () => readOrchestrationProviders().pipe( diff --git a/apps/server/src/provider/Drivers/CodexDriver.test.ts b/apps/server/src/provider/Drivers/CodexDriver.test.ts index c9ec9e8e6fff6..45552f709e0ce 100644 --- a/apps/server/src/provider/Drivers/CodexDriver.test.ts +++ b/apps/server/src/provider/Drivers/CodexDriver.test.ts @@ -679,11 +679,17 @@ it.layer(layerTest)("CodexDriver", (it) => { ...CodexDriver.defaultConfig(), homePath, setupMode, - customModels: [model, model.slug], + customModels: [model, model.slug, "gpt-6.2-astra"], }, }); expect((yield* instance.snapshot.getSnapshot).models).toEqual([ { ...model, isCustom: setupMode === "existing" }, + { + slug: "gpt-6.2-astra", + name: setupMode === "inference" ? "GPT-6.2-Astra" : "gpt-6.2-astra", + capabilities: null, + isCustom: setupMode === "existing", + }, ]); }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn)), ); diff --git a/apps/server/src/provider/ProviderRegistry.test.ts b/apps/server/src/provider/ProviderRegistry.test.ts index 6e6fc9a8f5591..6f30a1ac5c213 100644 --- a/apps/server/src/provider/ProviderRegistry.test.ts +++ b/apps/server/src/provider/ProviderRegistry.test.ts @@ -1120,6 +1120,7 @@ it.layer( Effect.gen(function* () { let proxyStatus = 503; let poolAvailable = true; + let coolingDown = false; const mixedCatalog = driver === "pi" || driver === "opencode"; const routedSlug = mixedCatalog ? "t3proxy/claude-fixture" : "claude-fixture"; const offlineSlug = mixedCatalog ? "t3proxy/claude-offline" : "claude-offline"; @@ -1244,8 +1245,25 @@ it.layer( ).toEqual(status === 200 ? [routedSlug, "morph/project"] : ["morph/project"]); } } + coolingDown = true; + yield* TestClock.adjust("16 seconds"); + const cooled = (yield* registry.getProviders)[0]!; + expect(cooled.models.find((model) => model.slug === offlineSlug)).toMatchObject({ + isUnavailable: true, + }); + if (mixedCatalog) + expect( + cooled.workspaceSnapshots?.[0]?.models?.find((model) => model.slug === offlineSlug), + ).toMatchObject({ isUnavailable: true }); poolAvailable = false; yield* TestClock.adjust("16 seconds"); + const coolingPool = (yield* registry.getProviders)[0]!; + expect(coolingPool.availability).toBeUndefined(); + expect( + coolingPool.models.find((model) => model.slug === offlineSlug)?.isUnavailable, + ).toBe(true); + coolingDown = false; + yield* TestClock.adjust("16 seconds"); expect((yield* registry.getProviders)[0]?.availability).toBe( mixedCatalog ? undefined : "unavailable", ); @@ -1277,6 +1295,7 @@ it.layer( ? Effect.succeed({ availableDrivers: poolAvailable ? [driver] : [], models: ["claude-fixture"], + ...(coolingDown ? { unavailableModels: ["claude-offline"] } : {}), }) : Effect.fail(new InferenceProxyError({ message: "Unavailable" })), }), diff --git a/apps/web/src/components/ChatView.logic.test.ts b/apps/web/src/components/ChatView.logic.test.ts index da985f77d6418..34b6a299086ba 100644 --- a/apps/web/src/components/ChatView.logic.test.ts +++ b/apps/web/src/components/ChatView.logic.test.ts @@ -59,7 +59,7 @@ import { DraftId, hydrateImagesFromPersisted, useComposerDraftStore } from "../c import { derivePhase } from "../session-logic"; import { makeThreadFixture, makeThreadProjectionFixture } from "../test-fixtures"; import { - getAntigravitySendBlockReason, + getProviderModelSendBlockReason, resolveComposerProviderSelection, resolveWorktreeSetupProgress, peekRememberedThreadTimeline, @@ -745,7 +745,7 @@ describe("resolveComposerProviderSelection", () => { expect(selection.selectedProviderEntry?.instanceId).toBe(signedOutEntry.instanceId); expect( - getAntigravitySendBlockReason(selection.selectedProviderEntry?.snapshot, "gemini-pro"), + getProviderModelSendBlockReason(selection.selectedProviderEntry?.snapshot, "gemini-pro"), ).toBe("Sign in to Antigravity in provider settings before sending."); }); @@ -755,10 +755,10 @@ describe("resolveComposerProviderSelection", () => { models: catalogModels, }).snapshot; - expect(getAntigravitySendBlockReason(provider, "saved-model-not-in-current-catalog")).toBe( + expect(getProviderModelSendBlockReason(provider, "saved-model-not-in-current-catalog")).toBe( "That Antigravity model is no longer available. Choose another model.", ); - expect(getAntigravitySendBlockReason(provider, "gemini-pro")).toBeNull(); + expect(getProviderModelSendBlockReason(provider, "gemini-pro")).toBeNull(); }); it("allows a saved native model to retry after a provider error without changing it", () => { @@ -768,7 +768,7 @@ describe("resolveComposerProviderSelection", () => { }).snapshot; expect( - getAntigravitySendBlockReason(provider, "saved-model-not-in-current-catalog"), + getProviderModelSendBlockReason(provider, "saved-model-not-in-current-catalog"), ).toBeNull(); }); diff --git a/apps/web/src/components/ChatView.logic.ts b/apps/web/src/components/ChatView.logic.ts index 0b1c2b6ae5d84..e5bc39832dff3 100644 --- a/apps/web/src/components/ChatView.logic.ts +++ b/apps/web/src/components/ChatView.logic.ts @@ -1,5 +1,6 @@ import type { ProjectKnowledgeContextRecord } from "@t3tools/contracts"; import { toServerClock } from "@t3tools/client-runtime/clock"; +import { unavailableModelReason } from "@t3tools/client-runtime/provider-clients"; import { type OrchestrationV2ProviderTurn, type RunAttemptId } from "@t3tools/contracts"; import { deriveLatestThreadRun, @@ -560,14 +561,14 @@ export function resolveComposerInteractionMode(input: { }; } -export function getAntigravitySendBlockReason( +export function getProviderModelSendBlockReason( provider: | Pick | null | undefined, model: string, ): string | null { - if (provider?.driver !== "antigravity") return null; + if (provider?.driver !== "antigravity") return unavailableModelReason(provider, model); if (!provider.installed) { return "Install Antigravity in provider settings before sending."; } diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index 2348871aa564e..0929b606cfe90 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -92,7 +92,7 @@ import { import { DEFAULT_COMPOSER_PLACEHOLDER } from "../../composerPlaceholder"; import { deriveComposerSendState, - getAntigravitySendBlockReason, + getProviderModelSendBlockReason, resolveComposerInteractionMode, threadShellHasStarted, } from "../ChatView.logic"; @@ -958,6 +958,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) selectedProvider, selectedProviderModels, composerModelOptions, + hostedModelCatalog, selectedModel, composerProviderState, } = useComposerProviderModel({ @@ -1010,7 +1011,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) ? undefined : clampManagedRuntimeMode(managedPolicy, compatibleRuntimeMode, compatibleRuntimeModes); - const providerSendBlockReason = getAntigravitySendBlockReason( + const providerSendBlockReason = getProviderModelSendBlockReason( selectedProviderEntry?.snapshot, selectedModel, ); @@ -2657,6 +2658,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) selectedInstanceId={selectedInstanceId} selectedModel={selectedModel} selectedProviderModels={selectedProviderModels} + hostedModelCatalog={hostedModelCatalog} composerModelOptions={composerModelOptions} reportedModelSelection={reportedModelSelection} promptUltrathink={promptUltrathink} diff --git a/apps/web/src/components/chat/ComposerControlsBar.tsx b/apps/web/src/components/chat/ComposerControlsBar.tsx index a3e1081470c66..f2e2ceab0dbf4 100644 --- a/apps/web/src/components/chat/ComposerControlsBar.tsx +++ b/apps/web/src/components/chat/ComposerControlsBar.tsx @@ -50,6 +50,7 @@ export function ComposerControlsBar(props: { selectedInstanceId: ProviderInstanceId; selectedModel: string; selectedProviderModels: ServerProvider["models"]; + hostedModelCatalog: boolean; composerModelOptions: ReturnType["composerModelOptions"]; reportedModelSelection: ChatComposerProps["reportedModelSelection"]; promptUltrathink: PromptUltrathink; @@ -159,6 +160,7 @@ export function ComposerControlsBar(props: { const applyRouteThreadModelPick = useStableCallback(() => applyThreadModelPick(routeThreadRef)); const traitsInput = resolveComposerTraitsInput({ provider: selectedProvider, + hostedModelCatalog: props.hostedModelCatalog, instanceId: selectedInstanceId, ...(routeKind === "server" ? { threadRef: routeThreadRef, onDraftModelOptionsChange: applyRouteThreadModelPick } diff --git a/apps/web/src/components/chat/ModelListRow.tsx b/apps/web/src/components/chat/ModelListRow.tsx index 69da770f1f84b..6df6dd249f788 100644 --- a/apps/web/src/components/chat/ModelListRow.tsx +++ b/apps/web/src/components/chat/ModelListRow.tsx @@ -46,7 +46,7 @@ export const ModelListRow = memo(function ModelListRow(props: { selectedFill={false} index={props.index} value={modelPickerModelKey(props.instanceId, props.slug)} - disabled={Boolean(props.disabledReason)} + disabled={props.unavailable || Boolean(props.disabledReason)} className={cn( "group relative w-full !min-w-0 max-w-full cursor-pointer", props.disabledReason && diff --git a/apps/web/src/components/chat/ModelPickerContent.tsx b/apps/web/src/components/chat/ModelPickerContent.tsx index 87bacfd7ca2a2..0e374a915f5d5 100644 --- a/apps/web/src/components/chat/ModelPickerContent.tsx +++ b/apps/web/src/components/chat/ModelPickerContent.tsx @@ -112,6 +112,7 @@ function shouldIncludeModelPickerOption(input: { return false; } if (isProviderInstancePickerReady(input.entry)) return true; + if (input.option.isUnavailable && isProviderInstancePickerVisible(input.entry)) return true; return ( input.entry.enabled && hasRuntimeModelCatalog(input.entry.driverKind) && @@ -501,7 +502,8 @@ export const ModelPickerContent = memo(function ModelPickerContent(props: { return null; } const resolvedModel = resolveSelectableModel(entry.driverKind, modelSlug, options); - if (!resolvedModel) return null; + if (!resolvedModel || options.find((option) => option.slug === resolvedModel)?.isUnavailable) + return null; if (additive && onToggleModel) { onToggleModel(instanceId, resolvedModel); } else { @@ -534,13 +536,13 @@ export const ModelPickerContent = memo(function ModelPickerContent(props: { if (activeInstanceHasSelectableUnavailableModel) { instanceIds.add(props.activeInstanceId); } - if (canOpenProviderSetup) { - for (const entry of instanceEntries) { - if ( - shouldOfferModelPickerSetup(entry, modelOptionsByInstance.get(entry.instanceId) ?? []) - ) { - instanceIds.add(entry.instanceId); - } + for (const entry of instanceEntries) { + const models = modelOptionsByInstance.get(entry.instanceId) ?? []; + if ( + (canOpenProviderSetup && shouldOfferModelPickerSetup(entry, models)) || + (entry.status === "warning" && models.some((model) => model.isUnavailable)) + ) { + instanceIds.add(entry.instanceId); } } return instanceIds.size > 0 ? instanceIds : undefined; @@ -742,7 +744,9 @@ export const ModelPickerContent = memo(function ModelPickerContent(props: { }); const modelJumpModelKeys = visibleModels - .filter((model) => !getModelDisabledReason?.(model.instanceId, model.slug)) + .filter( + (model) => !model.isUnavailable && !getModelDisabledReason?.(model.instanceId, model.slug), + ) .slice(0, MODEL_PICKER_JUMP_KEYBINDING_COMMANDS.length) .map((model) => modelPickerModelKey(model.instanceId, model.slug)); const allItemKeys = ((): string[] => [ diff --git a/apps/web/src/components/chat/TraitsPicker.tsx b/apps/web/src/components/chat/TraitsPicker.tsx index 84c3a5b65896a..47a4addaf81d0 100644 --- a/apps/web/src/components/chat/TraitsPicker.tsx +++ b/apps/web/src/components/chat/TraitsPicker.tsx @@ -20,7 +20,6 @@ import { getProviderOptionCurrentLabel, getProviderOptionDescriptors, isClaudeUltrathinkPrompt, - normalizeModelSlug, } from "@t3tools/shared/model"; import { memo } from "react"; import { GaugeIcon } from "lucide-react"; @@ -119,11 +118,15 @@ function getSelectedTraits( modelOptions: ProviderOptions | null | undefined, allowPromptInjectedEffort: boolean, planModeEnabled: boolean, + hostedModelCatalog = false, ) { const caps = getProviderModelCapabilities(models, model, provider, planModeEnabled); - const modelIsUnavailable = - keepsUnavailableModelOptions(provider, model) && - !models.some((candidate) => candidate.slug === normalizeModelSlug(model, provider)); + const modelIsUnavailable = keepsUnavailableModelOptions( + provider, + model, + models, + hostedModelCatalog, + ); const descriptors = modelIsUnavailable ? buildUnavailableModelOptionDescriptors( planModeEnabled @@ -191,6 +194,7 @@ function getSelectedTraits( export function getTraitsSectionVisibility(input: { provider: ProviderDriverKind; + hostedModelCatalog?: boolean; models: ReadonlyArray; model: string | null | undefined; promptUltrathink?: PromptUltrathink | undefined; @@ -206,6 +210,7 @@ export function getTraitsSectionVisibility(input: { input.modelOptions, input.allowPromptInjectedEffort ?? true, input.planModeEnabled, + input.hostedModelCatalog, ); const showEffort = selected.primarySelectDescriptor !== null; @@ -284,6 +289,7 @@ export function getReasoningTraitStep(input: { export interface TraitsMenuContentProps { provider: ProviderDriverKind; + hostedModelCatalog?: boolean; instanceId?: ProviderInstanceId; models: ReadonlyArray; model: string | null | undefined; @@ -302,6 +308,7 @@ export type TraitsSelectionInput = TraitsMenuContentProps & TraitsPersistence; /** The model's traits with their shown values, and writers that persist a change. */ export function useTraitsSelection({ provider, + hostedModelCatalog = false, instanceId, models, model, @@ -335,6 +342,7 @@ export function useTraitsSelection({ }; const selected = getTraitsSectionVisibility({ provider, + hostedModelCatalog, models, model, promptUltrathink, diff --git a/apps/web/src/components/chat/chatViewSend.ts b/apps/web/src/components/chat/chatViewSend.ts index 7f203a25c2471..a96049ffb474e 100644 --- a/apps/web/src/components/chat/chatViewSend.ts +++ b/apps/web/src/components/chat/chatViewSend.ts @@ -39,7 +39,7 @@ import type { ProviderInstanceEntry } from "../../providerInstances"; import { getComposerPromptInjectionState, getComposerProviderState } from "./composerProviderState"; import { formatOutgoingPrompt, - getAntigravitySendBlockReason, + getProviderModelSendBlockReason, resolveComposerInteractionMode, cloneComposerImageForRetry, type ComposerSendSnapshot, @@ -310,7 +310,7 @@ export function resolveMultipleModelTargets(input: { if (!provider?.enabled || !provider.isAvailable || provider.status !== "ready") { return { error: `Provider for ${selection.model} is unavailable.` }; } - const providerBlockReason = getAntigravitySendBlockReason(provider.snapshot, selection.model); + const providerBlockReason = getProviderModelSendBlockReason(provider.snapshot, selection.model); if (providerBlockReason) return { error: providerBlockReason }; const providerState = getComposerProviderState({ provider: provider.driverKind, diff --git a/apps/web/src/components/chat/composerProviderState.tsx b/apps/web/src/components/chat/composerProviderState.tsx index 2bad287673c33..db94e500e47ab 100644 --- a/apps/web/src/components/chat/composerProviderState.tsx +++ b/apps/web/src/components/chat/composerProviderState.tsx @@ -12,7 +12,6 @@ import { getProviderOptionCurrentValue, getProviderOptionDescriptors, isClaudeUltrathinkPrompt, - normalizeModelSlug, withImplicitFastModeDefault, } from "@t3tools/shared/model"; @@ -26,6 +25,7 @@ import { type ComposerProviderStateInput = { provider: ProviderDriverKind; + hostedModelCatalog?: boolean; model: string; models: ReadonlyArray; promptInjectionState?: ComposerPromptInjectionState; @@ -46,6 +46,7 @@ type ComposerProviderState = { type ComposerTraitsTarget = { provider: ProviderDriverKind; + hostedModelCatalog?: boolean; instanceId: ProviderInstanceId; threadRef?: ScopedThreadRef | undefined; draftId?: DraftId | undefined; @@ -86,20 +87,16 @@ export function getComposerProviderState(input: ComposerProviderStateInput): Com promptInjectionState = "none", planModeEnabled, } = input; - if (keepsUnavailableModelOptions(provider, model)) { - const normalizedModel = normalizeModelSlug(model, provider); - const modelIsInCatalog = models.some((candidate) => candidate.slug === normalizedModel); - if (!modelIsInCatalog) { - const preservedOptions = modelOptions?.filter( - (option) => planModeEnabled || option.id !== "agent" || option.value !== "plan", - ); - return { - provider, - promptEffort: null, - modelOptionsForDispatch: - preservedOptions && preservedOptions.length > 0 ? preservedOptions : undefined, - }; - } + if (keepsUnavailableModelOptions(provider, model, models, input.hostedModelCatalog)) { + const preservedOptions = modelOptions?.filter( + (option) => planModeEnabled || option.id !== "agent" || option.value !== "plan", + ); + return { + provider, + promptEffort: null, + modelOptionsForDispatch: + preservedOptions && preservedOptions.length > 0 ? preservedOptions : undefined, + }; } const { caps, selections } = resolveComposerOptionSelections( models, diff --git a/apps/web/src/components/chat/useComposerProviderModel.ts b/apps/web/src/components/chat/useComposerProviderModel.ts index dd9c8c40ff1a0..774cf58eb5ddb 100644 --- a/apps/web/src/components/chat/useComposerProviderModel.ts +++ b/apps/web/src/components/chat/useComposerProviderModel.ts @@ -15,6 +15,7 @@ import { NO_PROVIDER_MODEL_SELECTION, type ProviderInstanceEntry, deriveComposerProviderInstanceEntries, + usesHostedModelCatalog, } from "../../providerInstances"; import { type UnifiedSettings } from "@t3tools/contracts/settings"; import { getComposerPromptInjectionState, getComposerProviderState } from "./composerProviderState"; @@ -86,8 +87,14 @@ export function useComposerProviderModel( settings, }); const selectedProviderModels = selection.selectedProviderEntry?.models ?? NO_PROVIDER_MODELS; + const hostedModelCatalog = usesHostedModelCatalog( + selectedInstanceId, + selectedProvider, + settings.providerInstances[selectedInstanceId]?.config, + ); const composerProviderState = getComposerProviderState({ provider: selectedProvider, + hostedModelCatalog, model: modelState.selectedModel, models: selectedProviderModels, promptInjectionState, @@ -100,6 +107,7 @@ export function useComposerProviderModel( selectedInstanceId, selectedProvider, selectedProviderModels, + hostedModelCatalog, composerModelOptions: modelState.modelOptions, selectedModel: modelState.selectedModel, composerProviderState, diff --git a/apps/web/src/components/settings/ProviderModelsSection.tsx b/apps/web/src/components/settings/ProviderModelsSection.tsx index 884f91b7acc2a..6586efbad361d 100644 --- a/apps/web/src/components/settings/ProviderModelsSection.tsx +++ b/apps/web/src/components/settings/ProviderModelsSection.tsx @@ -566,6 +566,9 @@ const ProviderModelRow = memo(function ProviderModelRow({ {model.isCustom ? ( custom ) : null} + {model.isUnavailable ? ( + Unavailable + ) : null} {capLabels.length > 0 ? ( diff --git a/apps/web/src/modelSelection.test.ts b/apps/web/src/modelSelection.test.ts index ab8e468f6e1bd..8963dfcf4d74f 100644 --- a/apps/web/src/modelSelection.test.ts +++ b/apps/web/src/modelSelection.test.ts @@ -316,6 +316,48 @@ describe("instance-scoped model selection", () => { instanceId, model: "gpt-hosted-default", }); + const cooling = { + ...entry!, + models: entry!.models.map((model) => ({ ...model, isUnavailable: model.isDefault })), + }; + expect(getAppModelOptionsForInstance(settings, cooling)).toMatchObject([ + { slug: "gpt-hosted-default", isUnavailable: true }, + { slug: "gpt-hosted" }, + ]); + expect( + resolveAppModelSelectionForInstance( + instanceId, + settings, + [{ ...entry!.snapshot, models: cooling.models }], + null, + ), + ).toBe("gpt-hosted"); + const selected = createModelSelection(instanceId, "gpt-6.2-astra", [ + { id: "reasoningEffort", value: "medium" }, + ]); + const state = deriveEffectiveComposerModelState({ + threadStarted: true, + draft: null, + providers, + selectedProvider: driver, + selectedInstanceId: instanceId, + threadModelSelection: selected, + projectModelSelection: null, + settings, + }); + expect(state.selectedModel).toBe(selected.model); + expect(getAppModelOptionsForInstance(settings, entry!, state.selectedModel)).toContainEqual( + expect.objectContaining({ slug: selected.model, name: "GPT-6.2-Astra", isUnavailable: true }), + ); + const dispatch = getComposerProviderState({ + provider: driver, + model: state.selectedModel, + models: entry!.models, + hostedModelCatalog: true, + modelOptions: state.modelOptions?.[instanceId], + planModeEnabled: false, + }); + expect(dispatch.modelOptionsForDispatch).toEqual(selected.options); }); it("keeps custom models on the provider instance that declared them", () => { diff --git a/apps/web/src/modelSelection.ts b/apps/web/src/modelSelection.ts index 360efd7ba90f6..a8b57727f63c4 100644 --- a/apps/web/src/modelSelection.ts +++ b/apps/web/src/modelSelection.ts @@ -14,6 +14,7 @@ import { import { type CustomModelDefinition, createModelSelection, + formatModelSlugName, normalizeCustomModelSlug, readCustomModelEntries, resolveSelectableModel, @@ -84,8 +85,9 @@ function appendUnavailableDynamicModelSelection( provider: ProviderDriverKind, selectedModel: string | null | undefined, hiddenModels: ReadonlyArray, + hostedModelCatalog: boolean, ): AppModelOption[] { - if (!hasRuntimeModelCatalog(provider)) return options; + if (!hostedModelCatalog && !hasRuntimeModelCatalog(provider)) return options; const slug = normalizeCustomModelSlug(selectedModel); if (!slug) return options; if (isRuntimeModelDefaultAlias(provider, slug)) return options; @@ -94,7 +96,8 @@ function appendUnavailableDynamicModelSelection( if (hiddenModels.includes(slug)) return options; if (options.some((option) => option.slug === slug)) return options; - return [...options, { slug, name: slug, isCustom: false, isUnavailable: true }]; + const name = hostedModelCatalog ? formatModelSlugName(slug) : slug; + return [...options, { slug, name, isCustom: false, isUnavailable: true }]; } function toAppModelOption( @@ -113,6 +116,7 @@ function toAppModelOption( if (model.badge) option.badge = model.badge; if (model.isDefault) option.isDefault = true; if (model.isLegacy) option.isLegacy = true; + if (model.isUnavailable) option.isUnavailable = true; return option; } @@ -193,10 +197,11 @@ export function getAppModelOptionsForInstance( const preferences = readInstanceModelPreferences(settings, entry.instanceId); return appendUnavailableDynamicModelSelection( applyInstanceModelPreferences(options, preferences), - entry.models, + hostedModelCatalog ? serverModels : entry.models, entry.driverKind, selectedModel, preferences.hiddenModels, + hostedModelCatalog, ); } @@ -218,10 +223,11 @@ export function resolveAppModelSelection( selectedModel, ); const config = settings.providerInstances[instanceId]?.config; + const available = options.filter((option) => !option.isUnavailable); return ( resolveSelectableModel(resolvedProvider, selectedModel, options) ?? (usesHostedModelCatalog(instanceId, resolvedProvider, config) - ? (options.find((option) => option.isDefault)?.slug ?? options[0]?.slug) + ? (available.find((option) => option.isDefault)?.slug ?? available[0]?.slug) : undefined) ?? getDefaultServerModel(providers, resolvedProvider) ); @@ -247,19 +253,12 @@ export function resolveAppModelSelectionForInstance( if (resolvedSelection) { return resolvedSelection; } - if (resolutionOptions?.preserveUnavailableSelection && hasRuntimeModelCatalog(entry.driverKind)) { - const unavailableSelection = normalizeCustomModelSlug(selectedModel); - const hiddenModels = readInstanceModelPreferences(settings, entry.instanceId).hiddenModels; - if ( - unavailableSelection && - !hiddenModels.includes(unavailableSelection) && - resolveSelectableModel(entry.driverKind, selectedModel, entry.models) === null && - !isRuntimeModelDefaultAlias(entry.driverKind, unavailableSelection) - ) { - return unavailableSelection; - } - } - return options.find((option) => option.isDefault)?.slug ?? options[0]?.slug ?? null; + return ( + options.find((option) => option.isDefault && !option.isUnavailable)?.slug ?? + options.find((option) => !option.isUnavailable)?.slug ?? + options[0]?.slug ?? + null + ); } // New drafts carry the stored selection; capabilities resolve its raw options later. @@ -395,7 +394,9 @@ export function resolveAppModelSelectionState( if (entry) { const selectedModel = selectedEntry ? selection.model : null; // Use the cheap generation default; the catalog default is the expensive chat flagship. - const options = getTextGenerationModelOptionsForInstance(settings, entry); + const options = getTextGenerationModelOptionsForInstance(settings, entry).filter( + (option) => !option.isUnavailable || option.slug === selectedModel, + ); const model = resolveSelectableModel(entry.driverKind, selectedModel, options) ?? resolveSelectableModel( diff --git a/apps/web/src/providerInstances.ts b/apps/web/src/providerInstances.ts index 02a602421061c..2924fdb9f15f1 100644 --- a/apps/web/src/providerInstances.ts +++ b/apps/web/src/providerInstances.ts @@ -49,7 +49,11 @@ export function normalizeHostedModelCatalog( : null; // Older hosts mark models discovered through CLI proxy as custom. return models - .filter((model) => (hostedSlugs === null ? !model.isCustom : hostedSlugs.has(model.slug))) + .filter( + (model) => + (!model.isCustom && model.isUnavailable) || + (hostedSlugs === null ? !model.isCustom : hostedSlugs.has(model.slug)), + ) .map((model) => (model.isCustom ? { ...model, isCustom: false } : model)); } @@ -273,13 +277,15 @@ export function getDefaultProviderInstanceModel( const entry = getProviderInstanceEntry(providers, instanceId); if (!entry) return undefined; const config = settings?.providerInstances[instanceId]?.config; - const models = usesHostedModelCatalog(instanceId, entry.driverKind, config) + const catalog = usesHostedModelCatalog(instanceId, entry.driverKind, config) ? normalizeHostedModelCatalog(entry.models, config) : entry.models; + const models = catalog.filter((model) => !model.isUnavailable); return ( models.find((model) => model.isDefault && !model.isCustom)?.slug ?? models.find((model) => !model.isCustom)?.slug ?? models[0]?.slug ?? + catalog[0]?.slug ?? DEFAULT_MODEL_BY_PROVIDER[entry.driverKind] ); } diff --git a/apps/web/src/providerModels.ts b/apps/web/src/providerModels.ts index fba07d642cd66..4887dcbe9210f 100644 --- a/apps/web/src/providerModels.ts +++ b/apps/web/src/providerModels.ts @@ -42,11 +42,14 @@ export function isRuntimeModelDefaultAlias( export function keepsUnavailableModelOptions( provider: ProviderDriverKind, model: string | null | undefined, + models: ReadonlyArray, + hostedModelCatalog = false, ): boolean { return ( provider !== "antigravity" && !isRuntimeModelDefaultAlias(provider, model) && - hasRuntimeModelCatalog(provider) + (hasRuntimeModelCatalog(provider) || hostedModelCatalog) && + resolveSelectableModel(provider, model, models) === null ); } @@ -123,11 +126,13 @@ export function getDefaultServerModel( providers: ReadonlyArray, provider: ProviderDriverKind, ): string { - const models = getProviderModels(providers, provider); + const catalog = getProviderModels(providers, provider); + const models = catalog.filter((model) => !model.isUnavailable); return ( models.find((model) => model.isDefault && !model.isCustom)?.slug ?? models.find((model) => !model.isCustom)?.slug ?? models[0]?.slug ?? + catalog[0]?.slug ?? DEFAULT_MODEL_BY_PROVIDER[provider] ?? DEFAULT_MODEL ); diff --git a/docs/user/providers-claude.md b/docs/user/providers-claude.md index 4b30993b955b7..6b5df76e86261 100644 --- a/docs/user/providers-claude.md +++ b/docs/user/providers-claude.md @@ -64,6 +64,8 @@ The host receives the model request, including its prompt and tool results. The agent runs tools and stores files and thread history on its executing environment. Choosing another host in Settings changes the same provider. Local inference is labeled **Claude** or **Codex**. Open the model picker or refresh provider status in Settings to update the model list. +When every route serving a model reports a cooldown, the model stays visible as unavailable. +You can choose another model or keep your selection until the host can serve it again. While a client is connected to the executing environment and other shared inference hosts, T3 discovers hosts that serve the same provider and saves them as fallback routes. If the selected host is unreachable, rejects authentication, or reaches a temporary limit, a new diff --git a/packages/client-runtime/src/providerClients.ts b/packages/client-runtime/src/providerClients.ts index 3759cecbc7c7d..0e4d8ee6446c5 100644 --- a/packages/client-runtime/src/providerClients.ts +++ b/packages/client-runtime/src/providerClients.ts @@ -4,6 +4,7 @@ import { CodexSettings, isProviderDriverKind, ProviderDriverKind, + type ServerProvider, } from "@t3tools/contracts"; import { acpRegistryClient } from "@t3tools/provider-acp-registry/client"; import { makeProviderClientRegistry, type ProviderIcon } from "@t3tools/provider-core/client"; @@ -63,6 +64,18 @@ export function getProviderClient(driver: string | null | undefined) { return isProviderDriverKind(driver) ? providerClients.get(driver) : undefined; } +export function unavailableModelReason( + provider: Pick | null | undefined, + slug: string, +): string | null { + const model = provider?.models.find( + (entry) => entry.slug === slug || entry.aliases?.includes(slug), + ); + return model?.isUnavailable + ? `${model.name} is temporarily unavailable. Choose another model or try again later.` + : null; +} + export function resolveModelShortName( model: { readonly name: string; readonly shortName?: string | undefined }, driverKind: ProviderDriverKind | null | undefined, diff --git a/packages/contracts/src/inferenceProxy.ts b/packages/contracts/src/inferenceProxy.ts index 4393fdbd497ff..4151e8bc24004 100644 --- a/packages/contracts/src/inferenceProxy.ts +++ b/packages/contracts/src/inferenceProxy.ts @@ -55,6 +55,7 @@ export const InferenceDriver = Schema.Literals(INFERENCE_DRIVERS); export const InferenceRouteHealth = Schema.Struct({ availableDrivers: Schema.Array(InferenceDriver), models: Schema.Array(Schema.String), + unavailableModels: Schema.optionalKey(Schema.Array(Schema.String)), }); export type InferenceRouteHealth = typeof InferenceRouteHealth.Type; diff --git a/packages/contracts/src/server.ts b/packages/contracts/src/server.ts index af078a7af12dc..b832a119ee170 100644 --- a/packages/contracts/src/server.ts +++ b/packages/contracts/src/server.ts @@ -83,6 +83,7 @@ export const ServerProviderModel = Schema.Struct({ aliases: Schema.optional(Schema.Array(TrimmedNonEmptyString)), badge: Schema.optional(Schema.Literal("new")), isCustom: Schema.Boolean, + isUnavailable: Schema.optional(Schema.Boolean), isDefault: Schema.optional(Schema.Boolean), isLegacy: Schema.optional(Schema.Boolean), // False when the provider can run this model in sessions but not for diff --git a/scripts/scenarios/inference-routing-discovers-hosted-models.e2e.ts b/scripts/scenarios/inference-routing-discovers-hosted-models.e2e.ts index e432a53dad100..54cd5b9572f20 100644 --- a/scripts/scenarios/inference-routing-discovers-hosted-models.e2e.ts +++ b/scripts/scenarios/inference-routing-discovers-hosted-models.e2e.ts @@ -1,3 +1,4 @@ +/// // @effect-diagnostics nodeBuiltinImport:off - prepares an isolated server and fake CLI binaries. import * as NodeFSP from "node:fs/promises"; import * as NodePath from "node:path"; @@ -8,7 +9,7 @@ import * as Effect from "effect/Effect"; import { APP_DIR } from "../e2e.config.ts"; import { SECOND_HOME, startSecondServer } from "./second-server.ts"; -import { environmentId, runPromise, test, WEB } from "./t3.ts"; +import { environmentId, makeState, runPromise, test, WEB } from "./t3.ts"; const fakeProxy = `#!/usr/bin/env node const { createServer } = require("node:http"); @@ -16,15 +17,23 @@ const { readFileSync } = require("node:fs"); const config = JSON.parse(readFileSync(process.argv[process.argv.indexOf("-config") + 1], "utf8")); createServer((request, response) => { const { url, headers, method } = request; - const key = url === "/v0/management/auth-files" + const key = url.startsWith("/v0/management/") ? config.management["secret-key"] : config.access["api-keys"][0]; let body; if (method === "GET" && headers.authorization === "Bearer " + key) { + const { models, coolingDown } = JSON.parse(readFileSync("./scenario-models.json", "utf8")); if (url === "/v0/management/auth-files") - body = { files: [{ name: "codex-fixture@example.invalid.json", provider: "codex", status: "active" }] }; + body = { files: [{ + name: "codex-fixture@example.invalid.json", + provider: "codex", + status: "active", + ...(coolingDown ? { cooldowns: [{ scope: "model", model_key: coolingDown, retry_at: "2099-01-01T00:00:00Z" }] } : {}), + }] }; + else if (url.startsWith("/v0/management/auth-files/models?")) + body = { models: models.map((id) => ({ id })) }; else if (url === "/v1/models") - body = { data: JSON.parse(readFileSync("./scenario-models.json", "utf8")).map((id) => ({ id, owned_by: "openai" })) }; + body = { data: models.filter((id) => id !== coolingDown).map((id) => ({ id, owned_by: "openai" })) }; } response.writeHead(body ? 200 : 404, { "content-type": "application/json" }); response.end(JSON.stringify(body ?? {})); @@ -89,7 +98,7 @@ async function prepare(home: string) { await NodeFSP.writeFile(NodePath.join(runtime, "cli-proxy-api"), fakeProxy, { mode: 0o755 }); await NodeFSP.writeFile(codex, fakeCodex, { mode: 0o755 }); await NodeFSP.writeFile(offlineModule, loopbackFetch); - await NodeFSP.writeFile(modelsFile, JSON.stringify([sol])); + await NodeFSP.writeFile(modelsFile, JSON.stringify({ models: [sol] })); const config: { enableProviderUpdateChecks: boolean; providerInstances: { codex: { config: { binaryPath: string } } }; @@ -108,7 +117,7 @@ async function prepare(home: string) { } test( - "The Codex switch connects to a hosted catalog and provider refresh adds newly advertised models", + "Hosted Codex models get display names and stay visible during a cooldown", WEB, async ({ t3, app, browser, screen }) => { await t3.project("demo"); @@ -116,6 +125,7 @@ test( try { const { host, token } = await server.ready; const machine = `machine=${encodeURIComponent(await runPromise(environmentId(host)))}`; + await makeState(SECOND_HOME).state.project("Hosted models"); await t3.open("/settings/connections"); await screen.getByRole("button", "Add environment").tap(); const dialog = screen.getByRole("dialog"); @@ -139,12 +149,45 @@ test( await expect(details.getByText("Available", { exact: true })).toBeVisible(); await expect(solRow).toBeVisible(); - await NodeFSP.writeFile(modelsFile, JSON.stringify([sol, astra])); - await screen.getByRole("button", /refresh provider status|checked just now/i).tap(); + const refresh = screen.getByRole("button", /refresh provider status|checked just now/i); + await NodeFSP.writeFile(modelsFile, JSON.stringify({ models: [sol, astra] })); + await refresh.tap(); await expect(astraRow).toBeVisible(); await expect(solRow).toBeVisible(); await astraRow.scrollIntoView(); await app.screenshot("hosted-models-after-refresh"); + + await app.open("/"); + await screen.getByRole("button", "New thread", { exact: true }).tap(); + await screen.getByRole("option", /Hosted models/).tap(); + await expect(browser).toHaveURL(/\/draft\//); + const draft = await browser.evaluate(() => window.location.pathname); + const picker = browser.locator("[data-chat-provider-model-picker]"); + const astraOption = screen.getByRole("option", /gpt-6\.2-astra/i); + const openPicker = async () => { + await picker.tap(); + await screen.getByPlaceholder("Search models...").fill("6.2"); + }; + await openPicker(); + await app.screenshot("hosted-models-in-composer"); + await astraOption.tap(); + + const updateCatalog = async (coolingDown?: string) => { + await NodeFSP.writeFile(modelsFile, JSON.stringify({ models: [sol, astra], coolingDown })); + await app.open(`/settings/providers?${machine}&instanceId=codex`); + await refresh.tap(); + await expect(refresh).toBeEnabled(); + await app.open(draft); + await openPicker(); + }; + await updateCatalog(astra); + await app.screenshot("astra-during-cooldown"); + await expect(astraOption).toBeDisabled(); + await expect(picker).toHaveAttribute("aria-label", /GPT-6\.2-Astra.*Unavailable/); + await expect(screen.getByRole("option", /gpt-6\.2-sol/i)).toBeEnabled(); + await updateCatalog(); + await expect(screen.getByRole("option", /GPT-6\.2-Astra/)).toBeEnabled(); + await expect(screen.getByRole("option", /GPT-6\.2-Sol/)).toBeEnabled(); } finally { await server.stop(); }