From 3c8274d241abf5c5b2531adf75587250786b7f86 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Thu, 19 Nov 2026 17:34:50 -0800 Subject: [PATCH 1/2] test(server): routing recovers once a host advertises the model A probe cached while CLIProxyAPI was warming up keeps returning no host. This assertion fails on main. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/inference/InferenceProxy.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/server/src/inference/InferenceProxy.test.ts b/apps/server/src/inference/InferenceProxy.test.ts index 67e958139308f..80d38499d90b5 100644 --- a/apps/server/src/inference/InferenceProxy.test.ts +++ b/apps/server/src/inference/InferenceProxy.test.ts @@ -1453,6 +1453,7 @@ it.effect( expect(yield* proxy.routeCandidates(local.key, hostId, "codex", astra)).toEqual([]); poolCooldowns = undefined; catalog = [...initialCatalog, { id: astra, owned_by: "openai" }]; + expect(yield* proxy.routeCandidates(local.key, hostId, "codex", astra)).toHaveLength(1); yield* proxy.refreshModelCatalog(true, ProviderInstanceId.make("codex")); expect(yield* proxy.routeCandidates(local.key, hostId, "codex", astra)).toHaveLength(1); catalog = initialCatalog; From 300ea8b065551b8d1f7034844847f8c1e6def93f Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Thu, 19 Nov 2026 17:35:18 -0800 Subject: [PATCH 2/2] fix(server): stale route probes no longer fail resumed turns After a restart, a route probe read CLIProxyAPI's model list before it was complete and cached that for 15 seconds. Every request in that window got "No inference host is currently available", so the resumed turn spent its retries on a stale answer. When the cached probes leave no host for the requested model, routing now refreshes the matching probes once before giving up. Requests that find a host keep the cached path, and hosts in a request cooldown are not re-probed. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/inference/InferenceProxy.ts | 42 ++++++++++++++------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/apps/server/src/inference/InferenceProxy.ts b/apps/server/src/inference/InferenceProxy.ts index 39a04203a050d..11ba4533147e6 100644 --- a/apps/server/src/inference/InferenceProxy.ts +++ b/apps/server/src/inference/InferenceProxy.ts @@ -1787,21 +1787,35 @@ const make = Effect.gen(function* () { (candidate) => Cache.get(routeProbes, { candidate, driver: activeDriver }), { concurrency: "unbounded" }, ); + const availableCandidates = (probes: typeof candidates, now: number) => + probes.flatMap((candidate, index) => { + if (!candidate) return []; + const host = matching[index]!; + const cooling = (slug: string) => routeCoolingDown(activeDriver, host.hostId, slug, now); + const models = candidate.models.filter((slug) => !cooling(slug)); + const unavailableModels = includeUnavailable + ? [...new Set([...candidate.unavailableModels, ...host.models.filter(cooling)])] + : []; + return (models.length && (model === undefined || models.includes(model))) || + unavailableModels.length + ? [{ ...candidate, hostId: host.hostId, models, unavailableModels }] + : []; + }); const now = yield* Clock.currentTimeMillis; - const available = candidates.flatMap((candidate, index) => { - if (!candidate) return []; - const host = matching[index]!; - const cooling = (slug: string) => routeCoolingDown(activeDriver, host.hostId, slug, now); - const models = candidate.models.filter((slug) => !cooling(slug)); - const unavailableModels = includeUnavailable - ? [...new Set([...candidate.unavailableModels, ...host.models.filter(cooling)])] - : []; - return (models.length && (model === undefined || models.includes(model))) || - unavailableModels.length - ? [{ ...candidate, hostId: host.hostId, models, unavailableModels }] - : []; - }); - return available; + const available = availableCandidates(candidates, now); + if ( + available.length || + !model || + matching.every((candidate) => routeCoolingDown(activeDriver, candidate.hostId, model, now)) + ) + return available; + + const refreshed = yield* Effect.forEach( + matching, + (candidate) => Cache.refresh(routeProbes, { candidate, driver: activeDriver }), + { concurrency: "unbounded" }, + ); + return availableCandidates(refreshed, yield* Clock.currentTimeMillis); }, Effect.mapError(safeFailure("Could not resolve inference hosts.")), );