diff --git a/apps/server/src/inference/InferenceProxy.test.ts b/apps/server/src/inference/InferenceProxy.test.ts index 67e958139308f..80d38499d90b5 100644 --- a/apps/server/src/inference/InferenceProxy.test.ts +++ b/apps/server/src/inference/InferenceProxy.test.ts @@ -1453,6 +1453,7 @@ it.effect( expect(yield* proxy.routeCandidates(local.key, hostId, "codex", astra)).toEqual([]); poolCooldowns = undefined; catalog = [...initialCatalog, { id: astra, owned_by: "openai" }]; + expect(yield* proxy.routeCandidates(local.key, hostId, "codex", astra)).toHaveLength(1); yield* proxy.refreshModelCatalog(true, ProviderInstanceId.make("codex")); expect(yield* proxy.routeCandidates(local.key, hostId, "codex", astra)).toHaveLength(1); catalog = initialCatalog; diff --git a/apps/server/src/inference/InferenceProxy.ts b/apps/server/src/inference/InferenceProxy.ts index 39a04203a050d..11ba4533147e6 100644 --- a/apps/server/src/inference/InferenceProxy.ts +++ b/apps/server/src/inference/InferenceProxy.ts @@ -1787,21 +1787,35 @@ const make = Effect.gen(function* () { (candidate) => Cache.get(routeProbes, { candidate, driver: activeDriver }), { concurrency: "unbounded" }, ); + const availableCandidates = (probes: typeof candidates, now: number) => + probes.flatMap((candidate, index) => { + if (!candidate) return []; + const host = matching[index]!; + const cooling = (slug: string) => routeCoolingDown(activeDriver, host.hostId, slug, now); + const models = candidate.models.filter((slug) => !cooling(slug)); + const unavailableModels = includeUnavailable + ? [...new Set([...candidate.unavailableModels, ...host.models.filter(cooling)])] + : []; + return (models.length && (model === undefined || models.includes(model))) || + unavailableModels.length + ? [{ ...candidate, hostId: host.hostId, models, unavailableModels }] + : []; + }); const now = yield* Clock.currentTimeMillis; - const available = candidates.flatMap((candidate, index) => { - if (!candidate) return []; - const host = matching[index]!; - const cooling = (slug: string) => routeCoolingDown(activeDriver, host.hostId, slug, now); - const models = candidate.models.filter((slug) => !cooling(slug)); - const unavailableModels = includeUnavailable - ? [...new Set([...candidate.unavailableModels, ...host.models.filter(cooling)])] - : []; - return (models.length && (model === undefined || models.includes(model))) || - unavailableModels.length - ? [{ ...candidate, hostId: host.hostId, models, unavailableModels }] - : []; - }); - return available; + const available = availableCandidates(candidates, now); + if ( + available.length || + !model || + matching.every((candidate) => routeCoolingDown(activeDriver, candidate.hostId, model, now)) + ) + return available; + + const refreshed = yield* Effect.forEach( + matching, + (candidate) => Cache.refresh(routeProbes, { candidate, driver: activeDriver }), + { concurrency: "unbounded" }, + ); + return availableCandidates(refreshed, yield* Clock.currentTimeMillis); }, Effect.mapError(safeFailure("Could not resolve inference hosts.")), );