From 2c8c16c5d46eb3e73a9e403f8cdade3d7e494794 Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Fri, 13 Nov 2026 10:16:45 +1100 Subject: [PATCH] Drop FAL endpoints the models API no longer serves (#641) The active search still lists fal-ai/video-prompt-generator, but any endpoint_id filter that includes it returns 404 Endpoint(s) not found and failed every models poll. Split that batch, omit only the missing id so the poll deprecates the row, and keep every other 404 as a failure. --- src/server/providers/fal-chat-facts.test.ts | 116 +++++++++++++- src/server/providers/fal.test.ts | 30 +++- src/server/providers/fal.ts | 159 ++++++++++++++++---- 3 files changed, 273 insertions(+), 32 deletions(-) diff --git a/src/server/providers/fal-chat-facts.test.ts b/src/server/providers/fal-chat-facts.test.ts index 84438225..5565d75c 100644 --- a/src/server/providers/fal-chat-facts.test.ts +++ b/src/server/providers/fal-chat-facts.test.ts @@ -320,6 +320,15 @@ describe('fal listModels', () => { ) } + function endpointNotFound(): Response { + return new Response( + JSON.stringify({ + error: { type: 'not_found', message: 'Endpoint(s) not found' }, + }), + { status: 404, headers: { 'Content-Type': 'application/json' } }, + ) + } + function respond( input: string, llms: (pathname: string) => string, @@ -495,12 +504,13 @@ describe('fal listModels', () => { }) it('skips fal-ai/whisper when the direct lookup 404s', async () => { + const error = vi.spyOn(console, 'error').mockImplementation(() => undefined) vi.stubGlobal('fetch', (input: string) => { const url = new URL(input) if (url.hostname === 'fal.ai') return Promise.resolve(new Response(NO_TOKEN_PRICE)) if (url.searchParams.get('endpoint_id') === 'fal-ai/whisper') { - return Promise.resolve(new Response('missing', { status: 404 })) + return Promise.resolve(endpointNotFound()) } return Promise.resolve( url.searchParams.has('endpoint_id') @@ -511,6 +521,110 @@ describe('fal listModels', () => { const { models } = await falProvider.listModels({ FAL_KEY: 'k' }) expect(models.some((model) => model.rawId === 'fal-ai/whisper')).toBe(false) expect(models).toHaveLength(listed.length + 1) + expect(error).toHaveBeenCalledWith( + expect.stringContaining( + 'fal-ai/whisper: no longer served at https://api.fal.ai/v1/models?', + ) as string, + ) + const line = String(error.mock.calls[0]?.[0]) + expect(line).toContain('endpoint_id=fal-ai%2Fwhisper') + expect(line).toContain('404 Endpoint(s) not found') + error.mockRestore() + }) + + it('fails the poll when an unlisted lookup 404s for another reason', async () => { + vi.stubGlobal('fetch', (input: string) => { + const url = new URL(input) + if (url.hostname === 'fal.ai') + return Promise.resolve(new Response(NO_TOKEN_PRICE)) + if (url.searchParams.get('endpoint_id') === 'fal-ai/whisper') { + return Promise.resolve(new Response('missing', { status: 404 })) + } + return Promise.resolve( + url.searchParams.has('endpoint_id') + ? page(fixtures) + : page([image, ...listed]), + ) + }) + await expect(falProvider.listModels({ FAL_KEY: 'k' })).rejects.toThrow( + 'fal models fetch failed: 404', + ) + }) + + it('omits a listed chat endpoint whose endpoint_id filter 404s', async () => { + const goneId = 'fal-ai/video-prompt-generator' + const error = vi.spyOn(console, 'error').mockImplementation(() => undefined) + vi.stubGlobal('fetch', (input: string) => { + const url = new URL(input) + if (url.hostname === 'fal.ai') + return Promise.resolve(new Response(NO_TOKEN_PRICE)) + const ids = url.searchParams.getAll('endpoint_id') + if (ids.includes(goneId)) return Promise.resolve(endpointNotFound()) + if (ids.length === 0) { + return Promise.resolve( + page([ + image, + ...listed, + { endpoint_id: goneId, metadata: { category: 'llm' } }, + ]), + ) + } + if (ids.length === 1 && ids[0] === 'fal-ai/whisper') { + return Promise.resolve(page([])) + } + const wanted = new Set(ids) + return Promise.resolve( + page(fixtures.filter((model) => wanted.has(model.endpoint_id))), + ) + }) + const result = await falProvider.listModels({ FAL_KEY: 'k' }) + expect(result.models.some((model) => model.rawId === goneId)).toBe(false) + expect(result.models).toHaveLength(listed.length + 1) + expect(result.docsFailures?.failed).toBe(0) + expect( + result.models.find( + (model) => model.rawId === 'fal-ai/bytedance/seed/v2/mini', + ), + ).toMatchObject({ activity: 'chat', contextWindow: 256_000 }) + const line = error.mock.calls.map((call) => String(call[0])).join('\n') + expect(line).toContain( + `${goneId}: no longer served at https://api.fal.ai/v1/models?`, + ) + expect(line).toContain('endpoint_id=fal-ai%2Fvideo-prompt-generator') + expect(line).toContain('expand=openapi-3.0') + expect(line).not.toContain('fal-ai/whisper') + error.mockRestore() + }) + + it('keeps the rows when a filtered 404 is not Endpoint(s) not found', async () => { + const goneId = 'fal-ai/video-prompt-generator' + vi.stubGlobal('fetch', (input: string) => { + const url = new URL(input) + if (url.hostname === 'fal.ai') + return Promise.resolve(new Response(NO_TOKEN_PRICE)) + const ids = url.searchParams.getAll('endpoint_id') + if (ids.includes(goneId)) { + return Promise.resolve(new Response('nope', { status: 404 })) + } + if (ids.length === 1 && ids[0] === 'fal-ai/whisper') { + return Promise.resolve(page([])) + } + return Promise.resolve( + ids.length === 0 + ? page([ + image, + ...listed, + { endpoint_id: goneId, metadata: { category: 'llm' } }, + ]) + : page(fixtures), + ) + }) + const result = await falProvider.listModels({ FAL_KEY: 'k' }) + expect(result.models.some((model) => model.rawId === goneId)).toBe(true) + expect(result.docsFailures?.first[0]?.error).toContain( + 'fal models fetch failed: 404', + ) + expect(result.models).toHaveLength(listed.length + 2) }) it('marks an unlisted endpoint deprecated once retires_at is past', async () => { diff --git a/src/server/providers/fal.test.ts b/src/server/providers/fal.test.ts index 04f4b1a7..8cc378cd 100644 --- a/src/server/providers/fal.test.ts +++ b/src/server/providers/fal.test.ts @@ -87,7 +87,7 @@ describe('fetchPageWithRetry', () => { }) it('gives up after the attempt budget with the last failure in the message', async () => { - fetchMock.mockResolvedValue(errorResponse(429)) + fetchMock.mockImplementation(() => Promise.resolve(errorResponse(429))) await expect( withTimersFlushed(fetchPageWithRetry('https://x', 'key', 3)), ).rejects.toThrow('fal models fetch failed after 3 attempts: 429') @@ -353,6 +353,34 @@ describe('fal AsyncAPI discovery', () => { expect(fetched.asyncApiRawIds).toEqual([]) }) + it('throws when the unlisted lookup 404s without Endpoint(s) not found', async () => { + fetchMock.mockImplementation((input) => { + const url = requestUrl(input) + if (url.startsWith('https://api.fal.ai/v1/models')) { + if (new URL(url).searchParams.get('endpoint_id') === 'fal-ai/whisper') { + return Promise.resolve(new Response('missing', { status: 404 })) + } + return Promise.resolve( + jsonBody({ + models: [ + { + endpoint_id: 'fal-ai/flux/dev', + openapi: FLUX_OPENAPI, + metadata: { category: 'text-to-image' }, + }, + ], + has_more: false, + next_cursor: null, + }), + ) + } + return Promise.resolve(new Response('nope', { status: 404 })) + }) + await expect( + falProvider.fetchSpec({ FAL_KEY: 'test-key' }), + ).rejects.toThrow('404') + }) + it('throws when the unlisted lookup fails with anything but 404', async () => { fetchMock.mockImplementation((input) => { const url = requestUrl(input) diff --git a/src/server/providers/fal.ts b/src/server/providers/fal.ts index 10fa749d..17952b6d 100644 --- a/src/server/providers/fal.ts +++ b/src/server/providers/fal.ts @@ -75,6 +75,13 @@ interface FalApiModel { */ const FAL_UNLISTED_ENDPOINT_IDS = ['fal-ai/whisper'] +/** + * Body of a filtered models request whose `endpoint_id` FAL no longer + * serves. One such id 404s the whole batch (`Endpoint(s) not found`). + * Any other 404 is a different failure and must still throw. + */ +const FAL_ENDPOINT_NOT_FOUND = 'Endpoint(s) not found' + function openApiCategory(model: FalApiModel): string | null { const meta = model.openapi?.info?.['x-fal-metadata'] if (typeof meta !== 'object' || meta === null || !('category' in meta)) { @@ -179,11 +186,48 @@ class FalFetchError extends Error { constructor( message: string, readonly status: number, + readonly body: string, ) { super(message) } } +/** True when FAL says this filtered request names an endpoint it no longer serves. */ +function falEndpointGone(error: unknown): error is FalFetchError { + if (!(error instanceof FalFetchError) || error.status !== 404) return false + try { + const parsed: unknown = JSON.parse(error.body) + if (typeof parsed !== 'object' || parsed === null || !('error' in parsed)) { + return false + } + const inner = parsed.error + if (typeof inner !== 'object' || inner === null) return false + return ( + 'type' in inner && + inner.type === 'not_found' && + 'message' in inner && + inner.message === FAL_ENDPOINT_NOT_FOUND + ) + } catch { + return false + } +} + +/** The single-id request that proved `id` is gone, for the poll log. */ +function goneEndpointLocation(id: string, expandOpenApi: boolean): string { + const params = new URLSearchParams({ status: 'active' }) + if (expandOpenApi) params.set('expand', 'openapi-3.0') + params.append('endpoint_id', id) + return `${FAL_MODELS_URL}?${params.toString()}` +} + +function reportGoneEndpoint(id: string, expandOpenApi: boolean): void { + const location = goneEndpointLocation(id, expandOpenApi) + console.error( + `fal ${id}: no longer served at ${location} (404 ${FAL_ENDPOINT_NOT_FOUND}); omitting the row so the poll deprecates it`, + ) +} + /** * The paginated crawl is ~30 requests per pass and runs from both cron * tiers, so transient 429s/5xxs and dropped connections are routine — each @@ -204,11 +248,13 @@ export async function fetchPageWithRetry( signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), }) if (response.ok) return (await response.json()) as FalApiResponse + const body = await response.text() lastFailure = `${String(response.status)} ${response.statusText}` if (!RETRYABLE_STATUSES.has(response.status)) { throw new FalFetchError( `fal models fetch failed: ${lastFailure}`, response.status, + body, ) } const retryAfter = Number(response.headers.get('retry-after')) @@ -253,11 +299,53 @@ async function fetchFalModels( return allModels } +/** + * Fetch by `endpoint_id`. A removed id 404s every filter it sits in, so + * split until that id is alone, log it, and return the rows FAL still + * serves. A 404 that is not {@link FAL_ENDPOINT_NOT_FOUND} still throws. + */ +async function fetchFalModelsAllowingGone( + apiKey: string, + expandOpenApi: boolean, + endpointIds: Array, +): Promise<{ models: Array; gone: Array }> { + try { + return { + models: await fetchFalModels(apiKey, expandOpenApi, endpointIds), + gone: [], + } + } catch (error) { + if (!falEndpointGone(error) || endpointIds.length === 0) throw error + if (endpointIds.length === 1) { + const only = endpointIds[0] + if (only === undefined) throw error + reportGoneEndpoint(only, expandOpenApi) + return { models: [], gone: [only] } + } + const mid = Math.ceil(endpointIds.length / 2) + const left = await fetchFalModelsAllowingGone( + apiKey, + expandOpenApi, + endpointIds.slice(0, mid), + ) + const right = await fetchFalModelsAllowingGone( + apiKey, + expandOpenApi, + endpointIds.slice(mid), + ) + return { + models: [...left.models, ...right.models], + gone: [...left.gone, ...right.gone], + } + } +} + /** * Append {@link FAL_UNLISTED_ENDPOINT_IDS} that the active crawl did not - * return. Only a 404 (the id is gone) is skipped, so the row is marked - * removed. Any other failure throws: skipping it would also drop the row, - * and the poller would emit a false `model.removed`. + * return. Only an id FAL reports as {@link FAL_ENDPOINT_NOT_FOUND} is + * skipped, so the row is marked removed. Any other failure throws: + * skipping it would also drop the row, and the poller would emit a false + * `model.removed`. */ async function appendUnlistedFalModels( apiKey: string, @@ -268,13 +356,11 @@ async function appendUnlistedFalModels( const extra: Array = [] for (const id of FAL_UNLISTED_ENDPOINT_IDS) { if (seen.has(id)) continue - let page: Array - try { - page = await fetchFalModels(apiKey, expandOpenApi, [id]) - } catch (error) { - if (error instanceof FalFetchError && error.status === 404) continue - throw error - } + const { models: page } = await fetchFalModelsAllowingGone( + apiKey, + expandOpenApi, + [id], + ) const match = page.find((model) => model.endpoint_id === id) if (!match || seen.has(match.endpoint_id)) continue seen.add(match.endpoint_id) @@ -546,37 +632,43 @@ async function listModels( }) const chatIds = chat.map((m) => m.endpoint_id) const docs = docsRun() + // Ids the active search still returns but `endpoint_id` lookup 404s. + // Left out of the rows below, so the poll deprecates them. + const gone = new Set() const chatFacts = await tryDocs( docs, `${FAL_MODELS_URL}?expand=openapi-3.0 (chat endpoints)`, async () => { const chatSpecs = new Map() for (let i = 0; i < chatIds.length; i += ENDPOINT_FILTER_MAX) { - const page = await fetchFalModels( - apiKey, - true, - chatIds.slice(i, i + ENDPOINT_FILTER_MAX), - ) + const batch = chatIds.slice(i, i + ENDPOINT_FILTER_MAX) + const { models: page, gone: batchGone } = + await fetchFalModelsAllowingGone(apiKey, true, batch) + for (const id of batchGone) gone.add(id) for (const m of page) chatSpecs.set(m.endpoint_id, m.openapi) } - const unanswered = chatIds.filter((id) => !chatSpecs.has(id)) + const unanswered = chatIds.filter( + (id) => !chatSpecs.has(id) && !gone.has(id), + ) if (unanswered.length > 0) { throw new Error( `fal chat specs: models API left out ${unanswered.join(', ')}`, ) } return new Map( - chat.map((m) => [ - m.endpoint_id, - falChatFacts( - { - endpoint_id: m.endpoint_id, - description: m.metadata?.description, - openapi: chatSpecs.get(m.endpoint_id), - }, - FAL_MODELS_URL, - ), - ]), + chat + .filter((m) => !gone.has(m.endpoint_id)) + .map((m) => [ + m.endpoint_id, + falChatFacts( + { + endpoint_id: m.endpoint_id, + description: m.metadata?.description, + openapi: chatSpecs.get(m.endpoint_id), + }, + FAL_MODELS_URL, + ), + ]), ) }, ) @@ -588,11 +680,18 @@ async function listModels( 'reasoning', 'requestMap', ) - const tokenPrices = await loadChatTokenPrices(chatIds, docs, kv) + const tokenPrices = await loadChatTokenPrices( + chatIds.filter((id) => !gone.has(id)), + docs, + kv, + ) // After the chat spec request, so that request stays the second models // call. The lookup expands OpenAPI: unlisted rows carry no metadata, and - // the category is on the document. - const falModels = await appendUnlistedFalModels(apiKey, listedModels, true) + // the category is on the document. A chat id the filter proved gone is + // dropped here: the active search still listed it. + const falModels = ( + await appendUnlistedFalModels(apiKey, listedModels, true) + ).filter((model) => !gone.has(model.endpoint_id)) const models: Array = falModels.map((m) => { const facts = chatFacts?.get(m.endpoint_id) const price = tokenPrices.get(m.endpoint_id)