Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 115 additions & 1 deletion src/server/providers/fal-chat-facts.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,15 @@ describe('fal listModels', () => {
)
}

function endpointNotFound(): Response {
return new Response(
JSON.stringify({
error: { type: 'not_found', message: 'Endpoint(s) not found' },
}),
{ status: 404, headers: { 'Content-Type': 'application/json' } },
)
}

function respond(
input: string,
llms: (pathname: string) => string,
Expand Down Expand Up @@ -495,12 +504,13 @@ describe('fal listModels', () => {
})

it('skips fal-ai/whisper when the direct lookup 404s', async () => {
const error = vi.spyOn(console, 'error').mockImplementation(() => undefined)
vi.stubGlobal('fetch', (input: string) => {
const url = new URL(input)
if (url.hostname === 'fal.ai')
return Promise.resolve(new Response(NO_TOKEN_PRICE))
if (url.searchParams.get('endpoint_id') === 'fal-ai/whisper') {
return Promise.resolve(new Response('missing', { status: 404 }))
return Promise.resolve(endpointNotFound())
}
return Promise.resolve(
url.searchParams.has('endpoint_id')
Expand All @@ -511,6 +521,110 @@ describe('fal listModels', () => {
const { models } = await falProvider.listModels({ FAL_KEY: 'k' })
expect(models.some((model) => model.rawId === 'fal-ai/whisper')).toBe(false)
expect(models).toHaveLength(listed.length + 1)
expect(error).toHaveBeenCalledWith(
expect.stringContaining(
'fal-ai/whisper: no longer served at https://api.fal.ai/v1/models?',
) as string,
)
const line = String(error.mock.calls[0]?.[0])
expect(line).toContain('endpoint_id=fal-ai%2Fwhisper')
expect(line).toContain('404 Endpoint(s) not found')
error.mockRestore()
})

it('fails the poll when an unlisted lookup 404s for another reason', async () => {
vi.stubGlobal('fetch', (input: string) => {
const url = new URL(input)
if (url.hostname === 'fal.ai')
return Promise.resolve(new Response(NO_TOKEN_PRICE))
if (url.searchParams.get('endpoint_id') === 'fal-ai/whisper') {
return Promise.resolve(new Response('missing', { status: 404 }))
}
return Promise.resolve(
url.searchParams.has('endpoint_id')
? page(fixtures)
: page([image, ...listed]),
)
})
await expect(falProvider.listModels({ FAL_KEY: 'k' })).rejects.toThrow(
'fal models fetch failed: 404',
)
})

it('omits a listed chat endpoint whose endpoint_id filter 404s', async () => {
const goneId = 'fal-ai/video-prompt-generator'
const error = vi.spyOn(console, 'error').mockImplementation(() => undefined)
vi.stubGlobal('fetch', (input: string) => {
const url = new URL(input)
if (url.hostname === 'fal.ai')
return Promise.resolve(new Response(NO_TOKEN_PRICE))
const ids = url.searchParams.getAll('endpoint_id')
if (ids.includes(goneId)) return Promise.resolve(endpointNotFound())
if (ids.length === 0) {
return Promise.resolve(
page([
image,
...listed,
{ endpoint_id: goneId, metadata: { category: 'llm' } },
]),
)
}
if (ids.length === 1 && ids[0] === 'fal-ai/whisper') {
return Promise.resolve(page([]))
}
const wanted = new Set(ids)
return Promise.resolve(
page(fixtures.filter((model) => wanted.has(model.endpoint_id))),
)
})
const result = await falProvider.listModels({ FAL_KEY: 'k' })
expect(result.models.some((model) => model.rawId === goneId)).toBe(false)
expect(result.models).toHaveLength(listed.length + 1)
expect(result.docsFailures?.failed).toBe(0)
expect(
result.models.find(
(model) => model.rawId === 'fal-ai/bytedance/seed/v2/mini',
),
).toMatchObject({ activity: 'chat', contextWindow: 256_000 })
const line = error.mock.calls.map((call) => String(call[0])).join('\n')
expect(line).toContain(
`${goneId}: no longer served at https://api.fal.ai/v1/models?`,
)
expect(line).toContain('endpoint_id=fal-ai%2Fvideo-prompt-generator')
expect(line).toContain('expand=openapi-3.0')
expect(line).not.toContain('fal-ai/whisper')
error.mockRestore()
})

it('keeps the rows when a filtered 404 is not Endpoint(s) not found', async () => {
const goneId = 'fal-ai/video-prompt-generator'
vi.stubGlobal('fetch', (input: string) => {
const url = new URL(input)
if (url.hostname === 'fal.ai')
return Promise.resolve(new Response(NO_TOKEN_PRICE))
const ids = url.searchParams.getAll('endpoint_id')
if (ids.includes(goneId)) {
return Promise.resolve(new Response('nope', { status: 404 }))
}
if (ids.length === 1 && ids[0] === 'fal-ai/whisper') {
return Promise.resolve(page([]))
}
return Promise.resolve(
ids.length === 0
? page([
image,
...listed,
{ endpoint_id: goneId, metadata: { category: 'llm' } },
])
: page(fixtures),
)
})
const result = await falProvider.listModels({ FAL_KEY: 'k' })
expect(result.models.some((model) => model.rawId === goneId)).toBe(true)
expect(result.docsFailures?.first[0]?.error).toContain(
'fal models fetch failed: 404',
)
expect(result.models).toHaveLength(listed.length + 2)
})

it('marks an unlisted endpoint deprecated once retires_at is past', async () => {
Expand Down
30 changes: 29 additions & 1 deletion src/server/providers/fal.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ describe('fetchPageWithRetry', () => {
})

it('gives up after the attempt budget with the last failure in the message', async () => {
fetchMock.mockResolvedValue(errorResponse(429))
fetchMock.mockImplementation(() => Promise.resolve(errorResponse(429)))
await expect(
withTimersFlushed(fetchPageWithRetry('https://x', 'key', 3)),
).rejects.toThrow('fal models fetch failed after 3 attempts: 429')
Expand Down Expand Up @@ -353,6 +353,34 @@ describe('fal AsyncAPI discovery', () => {
expect(fetched.asyncApiRawIds).toEqual([])
})

it('throws when the unlisted lookup 404s without Endpoint(s) not found', async () => {
fetchMock.mockImplementation((input) => {
const url = requestUrl(input)
if (url.startsWith('https://api.fal.ai/v1/models')) {
if (new URL(url).searchParams.get('endpoint_id') === 'fal-ai/whisper') {
return Promise.resolve(new Response('missing', { status: 404 }))
}
return Promise.resolve(
jsonBody({
models: [
{
endpoint_id: 'fal-ai/flux/dev',
openapi: FLUX_OPENAPI,
metadata: { category: 'text-to-image' },
},
],
has_more: false,
next_cursor: null,
}),
)
}
return Promise.resolve(new Response('nope', { status: 404 }))
})
await expect(
falProvider.fetchSpec({ FAL_KEY: 'test-key' }),
).rejects.toThrow('404')
})

it('throws when the unlisted lookup fails with anything but 404', async () => {
fetchMock.mockImplementation((input) => {
const url = requestUrl(input)
Expand Down
Loading
Loading