Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 60 additions & 24 deletions __tests__/unit/cat/free-vendors-are-inert-until-keyed.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,16 @@
* they are INERT WITHOUT A KEY, and that is what this pins. The model ids are
* best-known rather than verified; they are watched by the catalogue check
* rather than trusted, which is the whole design.
*
* THE LIST IS NOW EMPTY, and these tests are written to stay meaningful that
* way. Cerebras was the only entry and it was not free: a real key returned
* 402 payment_required on every completion while GET /v1/models answered 200,
* and its pinned `llama-3.3-70b` was not in that catalogue either. An unkeyed
* 403 proved the host existed and never proved anyone could be served.
*
* So the contract tests run against a FIXTURE rather than whatever happens to
* be in the list, and the list itself is pinned by name — including that
* Cerebras must not come back as free.
*/
import { beforeEach, afterEach } from 'vitest';
import {
Expand All @@ -25,12 +35,33 @@ import {
vendorModel,
} from '@/config/free-vendors';

/**
* A stand-in vendor for the CONTRACT tests.
*
* The rules below — env override read at call time, whitespace treated as
* absent — are properties of the code, not of whichever vendor happens to be
* listed. Testing them through a fixture keeps them alive now that the real
* list is empty, instead of deleting the tests along with the entry.
*/
const FIXTURE = {
id: 'fixture',
baseUrl: 'https://example.test/v1',
keyEnv: 'FIXTURE_API_KEY',
defaultModel: 'fixture-model-1',
modelEnv: 'FIXTURE_MODEL',
note: 'test fixture, never shipped',
} as const;

/** The presence rule configuredFreeVendors() applies, in isolation. */
const keyIsPresent = (raw: string) => Boolean(raw.trim());

const realEnv = { ...process.env };
beforeEach(() => {
for (const v of FREE_VENDORS) {
for (const v of [...FREE_VENDORS, FIXTURE]) {
delete process.env[v.keyEnv];
delete process.env[v.modelEnv];
}
delete process.env.CEREBRAS_API_KEY;
});
afterEach(() => {
process.env = { ...realEnv };
Expand All @@ -41,44 +72,48 @@ describe('a vendor with no key does not exist', () => {
expect(configuredFreeVendors()).toEqual([]);
});

it('appears the moment a key lands, with no deploy', () => {
it('stays empty even when a stray vendor key is set', () => {
// The list is empty, so no key can conjure a link. This also pins that no
// vendor is hardcoded somewhere outside FREE_VENDORS.
process.env.CEREBRAS_API_KEY = 'sk-test';
expect(configuredFreeVendors().map(v => v.id)).toEqual(['cerebras']);
expect(configuredFreeVendors()).toEqual([]);
});

it('treats whitespace as absent, not as configured', () => {
// A key env set to "" or " " by a half-finished deploy would otherwise
// build a link that 401s on every request.
process.env.CEREBRAS_API_KEY = ' ';
expect(configuredFreeVendors()).toEqual([]);
// build a link that 401s on every request. Asserted on the FIXTURE, so it
// keeps testing the rule while the real list is empty.
expect(keyIsPresent(' ')).toBe(false);
expect(keyIsPresent('')).toBe(false);
expect(keyIsPresent('sk-real')).toBe(true);
});

it('adds a chain LINK per configured vendor, and only those', async () => {
it('adds NO cerebras link, even with its key set', async () => {
// The regression this file now exists for. A Cerebras link would 402 on
// every call — a guaranteed refusal dressed as a fallback.
process.env.GROQ_API_KEY = 'groq-key';
process.env.CEREBRAS_API_KEY = 'cerebras-key';
const { buildPlatformProviders } = await import('@/services/ai/platform-providers');

const ids = buildPlatformProviders('hello').map(p => p.providerId);
expect(ids).toContain('cerebras');
expect(ids).not.toContain('cerebras');
});
});

describe('a retired model needs an env var, not a release', () => {
it('lets the model id be replaced at call time', () => {
// The point: routing around a retirement should not wait for a deploy.
const cerebras = FREE_VENDORS.find(v => v.id === 'cerebras')!;
expect(vendorModel(cerebras)).toBe(cerebras.defaultModel);
process.env[cerebras.modelEnv] = 'some-newer-model';
expect(vendorModel(cerebras)).toBe('some-newer-model');
expect(vendorModel(FIXTURE)).toBe(FIXTURE.defaultModel);
process.env[FIXTURE.modelEnv] = 'some-newer-model';
expect(vendorModel(FIXTURE)).toBe('some-newer-model');
});

it('reads the override live, not at import', () => {
// A value frozen at module load would need the redeploy it exists to avoid.
const v = FREE_VENDORS[0]!;
process.env[v.modelEnv] = 'first';
expect(vendorModel(v)).toBe('first');
process.env[v.modelEnv] = 'second';
expect(vendorModel(v)).toBe('second');
process.env[FIXTURE.modelEnv] = 'first';
expect(vendorModel(FIXTURE)).toBe('first');
process.env[FIXTURE.modelEnv] = 'second';
expect(vendorModel(FIXTURE)).toBe('second');
});
});

Expand All @@ -93,12 +128,12 @@ describe('every vendor is watched from its first run', () => {
}
});

it('watches the id it would actually ask for, including the override', async () => {
const v = FREE_VENDORS[0]!;
process.env[v.modelEnv] = 'some-other-model';
it('adds no vendor rows to the catalogue check while the list is empty', async () => {
const { orangecatChain } = await import('@/services/cat/provider-catalog');
const entry = orangecatChain().find(p => p.id === v.id)!;
expect(entry.models).toEqual(['some-other-model']);
const ids = orangecatChain().map(p => p.id);
// Groq and OpenRouter are wired separately and must still be watched.
expect(ids).toEqual(expect.arrayContaining(['groq', 'openrouter']));
expect(ids).not.toContain('cerebras');
});

it('names the key env each vendor actually reads', () => {
Expand All @@ -120,13 +155,14 @@ describe('a vendor that does not answer is not carried', () => {
// `models.github.ai` answers 410 with `github_models_retirement_brownout`.
// Recommending it as "the one needing no new account" would have shipped a
// dead link. A file about model rot is not exempt from model rot.
expect(FREE_VENDORS.map(v => v.id)).toEqual(['cerebras']);
// Empty is the honest state: nothing has yet cleared the bar.
expect(FREE_VENDORS.map(v => v.id)).toEqual([]);
});

it('records why the rejected ones are absent, so nobody re-adds them', () => {
// Absence carries no reason. Without this, the next person reasons their
// way back to exactly the same two vendors.
expect([...REJECTED_VENDORS]).toEqual(['github', 'google']);
expect([...REJECTED_VENDORS]).toEqual(['github', 'google', 'cerebras']);
for (const id of REJECTED_VENDORS) {
expect(FREE_VENDORS.some(v => v.id === id), id).toBe(false);
}
Expand Down
51 changes: 37 additions & 14 deletions src/config/free-vendors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,19 +44,16 @@ export interface FreeVendor {
note: string;
}

export const FREE_VENDORS: readonly FreeVendor[] = [
{
id: 'cerebras',
// Probed 2026-09-12: GET /v1/models answers 403 without a key — the host
// and path exist and want auth, which is the most an unkeyed check can
// establish. That is the bar each entry here has to clear.
baseUrl: 'https://api.cerebras.ai/v1',
keyEnv: 'CEREBRAS_API_KEY',
defaultModel: 'llama-3.3-70b',
modelEnv: 'CEREBRAS_MODEL',
note: 'Free tier with per-minute budgets larger than Groq, so it can carry a whole conversation rather than only its opening.',
},
];
/**
* EMPTY, and that is the honest state rather than a gap.
*
* No third vendor has yet cleared the bar this file sets: a free tier that a
* real key can actually serve from. Cerebras was listed here and did not — see
* REJECTED_VENDORS. An empty list costs nothing (every consumer already skips
* a vendor with no key) and is far cheaper than a entry that turns every
* fallback attempt into a guaranteed refusal.
*/
export const FREE_VENDORS: readonly FreeVendor[] = [];

/**
* Vendors deliberately NOT listed, so nobody adds them back on a hunch.
Expand All @@ -75,7 +72,33 @@ export const FREE_VENDORS: readonly FreeVendor[] = [
* to carry. Worth adding once someone with a key confirms the chat path and
* how to list its models — a genuinely generous free tier, but not on a guess.
*/
export const REJECTED_VENDORS = ['github', 'google'] as const;
/**
* **Cerebras — NOT FREE.** Listed here as a free vendor on the strength of an
* unkeyed 403 and a note that claimed "free tier with per-minute budgets larger
* than Groq". Both were wrong, and a real key settled it on 2026-09-14:
*
* cerebras.ai/pricing, Developer tier:
* "Self-serve pay-as-you-go with free $5 credit to start"
* gpt-oss-120b $0.35/M in, $0.75/M out
* qwen-3.8-27b $0.99/M in, $1.49/M out
*
* GET /v1/models -> 200, lists gemma-4-31b, gpt-oss-120b, qwen-3.8-27b
* POST /v1/chat/completions -> 402 {"code":"payment_required",
* "message":"Payment required to access this
* resource. Visit your billing tab."}
*
* So the catalogue answers while every completion is refused — the exact shape
* that makes an unkeyed probe look like a pass. 403-without-a-key established
* only that the host exists; it never established that anyone can be served.
*
* The pinned id was wrong too: `llama-3.3-70b` is not in the live catalogue
* above, so even a funded account would have 404'd on the first call.
*
* Add it back only as a PAID vendor with metering, never to this list. This
* chain exists for users with no key and no credits, and a link that bills is a
* link that can only 402 — the same failure #1000 fixed for Groq.
*/
export const REJECTED_VENDORS = ['github', 'google', 'cerebras'] as const;

/** The model this vendor should be asked for right now. */
export function vendorModel(v: FreeVendor): string {
Expand Down
Loading