Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 35 additions & 3 deletions __tests__/unit/cat/one-catalogue-check.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
* the thing being checked for. So the description reads from the config
* OrangeCat actually serves from, and this pins that.
*/
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import { orangecatChain } from '@/services/cat/provider-catalog';
import { getFreeModels } from '@/config/ai-models';
import { CONFIGURED_GROQ_MODEL_IDS } from '@/services/ai/groq-models';
Expand Down Expand Up @@ -45,10 +47,10 @@ describe('the chain description is read, not written', () => {
});

describe('the old per-provider probes are gone', () => {
const src = require('node:fs').readFileSync(
require('node:path').join(__dirname, '../../../src/services/cat/health-probes.ts'),
const src = readFileSync(
join(__dirname, '../../../src/services/cat/health-probes.ts'),
'utf8'
) as string;
);

it('leaves no second implementation behind to drift', () => {
// Adopting a shared check and leaving the copy in place is how two answers
Expand All @@ -74,3 +76,33 @@ describe('the old per-provider probes are gone', () => {
expect(src).toContain("v.live === null ? null : v.missing");
});
});

describe('the health probe asks for a model the chain serves', () => {
// This probed `llama-3.1-8b-instant` long after Groq withdrew the whole
// llama-3.x family — groq-models.ts names that retirement in its own
// comments. So the Groq health status, and `catCanAnswer` derived from it,
// came from asking for a decommissioned model. The OpenRouter probe directly
// below it carries a comment warning about exactly this drift: the fix was
// applied there and not here.
const src = readFileSync(
join(__dirname, '../../../src/services/cat/health-probes.ts'),
'utf8'
);

it('probes Groq with the model the platform actually offers', () => {
expect(src).toContain('PLATFORM_GROQ_MODEL');
expect(src).not.toContain("'llama-3.1-8b-instant'");
});

it('pins no literal model id in either provider probe', () => {
// A literal here is the drift itself: it keeps "passing" while chat 404s,
// and keeps "failing" after the registry is fixed.
const from = src.indexOf('export function probeGroq');
const to = src.indexOf('Does Groq', from);
const probes = src.slice(from, to);
expect(probes).toContain('PLATFORM_GROQ_MODEL');
expect(probes).toContain('DEFAULT_FREE_MODEL_ID');
// No quoted model-looking literal passed as the 4th probeProvider argument.
expect(probes).not.toMatch(/,\s*'[a-z0-9][a-z0-9._\/-]{4,}'\s*\n\s*\)/);
});
});
7 changes: 5 additions & 2 deletions src/app/api/cron/cat-health/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,11 @@
* all. Its only callers were the diagnose route and a Cat action, so the answer
* existed only when a human thought to ask for it. This puts it on a clock.
*
* Costs no tokens: the catalogue checks are GET /models, and the probes are the
* same ones the diagnose route already runs. That is what makes it schedulable.
* Cost, stated correctly after getting it wrong once: the catalogue checks are
* GET /models and cost nothing, but probeGroq and probeOpenRouter send REAL
* chat completions — a few tokens each, from the same free pools the check
* exists to protect. Cheap, not free. That is why this runs daily rather than
* hourly, and why the alert coalesces instead of stacking a row per run.
*/

import { runCatHealthProbes } from '@/services/cat/health-probes';
Expand Down
11 changes: 9 additions & 2 deletions src/services/cat/health-probes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

import { PROVIDER_BASE_URLS } from '@/config/ai-provider-runtime';
import { DEFAULT_FREE_MODEL_ID } from '@/config/ai-models';
import { promptFitsGroqOnDemand } from '@/services/ai/groq';
import { promptFitsGroqOnDemand, PLATFORM_GROQ_MODEL } from '@/services/ai/groq';
import { checkModelRot } from './provider-catalog';
import { webSearch, describeAttempts } from '@bitbaum/ai-kit/web';
import { buildCatSystemPrompt } from './system-prompt';
Expand Down Expand Up @@ -151,11 +151,18 @@ async function probeProvider(
}

export function probeGroq(): Promise<ProbeResult> {
// The model the PLATFORM actually serves, for the same reason the OpenRouter
// probe below stopped hardcoding one. This asked for `llama-3.1-8b-instant`
// long after Groq withdrew the whole llama-3.x family — groq-models.ts names
// that retirement in its own comments — so the health probe had been asking
// for a decommissioned model, and `catCanAnswer` is derived from its answer.
// A probe pinned to something the chain does not serve tests nothing a user
// depends on.
return probeProvider(
'groq',
'GROQ_API_KEY',
`${PROVIDER_BASE_URLS.groq}/chat/completions`,
'llama-3.1-8b-instant'
PLATFORM_GROQ_MODEL
);
}

Expand Down
Loading