diff --git a/__tests__/unit/cat/one-catalogue-check.test.ts b/__tests__/unit/cat/one-catalogue-check.test.ts index 84125a271..5121e83c6 100644 --- a/__tests__/unit/cat/one-catalogue-check.test.ts +++ b/__tests__/unit/cat/one-catalogue-check.test.ts @@ -13,6 +13,8 @@ * the thing being checked for. So the description reads from the config * OrangeCat actually serves from, and this pins that. */ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; import { orangecatChain } from '@/services/cat/provider-catalog'; import { getFreeModels } from '@/config/ai-models'; import { CONFIGURED_GROQ_MODEL_IDS } from '@/services/ai/groq-models'; @@ -45,10 +47,10 @@ describe('the chain description is read, not written', () => { }); describe('the old per-provider probes are gone', () => { - const src = require('node:fs').readFileSync( - require('node:path').join(__dirname, '../../../src/services/cat/health-probes.ts'), + const src = readFileSync( + join(__dirname, '../../../src/services/cat/health-probes.ts'), 'utf8' - ) as string; + ); it('leaves no second implementation behind to drift', () => { // Adopting a shared check and leaving the copy in place is how two answers @@ -74,3 +76,33 @@ describe('the old per-provider probes are gone', () => { expect(src).toContain("v.live === null ? null : v.missing"); }); }); + +describe('the health probe asks for a model the chain serves', () => { + // This probed `llama-3.1-8b-instant` long after Groq withdrew the whole + // llama-3.x family — groq-models.ts names that retirement in its own + // comments. So the Groq health status, and `catCanAnswer` derived from it, + // came from asking for a decommissioned model. The OpenRouter probe directly + // below it carries a comment warning about exactly this drift: the fix was + // applied there and not here. + const src = readFileSync( + join(__dirname, '../../../src/services/cat/health-probes.ts'), + 'utf8' + ); + + it('probes Groq with the model the platform actually offers', () => { + expect(src).toContain('PLATFORM_GROQ_MODEL'); + expect(src).not.toContain("'llama-3.1-8b-instant'"); + }); + + it('pins no literal model id in either provider probe', () => { + // A literal here is the drift itself: it keeps "passing" while chat 404s, + // and keeps "failing" after the registry is fixed. + const from = src.indexOf('export function probeGroq'); + const to = src.indexOf('Does Groq', from); + const probes = src.slice(from, to); + expect(probes).toContain('PLATFORM_GROQ_MODEL'); + expect(probes).toContain('DEFAULT_FREE_MODEL_ID'); + // No quoted model-looking literal passed as the 4th probeProvider argument. + expect(probes).not.toMatch(/,\s*'[a-z0-9][a-z0-9._\/-]{4,}'\s*\n\s*\)/); + }); +}); diff --git a/src/app/api/cron/cat-health/route.ts b/src/app/api/cron/cat-health/route.ts index 9ee3a2f6d..a9733d93b 100644 --- a/src/app/api/cron/cat-health/route.ts +++ b/src/app/api/cron/cat-health/route.ts @@ -8,8 +8,11 @@ * all. Its only callers were the diagnose route and a Cat action, so the answer * existed only when a human thought to ask for it. This puts it on a clock. * - * Costs no tokens: the catalogue checks are GET /models, and the probes are the - * same ones the diagnose route already runs. That is what makes it schedulable. + * Cost, stated correctly after getting it wrong once: the catalogue checks are + * GET /models and cost nothing, but probeGroq and probeOpenRouter send REAL + * chat completions — a few tokens each, from the same free pools the check + * exists to protect. Cheap, not free. That is why this runs daily rather than + * hourly, and why the alert coalesces instead of stacking a row per run. */ import { runCatHealthProbes } from '@/services/cat/health-probes'; diff --git a/src/services/cat/health-probes.ts b/src/services/cat/health-probes.ts index 2a1e8c0d7..0c656661f 100644 --- a/src/services/cat/health-probes.ts +++ b/src/services/cat/health-probes.ts @@ -13,7 +13,7 @@ import { PROVIDER_BASE_URLS } from '@/config/ai-provider-runtime'; import { DEFAULT_FREE_MODEL_ID } from '@/config/ai-models'; -import { promptFitsGroqOnDemand } from '@/services/ai/groq'; +import { promptFitsGroqOnDemand, PLATFORM_GROQ_MODEL } from '@/services/ai/groq'; import { checkModelRot } from './provider-catalog'; import { webSearch, describeAttempts } from '@bitbaum/ai-kit/web'; import { buildCatSystemPrompt } from './system-prompt'; @@ -151,11 +151,18 @@ async function probeProvider( } export function probeGroq(): Promise { + // The model the PLATFORM actually serves, for the same reason the OpenRouter + // probe below stopped hardcoding one. This asked for `llama-3.1-8b-instant` + // long after Groq withdrew the whole llama-3.x family — groq-models.ts names + // that retirement in its own comments — so the health probe had been asking + // for a decommissioned model, and `catCanAnswer` is derived from its answer. + // A probe pinned to something the chain does not serve tests nothing a user + // depends on. return probeProvider( 'groq', 'GROQ_API_KEY', `${PROVIDER_BASE_URLS.groq}/chat/completions`, - 'llama-3.1-8b-instant' + PLATFORM_GROQ_MODEL ); }