From 0c54ed18b947d2af0b99fa5ffe77ae2502974a59 Mon Sep 17 00:00:00 2001 From: Cato Date: Sun, 13 Sep 2026 14:18:56 +0200 Subject: [PATCH] fix(cat): the health probe asked Groq for a decommissioned model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit probeGroq pinned `llama-3.1-8b-instant`. groq-models.ts names that retirement in its own comments — Groq withdrew the whole llama-3.x family — and the id appears nowhere else in the codebase except those comments. So the Groq health probe has been asking for a model that does not exist, and `catCanAnswer` and `groqCanServeCatPrompt` are derived from its answer. The OpenRouter probe immediately below it already carries the fix and the reason: "a hardcoded id here once drifted from the registry and kept 'passing' while chat 404'd (and later kept 'failing' after the registry was fixed)". The lesson was applied to one probe and not its neighbour — the same asymmetry that left Groq without a catalogue check until #1004. It now probes PLATFORM_GROQ_MODEL, the model the platform chain actually serves a free user, so the health status tests something a user depends on. A gate pins that neither probe carries a literal model id. Also corrects a claim I made in #1003: that route's comment said the check "costs no tokens". The catalogue checks are GET /models and cost nothing, but probeGroq and probeOpenRouter send REAL chat completions — a few tokens each, from the same free pools the check exists to protect. Cheap, not free, which is why it runs daily rather than hourly. Verified: 2930 unit tests green, type-check and lint clean. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HJRuvHJEBd8t7iRA9Sb1iw --- .../unit/cat/one-catalogue-check.test.ts | 38 +++++++++++++++++-- src/app/api/cron/cat-health/route.ts | 7 +++- src/services/cat/health-probes.ts | 11 +++++- 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/__tests__/unit/cat/one-catalogue-check.test.ts b/__tests__/unit/cat/one-catalogue-check.test.ts index 84125a271..5121e83c6 100644 --- a/__tests__/unit/cat/one-catalogue-check.test.ts +++ b/__tests__/unit/cat/one-catalogue-check.test.ts @@ -13,6 +13,8 @@ * the thing being checked for. So the description reads from the config * OrangeCat actually serves from, and this pins that. */ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; import { orangecatChain } from '@/services/cat/provider-catalog'; import { getFreeModels } from '@/config/ai-models'; import { CONFIGURED_GROQ_MODEL_IDS } from '@/services/ai/groq-models'; @@ -45,10 +47,10 @@ describe('the chain description is read, not written', () => { }); describe('the old per-provider probes are gone', () => { - const src = require('node:fs').readFileSync( - require('node:path').join(__dirname, '../../../src/services/cat/health-probes.ts'), + const src = readFileSync( + join(__dirname, '../../../src/services/cat/health-probes.ts'), 'utf8' - ) as string; + ); it('leaves no second implementation behind to drift', () => { // Adopting a shared check and leaving the copy in place is how two answers @@ -74,3 +76,33 @@ describe('the old per-provider probes are gone', () => { expect(src).toContain("v.live === null ? null : v.missing"); }); }); + +describe('the health probe asks for a model the chain serves', () => { + // This probed `llama-3.1-8b-instant` long after Groq withdrew the whole + // llama-3.x family — groq-models.ts names that retirement in its own + // comments. So the Groq health status, and `catCanAnswer` derived from it, + // came from asking for a decommissioned model. The OpenRouter probe directly + // below it carries a comment warning about exactly this drift: the fix was + // applied there and not here. + const src = readFileSync( + join(__dirname, '../../../src/services/cat/health-probes.ts'), + 'utf8' + ); + + it('probes Groq with the model the platform actually offers', () => { + expect(src).toContain('PLATFORM_GROQ_MODEL'); + expect(src).not.toContain("'llama-3.1-8b-instant'"); + }); + + it('pins no literal model id in either provider probe', () => { + // A literal here is the drift itself: it keeps "passing" while chat 404s, + // and keeps "failing" after the registry is fixed. + const from = src.indexOf('export function probeGroq'); + const to = src.indexOf('Does Groq', from); + const probes = src.slice(from, to); + expect(probes).toContain('PLATFORM_GROQ_MODEL'); + expect(probes).toContain('DEFAULT_FREE_MODEL_ID'); + // No quoted model-looking literal passed as the 4th probeProvider argument. + expect(probes).not.toMatch(/,\s*'[a-z0-9][a-z0-9._\/-]{4,}'\s*\n\s*\)/); + }); +}); diff --git a/src/app/api/cron/cat-health/route.ts b/src/app/api/cron/cat-health/route.ts index 9ee3a2f6d..a9733d93b 100644 --- a/src/app/api/cron/cat-health/route.ts +++ b/src/app/api/cron/cat-health/route.ts @@ -8,8 +8,11 @@ * all. Its only callers were the diagnose route and a Cat action, so the answer * existed only when a human thought to ask for it. This puts it on a clock. * - * Costs no tokens: the catalogue checks are GET /models, and the probes are the - * same ones the diagnose route already runs. That is what makes it schedulable. + * Cost, stated correctly after getting it wrong once: the catalogue checks are + * GET /models and cost nothing, but probeGroq and probeOpenRouter send REAL + * chat completions — a few tokens each, from the same free pools the check + * exists to protect. Cheap, not free. That is why this runs daily rather than + * hourly, and why the alert coalesces instead of stacking a row per run. */ import { runCatHealthProbes } from '@/services/cat/health-probes'; diff --git a/src/services/cat/health-probes.ts b/src/services/cat/health-probes.ts index 2a1e8c0d7..0c656661f 100644 --- a/src/services/cat/health-probes.ts +++ b/src/services/cat/health-probes.ts @@ -13,7 +13,7 @@ import { PROVIDER_BASE_URLS } from '@/config/ai-provider-runtime'; import { DEFAULT_FREE_MODEL_ID } from '@/config/ai-models'; -import { promptFitsGroqOnDemand } from '@/services/ai/groq'; +import { promptFitsGroqOnDemand, PLATFORM_GROQ_MODEL } from '@/services/ai/groq'; import { checkModelRot } from './provider-catalog'; import { webSearch, describeAttempts } from '@bitbaum/ai-kit/web'; import { buildCatSystemPrompt } from './system-prompt'; @@ -151,11 +151,18 @@ async function probeProvider( } export function probeGroq(): Promise { + // The model the PLATFORM actually serves, for the same reason the OpenRouter + // probe below stopped hardcoding one. This asked for `llama-3.1-8b-instant` + // long after Groq withdrew the whole llama-3.x family — groq-models.ts names + // that retirement in its own comments — so the health probe had been asking + // for a decommissioned model, and `catCanAnswer` is derived from its answer. + // A probe pinned to something the chain does not serve tests nothing a user + // depends on. return probeProvider( 'groq', 'GROQ_API_KEY', `${PROVIDER_BASE_URLS.groq}/chat/completions`, - 'llama-3.1-8b-instant' + PLATFORM_GROQ_MODEL ); }