diff --git a/docs/commands.md b/docs/commands.md index 87f8b40..c16fd35 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -48,7 +48,7 @@ Run each command in your repo. Bugpatrol finds the `.bugpatrol/` folder in the c | --- | --- | | `bugpatrol adopt --config .bugpatrol/.yml` | Move the app map, endpoint inventory and routines of `.bugpatrol/` into the folder of that config, for a config that kept them there before profiles existed. Refer to [Configuration](configuration.md#migrating-an-existing-non-default-config) | | `bugpatrol dashboard [--port N]` | The local dashboard, on 127.0.0.1 | -| `bugpatrol doctor [--config ]` | Check that this machine can run the selected config: the config file, Node, the LLM of each agent, `gh` when `agents.github` is on, and what the platform needs. For web, Chromium for Playwright, with the install command for the pinned version. For web and api with no background setup command, that `connect.url` answers. For cli, `python3`; for desktop, Linux and the launch commands. A config with a `run` block also gets the pinned runner image and the AppModel. Exits 2 when a required check fails | +| `bugpatrol doctor [--gate] [--config ]` | Check that this machine can run the selected config: the config file, Node, the LLM of each agent, `gh` when `agents.github` is on, and what the platform needs. For web, Chromium for Playwright, with the install command for the pinned version. For web and api with no background setup command, that `connect.url` answers. For cli, `python3`; for desktop, Linux and the launch commands. A config with a `run` block also gets the pinned runner image and the AppModel. `--gate` checks only what the deterministic web gate needs: it leaves out the agent LLMs, `gh` and the checks of the `app` block (`connect.url`, `python3`, the desktop tools), since `bugpatrol run` calls no agent and starts the app from `run`, and fails when the config has no `run` block. Exits 2 when a required check fails | | `bugpatrol init [--yes] [--agent a]` | Detect the app and the LLMs, ask for each agent's provider, and write `.bugpatrol/bugpatrol.yml` and `.bugpatrol/instructions.md` | | `bugpatrol init --platform api [--url ] [--spec ]` | Write a starter config for an API that runs already. It asks for the URL and an optional spec, and detects no start command | | `bugpatrol init --gate` | Write a starter `.bugpatrol/bugpatrol.yml` for the deterministic web gate | diff --git a/docs/deterministic-gate.md b/docs/deterministic-gate.md index 8f566f4..e14e59f 100644 --- a/docs/deterministic-gate.md +++ b/docs/deterministic-gate.md @@ -7,6 +7,7 @@ For web apps, `bugpatrol run` is a merge gate that uses no agent. The agents exp ```bash PLAYWRIGHT_SKIP_BROWSER_GC=1 npx -y -p bugpatrol@latest playwright install chromium npx bugpatrol init --gate # write a starter .bugpatrol/bugpatrol.yml; correct each TODO value +npx bugpatrol doctor --gate # check the browser and the config, with no model key needed npx bugpatrol run --no-models # capture the baselines npx bugpatrol run --no-models # compare: clean ``` diff --git a/packages/cli/src/bin.ts b/packages/cli/src/bin.ts index 741c91f..40c8ee9 100644 --- a/packages/cli/src/bin.ts +++ b/packages/cli/src/bin.ts @@ -14,7 +14,7 @@ import { import { startDashboard } from '@bugpatrol/dashboard'; import { configArgs, legacyEnv, withDefaultCommand } from './argv.js'; import { runAgentCommand } from './commands/agents.js'; -import { doctorExitCode, readDoctorConfig, runChecks } from './commands/doctor.js'; +import { doctorExitCode, parseDoctorFlags, readDoctorConfig, runChecks } from './commands/doctor.js'; import { runEndpointsCommand } from './commands/endpoints.js'; import { runFindingsCommand } from './commands/findings.js'; import { runInit } from './commands/init.js'; @@ -79,7 +79,8 @@ try { } case 'doctor': { - const checks = await runChecks(root, readDoctorConfig(root, selected.configFile)); + const options = parseDoctorFlags(args); + const checks = await runChecks(root, readDoctorConfig(root, selected.configFile), undefined, options); for (const check of checks) { process.stdout.write(`${check.ok ? 'ok ' : check.fatal ? 'FAIL' : 'warn'} ${check.name}: ${check.detail}\n`); } diff --git a/packages/cli/src/commands/doctor.test.ts b/packages/cli/src/commands/doctor.test.ts index a363021..ef59f0d 100644 --- a/packages/cli/src/commands/doctor.test.ts +++ b/packages/cli/src/commands/doctor.test.ts @@ -5,7 +5,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { parseConfig } from '@bugpatrol/core'; import { describe, expect, it } from 'vitest'; -import { type DoctorProbes, readDoctorConfig, runChecks } from './doctor.js'; +import { type DoctorProbes, doctorExitCode, parseDoctorFlags, readDoctorConfig, runChecks } from './doctor.js'; function project(yaml = 'version: 1\napp: { platform: cli }\n', file = 'bugpatrol.yml'): string { const root = mkdtempSync(join(tmpdir(), 'bugpatrol-doctor-')); @@ -121,3 +121,73 @@ describe('runChecks', () => { ); }); }); + +describe('doctor --gate', () => { + // The config that `init --gate` writes: a run block, and agents on OpenRouter by default. + const gateConfig = () => + parseConfig({ + version: 1, + run: { command: 'npm run dev', url: 'http://localhost:3000' }, + agents: { + explorer: { use: { runtime: 'model', via: 'openrouter' } }, + judge: { use: { runtime: 'model', via: 'openrouter' } }, + }, + }); + + it('passes a gate config with no model key, because the gate calls no agent', async () => { + const root = project(); + const loaded = { path: join(root, '.bugpatrol', 'bugpatrol.yml'), config: gateConfig() }; + const saved = process.env.OPENROUTER_API_KEY; + delete process.env.OPENROUTER_API_KEY; + try { + const gate = await runChecks(root, loaded, installed, { gate: true }); + expect(gate.map((check) => check.name)).not.toContain('explorer'); + expect(gate.map((check) => check.name)).not.toContain('judge'); + expect(gate.map((check) => check.name)).toEqual(expect.arrayContaining(['browser', 'pinned-image'])); + expect(doctorExitCode(gate)).toBe(0); + // Without --gate the same config still needs the agent LLMs. + const all = await runChecks(root, loaded, installed); + expect(all.find((check) => check.name === 'explorer')).toMatchObject({ ok: false, fatal: true }); + expect(doctorExitCode(all)).toBe(2); + } finally { + if (saved !== undefined) process.env.OPENROUTER_API_KEY = saved; + } + }); + + it('checks only the gate URL, not the agent connect.url or platform tools', async () => { + const root = project(); + const config = parseConfig({ + version: 1, + app: { platform: 'web', connect: { url: 'http://127.0.0.1:1' } }, + run: { command: 'npm run dev', url: 'http://localhost:3000' }, + }); + const checks = await runChecks(root, { path: join(root, '.bugpatrol', 'bugpatrol.yml'), config }, installed, { + gate: true, + }); + expect(checks.map((check) => check.name)).toEqual([ + 'config', + 'node', + 'browser', + 'pinned-image', + 'app-model', + 'intent-ledger', + ]); + expect(doctorExitCode(checks)).toBe(0); + }); + + it('fails a config with no run block, which the gate cannot run', async () => { + const root = project(); + const config = parseConfig({ version: 1, app: { platform: 'cli' } }); + const checks = await runChecks(root, { path: join(root, '.bugpatrol', 'bugpatrol.yml'), config }, installed, { + gate: true, + }); + expect(checks.find((check) => check.name === 'gate')).toMatchObject({ ok: false, fatal: true }); + expect(doctorExitCode(checks)).toBe(2); + }); + + it('parses --gate and rejects an unknown flag', () => { + expect(parseDoctorFlags([])).toEqual({}); + expect(parseDoctorFlags(['--gate'])).toEqual({ gate: true }); + expect(() => parseDoctorFlags(['--gtae'])).toThrow('Unknown flag for `bugpatrol doctor`: --gtae'); + }); +}); diff --git a/packages/cli/src/commands/doctor.ts b/packages/cli/src/commands/doctor.ts index 12f8f54..a2711b1 100644 --- a/packages/cli/src/commands/doctor.ts +++ b/packages/cli/src/commands/doctor.ts @@ -2,7 +2,15 @@ import { spawnSync } from 'node:child_process'; import { existsSync } from 'node:fs'; import { dirname, relative, resolve } from 'node:path'; import { desktopCommands, onPath, runtimeProblem } from '@bugpatrol/agents'; -import { type BugpatrolConfig, ExitCode, type ExitCodeValue, legacyLayout, loadConfig, paths } from '@bugpatrol/core'; +import { + type BugpatrolConfig, + ConfigError, + ExitCode, + type ExitCodeValue, + legacyLayout, + loadConfig, + paths, +} from '@bugpatrol/core'; import { chromiumProblem } from '@bugpatrol/drivers'; export type DoctorCheck = { name: string; ok: boolean; detail: string; fatal: boolean }; @@ -13,6 +21,19 @@ export type DoctorConfig = { path: string; config?: BugpatrolConfig; error?: str /** The machine probes, as a seam for the tests. */ export type DoctorProbes = { chromium: () => string | undefined }; +/** `gate`: check only what the deterministic web gate (`bugpatrol run`) needs, not the agents. */ +export type DoctorOptions = { gate?: boolean }; + +/** `bugpatrol doctor [--gate]`. Unknown flags are rejected, as `run` does. */ +export function parseDoctorFlags(args: string[]): DoctorOptions { + const options: DoctorOptions = {}; + for (const arg of args) { + if (arg === '--gate') options.gate = true; + else throw new ConfigError(`Unknown flag for \`bugpatrol doctor\`: ${arg}`); + } + return options; +} + export function readDoctorConfig(root: string, configFile?: string): DoctorConfig { const path = paths.config(root, configFile); if (!existsSync(path)) return { path }; @@ -39,11 +60,14 @@ function configCheck(root: string, { path, config, error }: DoctorConfig, select * the config itself, Node, the LLM of each agent, and what the platform * needs (the browser and the app URL for web, python3 for cli, Xvfb tools * for desktop). A gate config also gets the determinism checks. + * With `gate`, it checks the config for the gate only: the gate calls no + * agent, so the agent LLMs and gh are left out, and the config needs a run block. */ export async function runChecks( root: string, loaded: DoctorConfig, probes: DoctorProbes = { chromium: chromiumProblem }, + options: DoctorOptions = {}, ): Promise { const { config } = loaded; const checks: DoctorCheck[] = [configCheck(root, loaded, loaded.path !== paths.config(root))]; @@ -68,16 +92,24 @@ export async function runChecks( // With no config there is no platform to check for: `init` writes one. if (!config) return checks; - checks.push(...agentChecks(config, resolve(root, config.app.source))); - if (config.app.platform === 'web' || config.run) { + const browserCheck = (): DoctorCheck => { const problem = probes.chromium(); + return { name: 'browser', ok: !problem, detail: problem ?? 'Chromium for Playwright is installed', fatal: true }; + }; + // The gate starts run.command and drives run.url in Chromium. The app block + // (platform, connect, setup) and the agents are for the agent commands only. + if (options.gate) { + if (config.run) return [...checks, browserCheck(), ...gateChecks(root, config.determinism.image)]; checks.push({ - name: 'browser', - ok: !problem, - detail: problem ?? 'Chromium for Playwright is installed', + name: 'gate', + ok: false, + detail: `${loaded.path} has no run block, so \`bugpatrol run\` cannot start the app. Run \`bugpatrol init --gate\`, or drop --gate to check the agents.`, fatal: true, }); + return checks; } + checks.push(...agentChecks(config, resolve(root, config.app.source))); + if (config.app.platform === 'web' || config.run) checks.push(browserCheck()); const url = await urlCheck(config); if (url) checks.push(url); if (config.app.platform === 'cli') @@ -110,9 +142,13 @@ export async function runChecks( } // The pinned image and the AppModel belong to the deterministic web gate // (`bugpatrol run`). They do not apply to the agents. - if (!config.run) return checks; + if (config.run) checks.push(...gateChecks(root, config.determinism.image)); + return checks; +} - const image = config.determinism.image; +/** The checks of the deterministic web gate: the pinned runner image, the AppModel and the Intent Ledger. */ +function gateChecks(root: string, image: string): DoctorCheck[] { + const checks: DoctorCheck[] = []; const pinned = image.includes('@sha256:'); checks.push({ name: 'pinned-image', diff --git a/packages/cli/src/commands/init.ts b/packages/cli/src/commands/init.ts index 2f7a470..3e13b99 100644 --- a/packages/cli/src/commands/init.ts +++ b/packages/cli/src/commands/init.ts @@ -845,7 +845,7 @@ export async function runInit(root: string, args: string[], log: (line: string) log(`Wrote ${result.configPath}`); if (result.stack.framework) log(`Detected ${result.stack.framework}`); for (const note of result.notes) log(` note: ${note}`); - log('\nNext: review the TODO markers, then run `bugpatrol doctor`.'); + log('\nNext: review the TODO markers, then run `bugpatrol doctor --gate`.'); return; } if (existsSync(paths.config(root))) { diff --git a/packages/cli/src/usage.ts b/packages/cli/src/usage.ts index 4e01cd2..11dd451 100644 --- a/packages/cli/src/usage.ts +++ b/packages/cli/src/usage.ts @@ -9,6 +9,7 @@ Setup bugpatrol init [--yes] [--agent a] detect the app and the LLMs, write .bugpatrol/ bugpatrol init --gate write a starter config for the deterministic web gate bugpatrol doctor check the config, Node, the agents, and what the platform needs + bugpatrol doctor --gate check only what the deterministic web gate needs, not the agents bugpatrol adopt --config move the shared app map and routines into that config's folder Recon @@ -214,12 +215,14 @@ export const COMMAND_HELP: Record = { dashboard: `bugpatrol dashboard [--port N] Serve the dashboard on http://127.0.0.1:4311. It updates live, and it does not stop by itself. `, - doctor: `bugpatrol doctor [--config ] + doctor: `bugpatrol doctor [--gate] [--config ] Check that this machine can run the selected config: the config file, Node 22, the LLM of each agent, and gh when agents.github is on. Then what the platform needs: Chromium for Playwright for web, and that the app URL answers when no setup command starts it (web and api); python3 for cli; Linux and the launch commands for desktop. A config with a run block also gets the gate checks: - the pinned runner image and the AppModel. Exits 2 when a required check fails. + the pinned runner image and the AppModel. --gate checks only what bugpatrol run needs: Chromium and + the gate checks, not the agents, gh or the app block, and fails when there is no run block. + Exits 2 when a required check fails. `, };