diff --git a/CHANGELOG.md b/CHANGELOG.md index 927e6008..748ae4a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ Format based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +### Added +- `npm run eval:mcp`: the same question about a local page with a known answer, asked to an agent with the webperf MCP server, with Chrome DevTools MCP and the webperf skills, and with Chrome DevTools MCP alone, graded on the diagnosis and measured in tokens, turns and time. It runs by hand, not in CI. + ### Changed - CI skips the build and the end-to-end tests on its own when a pull request only changes documentation, instead of waiting for the `skip-e2e` label. Opening a pull request with the label fired two runs and cancelled one, which showed as a failed check. - Examples, CLI commands, the `OWN_DOMAINS` hint of the snippets and the test data use the domains reserved for documentation (`example.com`, `example.net`, `example.org`) instead of real sites. `scripts/example-domains.test.js`, in `npm run test:site`, fails when a real site comes back as an example. diff --git a/package.json b/package.json index 77729b8e..df224ceb 100644 --- a/package.json +++ b/package.json @@ -15,8 +15,9 @@ "check:emoji": "node scripts/check-emoji.js", "test:icons": "node --test lib/rehype-icons.test.js lib/strip-emoji.test.js scripts/check-emoji.test.js", "test:support": "node --test lib/browser-support.test.js lib/remark-browser-support.test.js", - "test:prompt": "node --test lib/snippet-prompt.test.mjs lib/remark-snippet-prompt.test.mjs scripts/eval-prompts/grade.test.mjs scripts/eval-prompts/summary.test.mjs", + "test:prompt": "node --test lib/snippet-prompt.test.mjs lib/remark-snippet-prompt.test.mjs scripts/eval-prompts/grade.test.mjs scripts/eval-prompts/summary.test.mjs scripts/eval-mcp/grade.test.mjs scripts/eval-mcp/summary.test.mjs", "eval:prompts": "node scripts/eval-prompts/run.mjs", + "eval:mcp": "node scripts/eval-mcp/run.mjs", "extension:build": "node scripts/build-extension.mjs", "extension:zip": "node scripts/package-extension.mjs", "test:extension": "node --test extension/lib/*.test.js lib/snippet-expression.test.mjs", diff --git a/scripts/eval-mcp/arms.mjs b/scripts/eval-mcp/arms.mjs new file mode 100644 index 00000000..3af8b2ff --- /dev/null +++ b/scripts/eval-mcp/arms.mjs @@ -0,0 +1,33 @@ +// The setups the MCP evals compare. Each one is an agent with the same model and prompt and no settings, +// skills or MCP servers of the machine, plus: +// webperf the MCP server of this repository (node cli/src/bin.js mcp) +// devtools-skill Chrome DevTools MCP and the webperf skills, the path the README describes +// devtools Chrome DevTools MCP alone, where the agent writes its own measurement code + +export const DEVTOOLS_MCP = 'chrome-devtools-mcp@1.11.0' + +export function arms({ root, pluginDir }) { + const devtools = { 'chrome-devtools': { command: 'npx', args: ['-y', DEVTOOLS_MCP, '--headless', '--isolated'] } } + return [ + { + id: 'webperf', + mcpServers: { webperf: { command: 'node', args: [`${root}/cli/src/bin.js`, 'mcp'] } }, + tools: '', + allowedTools: ['mcp__webperf'], + }, + { + id: 'devtools-skill', + mcpServers: devtools, + // Read for the scripts of the skills, Skill to load them + tools: 'Read,Skill', + allowedTools: ['mcp__chrome-devtools', 'Read', 'Skill'], + pluginDir, + }, + { + id: 'devtools', + mcpServers: devtools, + tools: '', + allowedTools: ['mcp__chrome-devtools'], + }, + ] +} diff --git a/scripts/eval-mcp/cases.mjs b/scripts/eval-mcp/cases.mjs new file mode 100644 index 00000000..eb183d8d --- /dev/null +++ b/scripts/eval-mcp/cases.mjs @@ -0,0 +1,34 @@ +// The tasks of the MCP evals. The prompts read like a person asking about a page: they name no tool and +// no snippet, so the evals measure what the agent does with each setup, not whether it follows a recipe. +// Each fixture has a known answer that the report has to give. + +export const CASES = [ + { + id: 'lcp-slow-image', + fixture: 'lcp-slow-image.html', + prompt: (url) => + `The Largest Contentful Paint of this page is slow: ${url}\n\nFind out why. Tell me which element is the LCP, which part of its loading takes the most time, and what to fix first.`, + expect: { + diagnosis: [ + // The product photo is the LCP element + { id: 'lcp-element', pattern: /product\.png|product photo| + `Users say this page feels slow when they click on it: ${url}\n\nFind which interaction is slow, what makes it slow, and what to fix.`, + expect: { + diagnosis: [ + // The "Add to cart" button, not the wishlist + { id: 'slow-target', pattern: /#add\b|add to cart/i }, + // Its click handler blocks the main thread: processing time + { id: 'slow-phase', pattern: /processing|event handler|click handler|\bhandler\b|long task|main thread|blocking/i }, + ], + }, + }, +] diff --git a/scripts/eval-mcp/fixtures/inp-slow-click.html b/scripts/eval-mcp/fixtures/inp-slow-click.html new file mode 100644 index 00000000..c34e83ee --- /dev/null +++ b/scripts/eval-mcp/fixtures/inp-slow-click.html @@ -0,0 +1,30 @@ + + + + + + Product page + + + +
+

Trail running shoes

+ + +

+
+ + + diff --git a/scripts/eval-mcp/fixtures/lcp-slow-image.html b/scripts/eval-mcp/fixtures/lcp-slow-image.html new file mode 100644 index 00000000..9b33cb15 --- /dev/null +++ b/scripts/eval-mcp/fixtures/lcp-slow-image.html @@ -0,0 +1,17 @@ + + + + + + Product page + + + +
+

Trail running shoes

+ + Product photo +

Light, breathable and made for long distances.

+
+ + diff --git a/scripts/eval-mcp/fixtures/product.png b/scripts/eval-mcp/fixtures/product.png new file mode 100644 index 00000000..0ff2d6e7 Binary files /dev/null and b/scripts/eval-mcp/fixtures/product.png differ diff --git a/scripts/eval-mcp/grade.mjs b/scripts/eval-mcp/grade.mjs new file mode 100644 index 00000000..e7d811f0 --- /dev/null +++ b/scripts/eval-mcp/grade.mjs @@ -0,0 +1,65 @@ +// Grades a run of the MCP evals from its transcript (parseTranscript of eval-prompts): whether the +// report gets the diagnosis right, and how the agent measured. A measurement is a published snippet run +// through the webperf MCP server, a published script given to evaluate_script of Chrome DevTools MCP +// (as code or by path), code the agent wrote itself, or another tool such as a performance trace. + +export { parseTranscript } from '../eval-prompts/grade.mjs' + +const WEBPERF = 'mcp__webperf__' +const DEVTOOLS = 'mcp__chrome-devtools__' +// The tools of the webperf server that run snippets, and the argument that names what they run +const WEBPERF_RUNS = { run_snippet: 'snippet', run_workflow: 'workflow', open_page: 'track' } +// Tools that only move around the page, which do not measure anything by themselves +const NAVIGATION = new Set(['navigate_page', 'new_page', 'list_pages', 'select_page', 'close_page', 'take_snapshot', 'take_screenshot', 'wait_for', 'resize_page']) + +const nameOf = (catalog, name) => catalog.find((path) => path.split('/')[1] === name)?.split('/')[1] + +// The published snippet an evaluate_script call runs, from the script:"Name" of its result object or +// the scripts/.js path of a skill script +function publishedScript(input, catalog) { + if (typeof input.function === 'string') { + const name = input.function.match(/\bscript:\s*["']([\w-]+)["']/)?.[1] + const found = name && nameOf(catalog, name) + return found ? { snippet: found, sent: 'code', bytes: input.function.length } : null + } + if (typeof input.sourcePath === 'string') { + const found = nameOf(catalog, input.sourcePath.match(/\/scripts\/([\w-]+)\.js$/)?.[1]) + return found ? { snippet: found, sent: 'path', bytes: 0 } : null + } + return null +} + +export function snippetRuns(transcript, catalog) { + const runs = [] + for (const { name, input } of transcript.calls) { + if (name.startsWith(WEBPERF)) { + const tool = name.slice(WEBPERF.length) + const arg = WEBPERF_RUNS[tool] + if (!arg) continue + for (const snippet of [input[arg]].flat()) runs.push({ via: tool, snippet }) + } else if (name === `${DEVTOOLS}evaluate_script`) { + const published = publishedScript(input, catalog) + if (published) runs.push({ via: 'evaluate_script', ...published }) + } + } + return runs +} + +export function gradeRun(evalCase, transcript, catalog) { + const checks = evalCase.expect.diagnosis.map(({ id, pattern }) => ({ id, pass: pattern.test(transcript.report) })) + + const published = snippetRuns(transcript, catalog).length + const evaluations = transcript.calls.filter((c) => c.name === `${DEVTOOLS}evaluate_script`).length + const devtoolsPublished = snippetRuns(transcript, catalog).filter((r) => r.via === 'evaluate_script').length + const ownCode = evaluations - devtoolsPublished + const otherTools = transcript.calls + .filter((c) => c.name.startsWith(DEVTOOLS) && c.name !== `${DEVTOOLS}evaluate_script`) + .map((c) => c.name.slice(DEVTOOLS.length)) + const measuring = otherTools.filter((tool) => !NAVIGATION.has(tool)) + checks.push({ + id: 'measured', + pass: published + ownCode + measuring.length > 0, + detail: { published, ownCode, otherTools: [...new Set(otherTools)] }, + }) + return checks +} diff --git a/scripts/eval-mcp/grade.test.mjs b/scripts/eval-mcp/grade.test.mjs new file mode 100644 index 00000000..5345475b --- /dev/null +++ b/scripts/eval-mcp/grade.test.mjs @@ -0,0 +1,65 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { snippetRuns, gradeRun } from './grade.mjs' + +const CATALOG = ['CoreWebVitals/LCP', 'CoreWebVitals/LCP-Subparts', 'CoreWebVitals/INP', 'Interaction/Interactions'] + +const transcript = (calls, report = '') => ({ calls, report, tokens: { input: 1000, output: 100 }, durationMs: 1000, turns: 3 }) +const call = (name, input = {}, output = '') => ({ name, input, output }) +const failedDiagnosis = (checks) => checks.filter((c) => c.id !== 'measured' && !c.pass).map((c) => c.id).sort() + +const LCP_CASE = { + id: 'lcp-slow-image', + expect: { diagnosis: [{ id: 'lcp-element', pattern: /hero\.png| { + const runs = snippetRuns(transcript([call('mcp__webperf__run_snippet', { snippet: 'LCP-Subparts' }), call('mcp__webperf__run_workflow', { workflow: 'core-web-vitals' }), call('mcp__webperf__list_snippets')]), CATALOG) + assert.deepEqual(runs, [ + { via: 'run_snippet', snippet: 'LCP-Subparts' }, + { via: 'run_workflow', snippet: 'core-web-vitals' }, + ]) +}) + +test('snippetRuns recognizes a published script sent as code to evaluate_script', () => { + const code = '// snippets/CoreWebVitals/LCP-Subparts.js | sha256:abc\n(async()=>{return{script:"LCP-Subparts",status:"ok"}})()' + const runs = snippetRuns(transcript([call('mcp__chrome-devtools__evaluate_script', { function: code })]), CATALOG) + assert.deepEqual(runs, [{ via: 'evaluate_script', snippet: 'LCP-Subparts', sent: 'code', bytes: code.length }]) +}) + +test('snippetRuns recognizes a published script passed by path to evaluate_script', () => { + const runs = snippetRuns(transcript([call('mcp__chrome-devtools__evaluate_script', { sourcePath: '/x/skills/webperf-core-web-vitals/scripts/LCP.js' })]), CATALOG) + assert.deepEqual(runs, [{ via: 'evaluate_script', snippet: 'LCP', sent: 'path', bytes: 0 }]) +}) + +test('snippetRuns leaves out code the agent wrote itself', () => { + const runs = snippetRuns(transcript([call('mcp__chrome-devtools__evaluate_script', { function: '() => performance.getEntriesByType("navigation")' })]), CATALOG) + assert.deepEqual(runs, []) +}) + +test('gradeRun passes the diagnosis when the report names the element and the slow phase', () => { + const checks = gradeRun(LCP_CASE, transcript([], 'The LCP element is the hero.png. Most of the time is resource load time (2.1 s).'), CATALOG) + assert.deepEqual(failedDiagnosis(checks), []) +}) + +test('gradeRun fails each part of the diagnosis the report misses', () => { + const checks = gradeRun(LCP_CASE, transcript([], 'The page has a slow server response.'), CATALOG) + assert.deepEqual(failedDiagnosis(checks), ['lcp-element', 'slow-phase']) +}) + +test('gradeRun reports how the agent measured: published snippets, its own code and the other tools', () => { + const t = transcript([ + call('mcp__chrome-devtools__navigate_page', { url: 'http://x' }), + call('mcp__chrome-devtools__evaluate_script', { function: '() => 1' }), + call('mcp__chrome-devtools__evaluate_script', { sourcePath: '/s/scripts/LCP.js' }), + call('mcp__chrome-devtools__performance_start_trace', {}), + ]) + const { measured } = Object.fromEntries(gradeRun(LCP_CASE, t, CATALOG).map((c) => [c.id, c])) + assert.deepEqual(measured.detail, { published: 1, ownCode: 1, otherTools: ['navigate_page', 'performance_start_trace'] }) + assert.equal(measured.pass, true) +}) + +test('gradeRun fails the measured check when the agent measured nothing', () => { + const { measured } = Object.fromEntries(gradeRun(LCP_CASE, transcript([], 'LCP is probably the image.'), CATALOG).map((c) => [c.id, c])) + assert.equal(measured.pass, false) +}) diff --git a/scripts/eval-mcp/run.mjs b/scripts/eval-mcp/run.mjs new file mode 100644 index 00000000..f5307edc --- /dev/null +++ b/scripts/eval-mcp/run.mjs @@ -0,0 +1,158 @@ +#!/usr/bin/env node +import { createServer } from 'node:http' +import { spawn } from 'node:child_process' +import { cpSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, extname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { parseArgs } from 'node:util' +import { CASES } from './cases.mjs' +import { arms as allArms, DEVTOOLS_MCP } from './arms.mjs' +import { parseTranscript, gradeRun, snippetRuns } from './grade.mjs' +import { summarize, markdownSummary } from './summary.mjs' + +// Evals of the MCP server: the same question about a local page with a known answer, asked to an agent +// with the webperf MCP server, with Chrome DevTools MCP and the webperf skills, and with Chrome DevTools +// MCP alone. Each run is graded on the diagnosis and measured in tokens, turns and time; transcripts and +// a summary go to workspace/evals-mcp/. Not part of CI: a full run takes a while and uses tokens. +// +// npm run eval:mcp +// npm run eval:mcp -- --arm webperf --case lcp-slow-image --repeat 1 + +const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../..') +const FIXTURE_DIRS = [join(ROOT, 'scripts/eval-mcp/fixtures'), join(ROOT, 'cli/tests/fixtures')] +const TYPES = { '.html': 'text/html', '.png': 'image/png' } +const CATALOG = readdirSync(join(ROOT, 'snippets'), { withFileTypes: true }) + .filter((d) => d.isDirectory() && !d.name.startsWith('_')) + .flatMap((d) => readdirSync(join(ROOT, 'snippets', d.name)).filter((f) => f.endsWith('.js')).map((f) => `${d.name}/${f.slice(0, -3)}`)) + +const { values: opts } = parseArgs({ + options: { + arm: { type: 'string', multiple: true }, + case: { type: 'string', multiple: true }, + repeat: { type: 'string', default: '3' }, + model: { type: 'string', default: 'sonnet' }, + concurrency: { type: 'string', default: '2' }, + 'max-cost': { type: 'string', default: '2' }, + }, +}) + +function run(command, args, { cwd, input } = {}) { + return new Promise((done, fail) => { + const child = spawn(command, args, { cwd, stdio: ['pipe', 'pipe', 'pipe'] }) + let stdout = '' + child.stdout.on('data', (d) => (stdout += d)) + child.stderr.on('data', () => {}) + child.on('error', fail) + child.on('close', (code) => done({ code, stdout })) + child.stdin.end(input ?? '') + }) +} + +// Serves the fixtures; `/slow//` serves a file after a delay +function startServer() { + const server = createServer((req, res) => { + const path = decodeURIComponent(new URL(req.url, 'http://x').pathname).replace(/^\/+/, '') + const [, delay = 0, name = path] = path.match(/^slow\/(\d+)\/(.+)$/) ?? [] + const file = FIXTURE_DIRS.map((dir) => join(dir, name)).find((f) => !name.includes('..') && existsSync(f)) + if (!file || !name) return res.writeHead(404).end() + setTimeout(() => res.writeHead(200, { 'content-type': TYPES[extname(file)] ?? 'application/octet-stream' }).end(readFileSync(file)), Number(delay)) + }) + return new Promise((done) => server.listen(0, '127.0.0.1', () => done(server))) +} + +// The generated skills, as a plugin that loads only for the devtools-skill setup +function buildPlugin(work) { + const dir = join(work, 'webperf-plugin') + mkdirSync(join(dir, '.claude-plugin'), { recursive: true }) + writeFileSync(join(dir, '.claude-plugin/plugin.json'), JSON.stringify({ name: 'webperf-skills', version: '0.0.0', description: 'WebPerf Snippets skills for the MCP evals' })) + cpSync(join(ROOT, '.claude/skills'), join(dir, 'skills'), { recursive: true }) + return dir +} + +// No settings, skills or MCP servers of the machine +const ISOLATED = ['--setting-sources', '', '--strict-mcp-config', '--no-session-persistence'] + +async function runAgent(arm, prompt, workdir) { + const config = join(workdir, '.mcp-eval.json') + writeFileSync(config, JSON.stringify({ mcpServers: arm.mcpServers })) + const args = [ + '-p', '--output-format', 'stream-json', '--verbose', + '--model', opts.model, + '--tools', arm.tools, '--allowedTools', arm.allowedTools.join(','), + '--mcp-config', config, + ...(arm.pluginDir ? ['--plugin-dir', arm.pluginDir] : []), + ...ISOLATED, + '--max-budget-usd', opts['max-cost'], + ] + const { stdout } = await run('claude', args, { cwd: workdir, input: prompt }) + return stdout +} + +async function pool(tasks, size) { + const results = [] + let next = 0 + const worker = async () => { + while (next < tasks.length) { + const i = next++ + results[i] = await tasks[i]() + } + } + await Promise.all(Array.from({ length: size }, worker)) + return results +} + +const thousands = (n) => `${(n / 1000).toFixed(1)}k` + +async function main() { + const work = mkdtempSync(join(tmpdir(), 'eval-mcp-')) + const arms = allArms({ root: ROOT, pluginDir: buildPlugin(work) }).filter((a) => !opts.arm || opts.arm.includes(a.id)) + const cases = CASES.filter((c) => !opts.case || opts.case.includes(c.id)) + if (!arms.length || !cases.length) throw new Error('No arm or case matches the filters') + const repeat = Number(opts.repeat) + const out = join(ROOT, 'workspace/evals-mcp', new Date().toISOString().replace(/[:.]/g, '-')) + mkdirSync(out, { recursive: true }) + console.log(`Arms: ${arms.map((a) => a.id).join(', ')}\nModel: ${opts.model}, ${repeat} runs each, ${DEVTOOLS_MCP}\nOutput: ${out}\n`) + + const server = await startServer() + const base = `http://127.0.0.1:${server.address().port}` + try { + const tasks = [] + for (const evalCase of cases) { + const prompt = evalCase.prompt(`${base}/${evalCase.fixture}`) + writeFileSync(join(out, `${evalCase.id}.prompt.md`), prompt) + for (const arm of arms) { + for (let i = 1; i <= repeat; i++) { + tasks.push(async () => { + const workdir = mkdtempSync(join(work, `${arm.id}-${evalCase.id}-${i}-`)) + const raw = await runAgent(arm, prompt, workdir) + writeFileSync(join(out, `${arm.id}.${evalCase.id}.${i}.jsonl`), raw) + const transcript = parseTranscript(raw) + const checks = gradeRun(evalCase, transcript, CATALOG) + const runs = snippetRuns(transcript, CATALOG) + const failed = checks.filter((c) => !c.pass).map((c) => c.id) + const { input, output } = transcript.tokens + console.log(`${arm.id} ${evalCase.id} #${i}: ${failed.length ? `FAIL ${failed.join(', ')}` : 'pass'} (${thousands(input)} in, ${thousands(output)} out, ${transcript.turns} turns, ${Math.round(transcript.durationMs / 1000)} s)`) + return { + arm: arm.id, case: evalCase.id, run: i, checks, tokens: transcript.tokens, durationMs: transcript.durationMs, turns: transcript.turns, + tools: transcript.calls.map((c) => c.name), snippetRuns: runs, report: transcript.report, + } + }) + } + } + } + const results = await pool(tasks, Number(opts.concurrency)) + writeFileSync(join(out, 'summary.json'), JSON.stringify({ model: opts.model, repeat, devtools: DEVTOOLS_MCP, runs: results }, null, 2)) + const markdown = markdownSummary(summarize(results), { model: opts.model, repeat, devtools: DEVTOOLS_MCP }) + writeFileSync(join(out, 'summary.md'), markdown) + console.log(`\n${markdown}`) + } finally { + server.close() + rmSync(work, { recursive: true, force: true }) + } +} + +main().catch((error) => { + console.error(error.message) + process.exit(1) +}) diff --git a/scripts/eval-mcp/summary.mjs b/scripts/eval-mcp/summary.mjs new file mode 100644 index 00000000..430cb6bb --- /dev/null +++ b/scripts/eval-mcp/summary.mjs @@ -0,0 +1,48 @@ +// The summary of a run of the MCP evals: per case and setup, how many runs got the diagnosis right, the +// medians of tokens, turns and time, and how the agent measured. + +const median = (values) => { + const sorted = [...values].sort((a, b) => a - b) + const mid = Math.floor(sorted.length / 2) + return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2 +} + +export function summarize(runs) { + const groups = new Map() + for (const run of runs) { + const key = `${run.case}\u0000${run.arm}` + if (!groups.has(key)) groups.set(key, []) + groups.get(key).push(run) + } + return [...groups.values()].map((group) => { + const measured = (run) => run.checks.find((c) => c.id === 'measured') + return { + case: group[0].case, + arm: group[0].arm, + runs: group.length, + diagnosis: group.filter((run) => run.checks.filter((c) => c.id !== 'measured').every((c) => c.pass)).length, + medianInput: median(group.map((run) => run.tokens.input)), + medianOutput: median(group.map((run) => run.tokens.output)), + medianTurns: median(group.map((run) => run.turns)), + medianSeconds: Math.round(median(group.map((run) => run.durationMs)) / 1000), + published: group.filter((run) => run.snippetRuns.length > 0).length, + ownCode: group.reduce((sum, run) => sum + (measured(run)?.detail.ownCode ?? 0), 0), + sentCodeBytes: group.reduce((sum, run) => sum + run.snippetRuns.reduce((s, r) => s + (r.bytes ?? 0), 0), 0), + } + }) +} + +const k = (n) => `${(n / 1000).toFixed(1)}k` + +export function markdownSummary(rows, { model, repeat, devtools }) { + const lines = [ + `Model: ${model}, ${repeat} runs per case and setup, ${devtools}.`, + '', + '| Case | Setup | Diagnosis | Input tokens | Output tokens | Turns | Time | Runs with published snippets | Own code calls | Script bytes sent |', + '| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |', + ...rows.map((r) => `| ${r.case} | ${r.arm} | ${r.diagnosis}/${r.runs} | ${k(r.medianInput)} | ${k(r.medianOutput)} | ${r.medianTurns} | ${r.medianSeconds} s | ${r.published}/${r.runs} | ${r.ownCode} | ${r.sentCodeBytes} |`), + '', + 'Tokens, turns and time are medians. Input tokens include the cache reads and writes.', + ] + return `${lines.join('\n')}\n` +} diff --git a/scripts/eval-mcp/summary.test.mjs b/scripts/eval-mcp/summary.test.mjs new file mode 100644 index 00000000..12e983b0 --- /dev/null +++ b/scripts/eval-mcp/summary.test.mjs @@ -0,0 +1,39 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { summarize, markdownSummary } from './summary.mjs' + +const runOf = (arm, n, { pass = true, input = 1000, output = 100, turns = 5, durationMs = 10000, snippetRuns = [], ownCode = 0 } = {}) => ({ + arm, case: 'lcp', run: n, + checks: [{ id: 'lcp-element', pass }, { id: 'slow-phase', pass: true }, { id: 'measured', pass: true, detail: { published: snippetRuns.length, ownCode, otherTools: [] } }], + tokens: { input, output }, turns, durationMs, snippetRuns, +}) + +test('summarize groups by case and arm, counts correct diagnoses and takes the medians', () => { + const [row] = summarize([ + runOf('webperf', 1, { input: 1000 }), + runOf('webperf', 2, { input: 3000, pass: false }), + runOf('webperf', 3, { input: 2000 }), + ]) + assert.deepEqual(row, { + case: 'lcp', arm: 'webperf', runs: 3, diagnosis: 2, + medianInput: 2000, medianOutput: 100, medianTurns: 5, medianSeconds: 10, + published: 0, ownCode: 0, sentCodeBytes: 0, + }) +}) + +test('summarize counts the runs with published snippets, the own code and the script bytes sent', () => { + const [row] = summarize([ + runOf('devtools-skill', 1, { snippetRuns: [{ via: 'evaluate_script', sent: 'code', bytes: 9000 }], ownCode: 1 }), + runOf('devtools-skill', 2, { snippetRuns: [{ via: 'evaluate_script', sent: 'path', bytes: 0 }] }), + ]) + assert.equal(row.published, 2) + assert.equal(row.ownCode, 1) + assert.equal(row.sentCodeBytes, 9000) +}) + +test('markdownSummary writes one row per case and arm', () => { + const md = markdownSummary(summarize([runOf('webperf', 1), runOf('devtools', 1)]), { model: 'sonnet', repeat: 1, devtools: 'chrome-devtools-mcp@1.11.0' }) + assert.match(md, /\| lcp \| webperf \| 1\/1 \|/) + assert.match(md, /\| lcp \| devtools \| 1\/1 \|/) + assert.match(md, /chrome-devtools-mcp@1\.11\.0/) +})