feat(eval): measure the MCP server against Chrome DevTools MCP (chapter 6)

CHANGELOG.md

@@ -8,6 +8,9 @@ Format based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## [Unreleased]
 
+### Added
+- `npm run eval:mcp`: the same question about a local page with a known answer, asked to an agent with the webperf MCP server, with Chrome DevTools MCP and the webperf skills, and with Chrome DevTools MCP alone, graded on the diagnosis and measured in tokens, turns and time. It runs by hand, not in CI.
+
 ### Changed
 - CI skips the build and the end-to-end tests on its own when a pull request only changes documentation, instead of waiting for the `skip-e2e` label. Opening a pull request with the label fired two runs and cancelled one, which showed as a failed check.
 - Examples, CLI commands, the `OWN_DOMAINS` hint of the snippets and the test data use the domains reserved for documentation (`example.com`, `example.net`, `example.org`) instead of real sites. `scripts/example-domains.test.js`, in `npm run test:site`, fails when a real site comes back as an example.

package.json

@@ -15,8 +15,9 @@
     "check:emoji": "node scripts/check-emoji.js",
     "test:icons": "node --test lib/rehype-icons.test.js lib/strip-emoji.test.js scripts/check-emoji.test.js",
     "test:support": "node --test lib/browser-support.test.js lib/remark-browser-support.test.js",
-    "test:prompt": "node --test lib/snippet-prompt.test.mjs lib/remark-snippet-prompt.test.mjs scripts/eval-prompts/grade.test.mjs scripts/eval-prompts/summary.test.mjs",
+    "test:prompt": "node --test lib/snippet-prompt.test.mjs lib/remark-snippet-prompt.test.mjs scripts/eval-prompts/grade.test.mjs scripts/eval-prompts/summary.test.mjs scripts/eval-mcp/grade.test.mjs scripts/eval-mcp/summary.test.mjs",
     "eval:prompts": "node scripts/eval-prompts/run.mjs",
+    "eval:mcp": "node scripts/eval-mcp/run.mjs",
     "extension:build": "node scripts/build-extension.mjs",
     "extension:zip": "node scripts/package-extension.mjs",
     "test:extension": "node --test extension/lib/*.test.js lib/snippet-expression.test.mjs",

scripts/eval-mcp/arms.mjs

@@ -0,0 +1,33 @@
+// The setups the MCP evals compare. Each one is an agent with the same model and prompt and no settings,
+// skills or MCP servers of the machine, plus:
+//   webperf         the MCP server of this repository (node cli/src/bin.js mcp)
+//   devtools-skill  Chrome DevTools MCP and the webperf skills, the path the README describes
+//   devtools        Chrome DevTools MCP alone, where the agent writes its own measurement code
+
+export const DEVTOOLS_MCP = 'chrome-devtools-mcp@1.11.0'
+
+export function arms({ root, pluginDir }) {
+  const devtools = { 'chrome-devtools': { command: 'npx', args: ['-y', DEVTOOLS_MCP, '--headless', '--isolated'] } }
+  return [
+    {
+      id: 'webperf',
+      mcpServers: { webperf: { command: 'node', args: [`${root}/cli/src/bin.js`, 'mcp'] } },
+      tools: '',
+      allowedTools: ['mcp__webperf'],
+    },
+    {
+      id: 'devtools-skill',
+      mcpServers: devtools,
+      // Read for the scripts of the skills, Skill to load them
+      tools: 'Read,Skill',
+      allowedTools: ['mcp__chrome-devtools', 'Read', 'Skill'],
+      pluginDir,
+    },
+    {
+      id: 'devtools',
+      mcpServers: devtools,
+      tools: '',
+      allowedTools: ['mcp__chrome-devtools'],
+    },
+  ]
+}

scripts/eval-mcp/cases.mjs

@@ -0,0 +1,34 @@
+// The tasks of the MCP evals. The prompts read like a person asking about a page: they name no tool and
+// no snippet, so the evals measure what the agent does with each setup, not whether it follows a recipe.
+// Each fixture has a known answer that the report has to give.
+
+export const CASES = [
+  {
+    id: 'lcp-slow-image',
+    fixture: 'lcp-slow-image.html',
+    prompt: (url) =>
+      `The Largest Contentful Paint of this page is slow: ${url}\n\nFind out why. Tell me which element is the LCP, which part of its loading takes the most time, and what to fix first.`,
+    expect: {
+      diagnosis: [
+        // The product photo is the LCP element
+        { id: 'lcp-element', pattern: /product\.png|product photo|<img|\bimg\b|\bimage\b/i },
+        // Its 2 s are spent downloading it, not on the server response or the render
+        { id: 'slow-phase', pattern: /resource load|load (time|duration)|download|loading the image|image (request|response)/i },
+      ],
+    },
+  },
+  {
+    id: 'inp-slow-click',
+    fixture: 'inp-slow-click.html',
+    prompt: (url) =>
+      `Users say this page feels slow when they click on it: ${url}\n\nFind which interaction is slow, what makes it slow, and what to fix.`,
+    expect: {
+      diagnosis: [
+        // The "Add to cart" button, not the wishlist
+        { id: 'slow-target', pattern: /#add\b|add to cart/i },
+        // Its click handler blocks the main thread: processing time
+        { id: 'slow-phase', pattern: /processing|event handler|click handler|\bhandler\b|long task|main thread|blocking/i },
+      ],
+    },
+  },
+]

scripts/eval-mcp/fixtures/inp-slow-click.html

@@ -0,0 +1,30 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>Product page</title>
+  <style>body { margin: 0; font-family: sans-serif; } main { padding: 16px; } button { display: block; width: 200px; height: 48px; margin: 8px 0; }</style>
+</head>
+<body>
+  <main>
+    <h1>Trail running shoes</h1>
+    <button id="add" type="button">Add to cart</button>
+    <button id="wish" type="button">Wishlist</button>
+    <p id="status" aria-live="polite"></p>
+  </main>
+  <script>
+    // Adding to the cart blocks the main thread for 250 ms; the wishlist takes 30 ms
+    document.getElementById("add").addEventListener("click", function addToCart() {
+      const end = performance.now() + 250;
+      while (performance.now() < end) {}
+      document.getElementById("status").textContent = "Added to the cart";
+    });
+    document.getElementById("wish").addEventListener("click", function addToWishlist() {
+      const end = performance.now() + 30;
+      while (performance.now() < end) {}
+      document.getElementById("status").textContent = "Saved to the wishlist";
+    });
+  </script>
+</body>
+</html>

scripts/eval-mcp/fixtures/lcp-slow-image.html

@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>Product page</title>
+  <style>body { margin: 0; font-family: sans-serif; } main { padding: 16px; } img { display: block; max-width: 100%; height: auto; }</style>
+</head>
+<body>
+  <main>
+    <h1>Trail running shoes</h1>
+    <!-- The product photo is the largest element, and the server takes 2 s to send it -->
+    <img src="/slow/2000/product.png" width="600" height="400" alt="Product photo">
+    <p>Light, breathable and made for long distances.</p>
+  </main>
+</body>
+</html>

scripts/eval-mcp/grade.mjs

@@ -0,0 +1,65 @@
+// Grades a run of the MCP evals from its transcript (parseTranscript of eval-prompts): whether the
+// report gets the diagnosis right, and how the agent measured. A measurement is a published snippet run
+// through the webperf MCP server, a published script given to evaluate_script of Chrome DevTools MCP
+// (as code or by path), code the agent wrote itself, or another tool such as a performance trace.
+
+export { parseTranscript } from '../eval-prompts/grade.mjs'
+
+const WEBPERF = 'mcp__webperf__'
+const DEVTOOLS = 'mcp__chrome-devtools__'
+// The tools of the webperf server that run snippets, and the argument that names what they run
+const WEBPERF_RUNS = { run_snippet: 'snippet', run_workflow: 'workflow', open_page: 'track' }
+// Tools that only move around the page, which do not measure anything by themselves
+const NAVIGATION = new Set(['navigate_page', 'new_page', 'list_pages', 'select_page', 'close_page', 'take_snapshot', 'take_screenshot', 'wait_for', 'resize_page'])
+
+const nameOf = (catalog, name) => catalog.find((path) => path.split('/')[1] === name)?.split('/')[1]
+
+// The published snippet an evaluate_script call runs, from the script:"Name" of its result object or
+// the scripts/<Name>.js path of a skill script
+function publishedScript(input, catalog) {
+  if (typeof input.function === 'string') {
+    const name = input.function.match(/\bscript:\s*["']([\w-]+)["']/)?.[1]
+    const found = name && nameOf(catalog, name)
+    return found ? { snippet: found, sent: 'code', bytes: input.function.length } : null
+  }
+  if (typeof input.sourcePath === 'string') {
+    const found = nameOf(catalog, input.sourcePath.match(/\/scripts\/([\w-]+)\.js$/)?.[1])
+    return found ? { snippet: found, sent: 'path', bytes: 0 } : null
+  }
+  return null
+}
+
+export function snippetRuns(transcript, catalog) {
+  const runs = []
+  for (const { name, input } of transcript.calls) {
+    if (name.startsWith(WEBPERF)) {
+      const tool = name.slice(WEBPERF.length)
+      const arg = WEBPERF_RUNS[tool]
+      if (!arg) continue
+      for (const snippet of [input[arg]].flat()) runs.push({ via: tool, snippet })
+    } else if (name === `${DEVTOOLS}evaluate_script`) {
+      const published = publishedScript(input, catalog)
+      if (published) runs.push({ via: 'evaluate_script', ...published })
+    }
+  }
+  return runs
+}
+
+export function gradeRun(evalCase, transcript, catalog) {
+  const checks = evalCase.expect.diagnosis.map(({ id, pattern }) => ({ id, pass: pattern.test(transcript.report) }))
+
+  const published = snippetRuns(transcript, catalog).length
+  const evaluations = transcript.calls.filter((c) => c.name === `${DEVTOOLS}evaluate_script`).length
+  const devtoolsPublished = snippetRuns(transcript, catalog).filter((r) => r.via === 'evaluate_script').length
+  const ownCode = evaluations - devtoolsPublished
+  const otherTools = transcript.calls
+    .filter((c) => c.name.startsWith(DEVTOOLS) && c.name !== `${DEVTOOLS}evaluate_script`)
+    .map((c) => c.name.slice(DEVTOOLS.length))
+  const measuring = otherTools.filter((tool) => !NAVIGATION.has(tool))
+  checks.push({
+    id: 'measured',
+    pass: published + ownCode + measuring.length > 0,
+    detail: { published, ownCode, otherTools: [...new Set(otherTools)] },
+  })
+  return checks
+}

scripts/eval-mcp/grade.test.mjs

@@ -0,0 +1,65 @@
+import { test } from 'node:test'
+import assert from 'node:assert/strict'
+import { snippetRuns, gradeRun } from './grade.mjs'
+
+const CATALOG = ['CoreWebVitals/LCP', 'CoreWebVitals/LCP-Subparts', 'CoreWebVitals/INP', 'Interaction/Interactions']
+
+const transcript = (calls, report = '') => ({ calls, report, tokens: { input: 1000, output: 100 }, durationMs: 1000, turns: 3 })
+const call = (name, input = {}, output = '') => ({ name, input, output })
+const failedDiagnosis = (checks) => checks.filter((c) => c.id !== 'measured' && !c.pass).map((c) => c.id).sort()
+
+const LCP_CASE = {
+  id: 'lcp-slow-image',
+  expect: { diagnosis: [{ id: 'lcp-element', pattern: /hero\.png|<img|\bimage\b/i }, { id: 'slow-phase', pattern: /resource load|load (time|duration)/i }] },
+}
+
+test('snippetRuns finds the snippets run through the webperf MCP server', () => {
+  const runs = snippetRuns(transcript([call('mcp__webperf__run_snippet', { snippet: 'LCP-Subparts' }), call('mcp__webperf__run_workflow', { workflow: 'core-web-vitals' }), call('mcp__webperf__list_snippets')]), CATALOG)
+  assert.deepEqual(runs, [
+    { via: 'run_snippet', snippet: 'LCP-Subparts' },
+    { via: 'run_workflow', snippet: 'core-web-vitals' },
+  ])
+})
+
+test('snippetRuns recognizes a published script sent as code to evaluate_script', () => {
+  const code = '// snippets/CoreWebVitals/LCP-Subparts.js | sha256:abc\n(async()=>{return{script:"LCP-Subparts",status:"ok"}})()'
+  const runs = snippetRuns(transcript([call('mcp__chrome-devtools__evaluate_script', { function: code })]), CATALOG)
+  assert.deepEqual(runs, [{ via: 'evaluate_script', snippet: 'LCP-Subparts', sent: 'code', bytes: code.length }])
+})
+
+test('snippetRuns recognizes a published script passed by path to evaluate_script', () => {
+  const runs = snippetRuns(transcript([call('mcp__chrome-devtools__evaluate_script', { sourcePath: '/x/skills/webperf-core-web-vitals/scripts/LCP.js' })]), CATALOG)
+  assert.deepEqual(runs, [{ via: 'evaluate_script', snippet: 'LCP', sent: 'path', bytes: 0 }])
+})
+
+test('snippetRuns leaves out code the agent wrote itself', () => {
+  const runs = snippetRuns(transcript([call('mcp__chrome-devtools__evaluate_script', { function: '() => performance.getEntriesByType("navigation")' })]), CATALOG)
+  assert.deepEqual(runs, [])
+})
+
+test('gradeRun passes the diagnosis when the report names the element and the slow phase', () => {
+  const checks = gradeRun(LCP_CASE, transcript([], 'The LCP element is the <img> hero.png. Most of the time is resource load time (2.1 s).'), CATALOG)
+  assert.deepEqual(failedDiagnosis(checks), [])
+})
+
+test('gradeRun fails each part of the diagnosis the report misses', () => {
+  const checks = gradeRun(LCP_CASE, transcript([], 'The page has a slow server response.'), CATALOG)
+  assert.deepEqual(failedDiagnosis(checks), ['lcp-element', 'slow-phase'])
+})
+
+test('gradeRun reports how the agent measured: published snippets, its own code and the other tools', () => {
+  const t = transcript([
+    call('mcp__chrome-devtools__navigate_page', { url: 'http://x' }),
+    call('mcp__chrome-devtools__evaluate_script', { function: '() => 1' }),
+    call('mcp__chrome-devtools__evaluate_script', { sourcePath: '/s/scripts/LCP.js' }),
+    call('mcp__chrome-devtools__performance_start_trace', {}),
+  ])
+  const { measured } = Object.fromEntries(gradeRun(LCP_CASE, t, CATALOG).map((c) => [c.id, c]))
+  assert.deepEqual(measured.detail, { published: 1, ownCode: 1, otherTools: ['navigate_page', 'performance_start_trace'] })
+  assert.equal(measured.pass, true)
+})
+
+test('gradeRun fails the measured check when the agent measured nothing', () => {
+  const { measured } = Object.fromEntries(gradeRun(LCP_CASE, transcript([], 'LCP is probably the image.'), CATALOG).map((c) => [c.id, c]))
+  assert.equal(measured.pass, false)
+})