From d15c48a80d7517c2b4a91625fc2b6c0ab9551e2b Mon Sep 17 00:00:00 2001 From: Aurora Scharff Date: Mon, 24 Aug 2026 18:51:16 +0200 Subject: [PATCH 01/10] [evals] Split instant and navigation-stage coverage --- evals/evals/agent-040-instant/EVAL.ts | 73 +++++++---------- evals/evals/agent-040-instant/PROMPT.md | 2 +- .../EVAL.ts | 80 +++++++++++++++++++ .../PROMPT.md | 1 + .../app/layout.tsx | 18 +++++ .../app/page.tsx | 21 +++++ .../app/projects/[slug]/page.tsx | 33 ++++++++ .../lib/data.ts | 32 ++++++++ .../next.config.ts | 7 ++ .../package.json | 23 ++++++ .../tsconfig.json | 27 +++++++ 11 files changed, 272 insertions(+), 45 deletions(-) create mode 100644 evals/evals/agent-044-prefetch-until-navigation/EVAL.ts create mode 100644 evals/evals/agent-044-prefetch-until-navigation/PROMPT.md create mode 100644 evals/evals/agent-044-prefetch-until-navigation/app/layout.tsx create mode 100644 evals/evals/agent-044-prefetch-until-navigation/app/page.tsx create mode 100644 evals/evals/agent-044-prefetch-until-navigation/app/projects/[slug]/page.tsx create mode 100644 evals/evals/agent-044-prefetch-until-navigation/lib/data.ts create mode 100644 evals/evals/agent-044-prefetch-until-navigation/next.config.ts create mode 100644 evals/evals/agent-044-prefetch-until-navigation/package.json create mode 100644 evals/evals/agent-044-prefetch-until-navigation/tsconfig.json diff --git a/evals/evals/agent-040-instant/EVAL.ts b/evals/evals/agent-040-instant/EVAL.ts index 6764f178f0ba..6602df477af0 100644 --- a/evals/evals/agent-040-instant/EVAL.ts +++ b/evals/evals/agent-040-instant/EVAL.ts @@ -1,67 +1,52 @@ /** - * Instant Navigations + * Instant Validation * - * Verifies that the agent enables instant navigation on the product route. - * - * The eval harness runs `next build` as a script — with `cacheComponents: true`, - * the build validates that the caching structure produces an instant static shell. - * This test checks the agent actually exported `instant` (the build would - * pass without it, since Suspense alone satisfies the build on a static route). + * Verifies that the agent opts the product route into Instant Validation and + * produces a meaningful Cache Components shell: the stable title is available + * immediately while live inventory streams behind a Suspense boundary. */ import { expect, test } from 'vitest' -import { readFileSync, existsSync, readdirSync, statSync } from 'fs' +import { existsSync, readFileSync, readdirSync, statSync } from 'fs' import { join } from 'path' +import { environment } from '@vercel/agent-eval/eval' -type SourceFile = { path: string; content: string } - -const IGNORE_DIRS = new Set([ - '.git', - '.next', - 'node_modules', - 'dist', - 'build', - 'coverage', -]) - -const IGNORE_FILES = new Set(['EVAL.ts', 'PROMPT.md']) +function stripComments(source: string): string { + return source.replace(/\/\*[\s\S]*?\*\//g, '').replace(/\/\/.*$/gm, '') +} -function readSourceFiles(dir: string): SourceFile[] { +function readRouteSource(dir: string): string[] { if (!existsSync(dir)) return [] - const files: SourceFile[] = [] + const files: string[] = [] for (const entry of readdirSync(dir)) { - if (IGNORE_DIRS.has(entry)) continue - const fullPath = join(dir, entry) const stats = statSync(fullPath) - if (stats.isDirectory()) { - files.push(...readSourceFiles(fullPath)) - continue - } - - if (IGNORE_FILES.has(entry)) continue - - if (/\.(ts|tsx|js|jsx)$/.test(entry)) { - files.push({ - path: fullPath, - content: readFileSync(fullPath, 'utf-8'), - }) + files.push(...readRouteSource(fullPath)) + } else if (/\.(ts|tsx|js|jsx)$/.test(entry)) { + files.push(stripComments(readFileSync(fullPath, 'utf-8'))) } } - return files } -const sourceFiles = readSourceFiles(process.cwd()) +const productRouteSource = readRouteSource( + join(process.cwd(), 'app/product') +).join('\n---FILE---\n') -test('instant exported from the product route', () => { - const productPage = sourceFiles.find( - (f) => f.path.includes('product') && /page\.(tsx?|jsx?)$/.test(f.path) +test('opts the product route into Instant Validation', () => { + expect(productRouteSource).toMatch( + /export\s+const\s+instant\s*=\s*(?:true|\{[\s\S]*?level\s*:\s*['"]warning['"][\s\S]*?\})/ ) +}) - expect(productPage).toBeDefined() - expect(productPage!.content).toMatch(/export\s+(const|var|let)\s+instant\b/) - expect(productPage!.content).toMatch(/prefetch\s*:\s*['"]static['"]/) +test('keeps the title in the shell while live inventory streams', async () => { + await expect(environment).toSatisfyCriterion( + `The product route must produce meaningful instant UI with Cache Components, without relying on Partial Prefetching. + +The product title "Premium Widget" must render in the product page's static shell, outside the Suspense boundary that contains the live inventory. The inventory count and price must render from an async child beneath that Suspense boundary, so the page returns its title and fallback without waiting for getInventory(). The runtime boundary in getInventory(), such as connection(), must be preserved; do not cache the randomized inventory or replace it with static values. + +Accept equivalent component and file organization. Reject solutions where the page awaits inventory before returning, the title is hidden behind the inventory boundary, the live inventory is cached, or Partial Prefetching is used as a substitute for a valid instant shell.` + ) }) diff --git a/evals/evals/agent-040-instant/PROMPT.md b/evals/evals/agent-040-instant/PROMPT.md index f8cf59cb0706..1cd2ea323df6 100644 --- a/evals/evals/agent-040-instant/PROMPT.md +++ b/evals/evals/agent-040-instant/PROMPT.md @@ -1 +1 @@ -Navigating from home to the product page is slow. The title should appear immediately. +Navigating from home to the product page is slow. Make the product title appear immediately while the live inventory loads, and have Next.js keep validating that this navigation stays instant. diff --git a/evals/evals/agent-044-prefetch-until-navigation/EVAL.ts b/evals/evals/agent-044-prefetch-until-navigation/EVAL.ts new file mode 100644 index 000000000000..922e1394fa3c --- /dev/null +++ b/evals/evals/agent-044-prefetch-until-navigation/EVAL.ts @@ -0,0 +1,80 @@ +/** + * Prefetch until navigation + * + * Verifies that the agent uses Partial Prefetching to include a selected + * project's title in a per-link prefetch while deferring the rest of the + * project page until navigation. + */ + +import { expect, test } from 'vitest' +import { existsSync, readFileSync, readdirSync, statSync } from 'fs' +import { join } from 'path' +import { environment } from '@vercel/agent-eval/eval' + +const IGNORE_DIRS = new Set([ + '.git', + '.next', + 'node_modules', + 'dist', + 'build', + 'coverage', +]) + +const IGNORE_FILES = new Set(['EVAL.ts', 'PROMPT.md']) + +function stripComments(source: string): string { + return source.replace(/\/\*[\s\S]*?\*\//g, '').replace(/\/\/.*$/gm, '') +} + +function readSourceFiles(dir: string): string[] { + if (!existsSync(dir)) return [] + + const files: string[] = [] + for (const entry of readdirSync(dir)) { + if (IGNORE_DIRS.has(entry)) continue + + const fullPath = join(dir, entry) + const stats = statSync(fullPath) + if (stats.isDirectory()) { + files.push(...readSourceFiles(fullPath)) + } else if (!IGNORE_FILES.has(entry) && /\.(ts|tsx|js|jsx)$/.test(entry)) { + files.push(stripComments(readFileSync(fullPath, 'utf-8'))) + } + } + + return files +} + +const source = readSourceFiles(process.cwd()).join('\n---FILE---\n') +const config = stripComments( + readFileSync(join(process.cwd(), 'next.config.ts'), 'utf-8') +) + +test('enables Partial Prefetching globally or for the project route', () => { + expect( + /partialPrefetching\s*:\s*true/.test(config) || + /export\s+const\s+prefetch\s*=\s*['"]partial['"]/.test(source) + ).toBe(true) +}) + +test('uses the navigation cache stage', () => { + const navigationImport = source.match( + /\bunstable_navigation(?:\s+as\s+([A-Za-z_$][\w$]*))?/ + ) + expect(navigationImport).not.toBeNull() + + const localName = navigationImport?.[1] ?? 'unstable_navigation' + expect(source).toMatch(new RegExp(`await\\s+${localName}\\s*\\(`)) +}) + +test('the title is prefetched and project details wait for navigation', async () => { + await expect(environment).toSatisfyCriterion( + `The implementation must make the selected project's title available before the click without prefetching the rest of the project page. + +A correct solution uses Partial Prefetching globally or on the project destination, and opts the dashboard's project Link into a per-link prefetch with prefetch={true}. The project route keeps its params-dependent title inside Suspense so the reusable App Shell can show a fallback while the per-link prefetch resolves the title. + +The rendered project title must appear before a nested Suspense boundary. Inside that boundary, an async component must await unstable_navigation() before it loads or renders the project activity, deployments, description, or other detail content. The unstable_navigation() call must be outside any use cache scope; cached data functions may be called after it. + +Accept equivalent component and file organization. Reject solutions that put the title behind unstable_navigation(), fetch project details before reaching unstable_navigation(), replace unstable_navigation() with connection() or an uncached fetch, disable prefetching, or prefetch the entire page.` + ) +}) diff --git a/evals/evals/agent-044-prefetch-until-navigation/PROMPT.md b/evals/evals/agent-044-prefetch-until-navigation/PROMPT.md new file mode 100644 index 000000000000..2aae89c2e3b1 --- /dev/null +++ b/evals/evals/agent-044-prefetch-until-navigation/PROMPT.md @@ -0,0 +1 @@ +When navigating from the dashboard to a project, I want the project's title to appear instantly, but I don't want to prefetch the rest of the project page. diff --git a/evals/evals/agent-044-prefetch-until-navigation/app/layout.tsx b/evals/evals/agent-044-prefetch-until-navigation/app/layout.tsx new file mode 100644 index 000000000000..7309bb5579ad --- /dev/null +++ b/evals/evals/agent-044-prefetch-until-navigation/app/layout.tsx @@ -0,0 +1,18 @@ +import type { Metadata } from 'next' + +export const metadata: Metadata = { + title: 'Projects', + description: 'Project dashboard', +} + +export default function RootLayout({ + children, +}: Readonly<{ + children: React.ReactNode +}>) { + return ( + + {children} + + ) +} diff --git a/evals/evals/agent-044-prefetch-until-navigation/app/page.tsx b/evals/evals/agent-044-prefetch-until-navigation/app/page.tsx new file mode 100644 index 000000000000..8e4bcaee437c --- /dev/null +++ b/evals/evals/agent-044-prefetch-until-navigation/app/page.tsx @@ -0,0 +1,21 @@ +import Link from 'next/link' + +const projects = [ + { slug: 'acme', name: 'Acme Storefront' }, + { slug: 'orbit', name: 'Orbit Analytics' }, +] + +export default function Dashboard() { + return ( +
+

Projects

+ +
+ ) +} diff --git a/evals/evals/agent-044-prefetch-until-navigation/app/projects/[slug]/page.tsx b/evals/evals/agent-044-prefetch-until-navigation/app/projects/[slug]/page.tsx new file mode 100644 index 000000000000..ea3ad54ed029 --- /dev/null +++ b/evals/evals/agent-044-prefetch-until-navigation/app/projects/[slug]/page.tsx @@ -0,0 +1,33 @@ +import { Suspense } from 'react' +import { getProjectDetails, getProjectTitle } from '@/lib/data' + +export default function ProjectPage({ params }: PageProps<'/projects/[slug]'>) { + return ( + Loading project...}> + + + ) +} + +async function Project({ + params, +}: Pick, 'params'>) { + const { slug } = await params + const [title, details] = await Promise.all([ + getProjectTitle(slug), + getProjectDetails(slug), + ]) + + return ( +
+

{title}

+

{details.description}

+

Recent activity

+
    + {details.activity.map((item) => ( +
  • {item}
    • + ))} +
+
+ ) +} diff --git a/evals/evals/agent-044-prefetch-until-navigation/lib/data.ts b/evals/evals/agent-044-prefetch-until-navigation/lib/data.ts new file mode 100644 index 000000000000..7dddf09f7768 --- /dev/null +++ b/evals/evals/agent-044-prefetch-until-navigation/lib/data.ts @@ -0,0 +1,32 @@ +const projects: Record< + string, + { title: string; description: string; activity: string[] } +> = { + acme: { + title: 'Acme Storefront', + description: 'Commerce storefront and checkout.', + activity: ['Production deployment', 'Checkout updated', 'Domain verified'], + }, + orbit: { + title: 'Orbit Analytics', + description: 'Product analytics and reporting.', + activity: ['Report published', 'Data source added', 'Team invited'], + }, +} + +export async function getProjectTitle(slug: string) { + 'use cache' + return projects[slug]?.title ?? 'Unknown project' +} + +export async function getProjectDetails(slug: string) { + 'use cache' + await new Promise((resolve) => setTimeout(resolve, 800)) + return ( + projects[slug] ?? { + title: 'Unknown project', + description: 'This project could not be found.', + activity: [], + } + ) +} diff --git a/evals/evals/agent-044-prefetch-until-navigation/next.config.ts b/evals/evals/agent-044-prefetch-until-navigation/next.config.ts new file mode 100644 index 000000000000..fa33c7c54f24 --- /dev/null +++ b/evals/evals/agent-044-prefetch-until-navigation/next.config.ts @@ -0,0 +1,7 @@ +import type { NextConfig } from 'next' + +const nextConfig: NextConfig = { + cacheComponents: true, +} + +export default nextConfig diff --git a/evals/evals/agent-044-prefetch-until-navigation/package.json b/evals/evals/agent-044-prefetch-until-navigation/package.json new file mode 100644 index 000000000000..97f5fdec21f9 --- /dev/null +++ b/evals/evals/agent-044-prefetch-until-navigation/package.json @@ -0,0 +1,23 @@ +{ + "private": true, + "type": "module", + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start" + }, + "dependencies": { + "next": "^16", + "react": "19.1.0", + "react-dom": "19.1.0" + }, + "devDependencies": { + "@types/node": "^20", + "@types/react": "^19", + "@types/react-dom": "^19", + "@vitejs/plugin-react": "^4.4.1", + "typescript": "^5", + "vite-tsconfig-paths": "^5.1.4", + "vitest": "^3.1.3" + } +} diff --git a/evals/evals/agent-044-prefetch-until-navigation/tsconfig.json b/evals/evals/agent-044-prefetch-until-navigation/tsconfig.json new file mode 100644 index 000000000000..64c21044c49f --- /dev/null +++ b/evals/evals/agent-044-prefetch-until-navigation/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "incremental": true, + "plugins": [ + { + "name": "next" + } + ], + "paths": { + "@/*": ["./*"] + }, + "target": "ES2017" + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], + "exclude": ["node_modules"] +} From 3a361b52dca17373375da68db02f5f854045de13 Mon Sep 17 00:00:00 2001 From: Aurora Scharff Date: Mon, 24 Aug 2026 19:36:42 +0200 Subject: [PATCH 02/10] [evals] Use default instant validation --- evals/evals/agent-040-instant/EVAL.ts | 41 +++---------------- evals/evals/agent-040-instant/PROMPT.md | 2 +- evals/evals/agent-040-instant/tsconfig.json | 2 +- .../tsconfig.json | 2 +- 4 files changed, 8 insertions(+), 39 deletions(-) diff --git a/evals/evals/agent-040-instant/EVAL.ts b/evals/evals/agent-040-instant/EVAL.ts index 6602df477af0..a6ca887659e3 100644 --- a/evals/evals/agent-040-instant/EVAL.ts +++ b/evals/evals/agent-040-instant/EVAL.ts @@ -1,51 +1,20 @@ /** * Instant Validation * - * Verifies that the agent opts the product route into Instant Validation and - * produces a meaningful Cache Components shell: the stable title is available - * immediately while live inventory streams behind a Suspense boundary. + * Starts with a non-instant navigation and verifies that the agent follows the + * framework's default validation feedback to produce a meaningful Cache + * Components shell: the stable title is available immediately while live + * inventory streams behind a Suspense boundary. */ import { expect, test } from 'vitest' -import { existsSync, readFileSync, readdirSync, statSync } from 'fs' -import { join } from 'path' import { environment } from '@vercel/agent-eval/eval' -function stripComments(source: string): string { - return source.replace(/\/\*[\s\S]*?\*\//g, '').replace(/\/\/.*$/gm, '') -} - -function readRouteSource(dir: string): string[] { - if (!existsSync(dir)) return [] - - const files: string[] = [] - for (const entry of readdirSync(dir)) { - const fullPath = join(dir, entry) - const stats = statSync(fullPath) - if (stats.isDirectory()) { - files.push(...readRouteSource(fullPath)) - } else if (/\.(ts|tsx|js|jsx)$/.test(entry)) { - files.push(stripComments(readFileSync(fullPath, 'utf-8'))) - } - } - return files -} - -const productRouteSource = readRouteSource( - join(process.cwd(), 'app/product') -).join('\n---FILE---\n') - -test('opts the product route into Instant Validation', () => { - expect(productRouteSource).toMatch( - /export\s+const\s+instant\s*=\s*(?:true|\{[\s\S]*?level\s*:\s*['"]warning['"][\s\S]*?\})/ - ) -}) - test('keeps the title in the shell while live inventory streams', async () => { await expect(environment).toSatisfyCriterion( `The product route must produce meaningful instant UI with Cache Components, without relying on Partial Prefetching. -The product title "Premium Widget" must render in the product page's static shell, outside the Suspense boundary that contains the live inventory. The inventory count and price must render from an async child beneath that Suspense boundary, so the page returns its title and fallback without waiting for getInventory(). The runtime boundary in getInventory(), such as connection(), must be preserved; do not cache the randomized inventory or replace it with static values. +The product title "Premium Widget" must render in the product page's static shell, outside the Suspense boundary that contains the live inventory. The inventory count and price must render from an async child beneath that Suspense boundary, so the page returns its title and fallback without waiting for getInventory(). The runtime boundary in getInventory(), such as connection(), must be preserved; do not cache the randomized inventory or replace it with static values. Keep the framework's default validation enabled; do not add instant = false to suppress the insight. Accept equivalent component and file organization. Reject solutions where the page awaits inventory before returning, the title is hidden behind the inventory boundary, the live inventory is cached, or Partial Prefetching is used as a substitute for a valid instant shell.` ) diff --git a/evals/evals/agent-040-instant/PROMPT.md b/evals/evals/agent-040-instant/PROMPT.md index 1cd2ea323df6..5ea6f2295695 100644 --- a/evals/evals/agent-040-instant/PROMPT.md +++ b/evals/evals/agent-040-instant/PROMPT.md @@ -1 +1 @@ -Navigating from home to the product page is slow. Make the product title appear immediately while the live inventory loads, and have Next.js keep validating that this navigation stays instant. +Navigating from home to the product page is slow. Make the product title appear immediately while the live inventory loads. diff --git a/evals/evals/agent-040-instant/tsconfig.json b/evals/evals/agent-040-instant/tsconfig.json index 64c21044c49f..8eb9f7f78971 100644 --- a/evals/evals/agent-040-instant/tsconfig.json +++ b/evals/evals/agent-040-instant/tsconfig.json @@ -23,5 +23,5 @@ "target": "ES2017" }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], - "exclude": ["node_modules"] + "exclude": ["node_modules", "EVAL.ts"] } diff --git a/evals/evals/agent-044-prefetch-until-navigation/tsconfig.json b/evals/evals/agent-044-prefetch-until-navigation/tsconfig.json index 64c21044c49f..8eb9f7f78971 100644 --- a/evals/evals/agent-044-prefetch-until-navigation/tsconfig.json +++ b/evals/evals/agent-044-prefetch-until-navigation/tsconfig.json @@ -23,5 +23,5 @@ "target": "ES2017" }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], - "exclude": ["node_modules"] + "exclude": ["node_modules", "EVAL.ts"] } From 45090e575c6e4a3c03939c911e63c9ec4098f1ad Mon Sep 17 00:00:00 2001 From: Aurora Scharff Date: Mon, 24 Aug 2026 19:45:00 +0200 Subject: [PATCH 03/10] [evals] Add URL data prefetch baseline --- .../evals/agent-044-prefetch-url-data/EVAL.ts | 80 +++++++++++++++++++ .../agent-044-prefetch-url-data/PROMPT.md | 1 + .../app/catalog/page.tsx | 39 +++++++++ .../app/layout.tsx | 18 +++++ .../agent-044-prefetch-url-data/app/page.tsx | 23 ++++++ .../agent-044-prefetch-url-data/lib/data.ts | 9 +++ .../next.config.ts | 0 .../package.json | 0 .../tsconfig.json | 0 .../EVAL.ts | 0 .../PROMPT.md | 0 .../app/layout.tsx | 0 .../app/page.tsx | 0 .../app/projects/[slug]/page.tsx | 0 .../lib/data.ts | 0 .../next.config.ts | 7 ++ .../package.json | 23 ++++++ .../tsconfig.json | 27 +++++++ 18 files changed, 227 insertions(+) create mode 100644 evals/evals/agent-044-prefetch-url-data/EVAL.ts create mode 100644 evals/evals/agent-044-prefetch-url-data/PROMPT.md create mode 100644 evals/evals/agent-044-prefetch-url-data/app/catalog/page.tsx create mode 100644 evals/evals/agent-044-prefetch-url-data/app/layout.tsx create mode 100644 evals/evals/agent-044-prefetch-url-data/app/page.tsx create mode 100644 evals/evals/agent-044-prefetch-url-data/lib/data.ts rename evals/evals/{agent-044-prefetch-until-navigation => agent-044-prefetch-url-data}/next.config.ts (100%) rename evals/evals/{agent-044-prefetch-until-navigation => agent-044-prefetch-url-data}/package.json (100%) rename evals/evals/{agent-044-prefetch-until-navigation => agent-044-prefetch-url-data}/tsconfig.json (100%) rename evals/evals/{agent-044-prefetch-until-navigation => agent-045-prefetch-until-navigation}/EVAL.ts (100%) rename evals/evals/{agent-044-prefetch-until-navigation => agent-045-prefetch-until-navigation}/PROMPT.md (100%) rename evals/evals/{agent-044-prefetch-until-navigation => agent-045-prefetch-until-navigation}/app/layout.tsx (100%) rename evals/evals/{agent-044-prefetch-until-navigation => agent-045-prefetch-until-navigation}/app/page.tsx (100%) rename evals/evals/{agent-044-prefetch-until-navigation => agent-045-prefetch-until-navigation}/app/projects/[slug]/page.tsx (100%) rename evals/evals/{agent-044-prefetch-until-navigation => agent-045-prefetch-until-navigation}/lib/data.ts (100%) create mode 100644 evals/evals/agent-045-prefetch-until-navigation/next.config.ts create mode 100644 evals/evals/agent-045-prefetch-until-navigation/package.json create mode 100644 evals/evals/agent-045-prefetch-until-navigation/tsconfig.json diff --git a/evals/evals/agent-044-prefetch-url-data/EVAL.ts b/evals/evals/agent-044-prefetch-url-data/EVAL.ts new file mode 100644 index 000000000000..ab17ec494bb2 --- /dev/null +++ b/evals/evals/agent-044-prefetch-url-data/EVAL.ts @@ -0,0 +1,80 @@ +/** + * Prefetch URL data + * + * Verifies the basic Partial Prefetching inclusion case: the agent caches data + * that depends on searchParams and makes it available in a per-link prefetch. + */ + +import { expect, test } from 'vitest' +import { existsSync, readFileSync, readdirSync, statSync } from 'fs' +import { join } from 'path' +import { environment } from '@vercel/agent-eval/eval' + +const IGNORE_DIRS = new Set([ + '.git', + '.next', + 'node_modules', + 'dist', + 'build', + 'coverage', +]) + +const IGNORE_FILES = new Set(['EVAL.ts', 'PROMPT.md']) + +function stripComments(source: string): string { + return source.replace(/\/\*[\s\S]*?\*\//g, '').replace(/\/\/.*$/gm, '') +} + +function readSourceFiles(dir: string): string[] { + if (!existsSync(dir)) return [] + + const files: string[] = [] + for (const entry of readdirSync(dir)) { + if (IGNORE_DIRS.has(entry)) continue + + const fullPath = join(dir, entry) + const stats = statSync(fullPath) + if (stats.isDirectory()) { + files.push(...readSourceFiles(fullPath)) + } else if (!IGNORE_FILES.has(entry) && /\.(ts|tsx|js|jsx)$/.test(entry)) { + files.push(stripComments(readFileSync(fullPath, 'utf-8'))) + } + } + + return files +} + +const source = readSourceFiles(process.cwd()).join('\n---FILE---\n') +const config = stripComments( + readFileSync(join(process.cwd(), 'next.config.ts'), 'utf-8') +) + +test('enables Partial Prefetching globally or for the catalog route', () => { + expect( + /partialPrefetching\s*:\s*true/.test(config) || + /export\s+const\s+prefetch\s*=\s*['"]partial['"]/.test(source) + ).toBe(true) +}) + +test('opts the category links into per-link prefetching', () => { + expect(source).toMatch(/]*\bprefetch\s*=\s*\{true\}/) +}) + +test('caches the category data with an explicit lifetime', () => { + expect(source).toMatch(/['"]use cache['"]/) + expect(source).toMatch(/\bcacheLife\s*\(/) +}) + +test('does not use a navigation-stage boundary', () => { + expect(source).not.toMatch(/\bunstable_navigation\b/) +}) + +test('prefetches the selected category and cached products', async () => { + await expect(environment).toSatisfyCriterion( + `The implementation must make the selected category heading and its cached products available from the completed per-link prefetch before navigation. + +A correct solution enables Partial Prefetching globally or on the catalog route and opts the category Links into prefetch={true}. The catalog route reads the category from searchParams beneath Suspense and renders both the matching heading and getProducts(category) result in the prefetched result. The agent must add a use cache scope with an explicit cacheLife() for the category data; searchParams must remain outside that cache scope. + +This is the basic URL-data inclusion case. Do not use unstable_navigation(), unstable_prefetch(), connection(), or an uncached request boundary to defer the category content. Reject solutions that only prefetch the reusable Catalog shell while leaving the selected heading or product list behind the loading fallback.` + ) +}) diff --git a/evals/evals/agent-044-prefetch-url-data/PROMPT.md b/evals/evals/agent-044-prefetch-url-data/PROMPT.md new file mode 100644 index 000000000000..b5bd2812273c --- /dev/null +++ b/evals/evals/agent-044-prefetch-url-data/PROMPT.md @@ -0,0 +1 @@ +When navigating from home to a product category, I want the selected category heading and its products to appear immediately instead of the loading state. diff --git a/evals/evals/agent-044-prefetch-url-data/app/catalog/page.tsx b/evals/evals/agent-044-prefetch-url-data/app/catalog/page.tsx new file mode 100644 index 000000000000..4b39f6859c1c --- /dev/null +++ b/evals/evals/agent-044-prefetch-url-data/app/catalog/page.tsx @@ -0,0 +1,39 @@ +import { Suspense } from 'react' +import { getProducts } from '@/lib/data' + +type CatalogSearchParams = Promise<{ category?: string }> + +export default function CatalogPage({ + searchParams, +}: { + searchParams: CatalogSearchParams +}) { + return ( +
+

Catalog

+ Loading category...

}> + + +
+ ) +} + +async function CategoryResults({ + searchParams, +}: { + searchParams: CatalogSearchParams +}) { + const { category = 'featured' } = await searchParams + const products = await getProducts(category) + + return ( +
+

{category === 'sale' ? 'Products on sale' : 'Featured products'}

+
    + {products.map((product) => ( +
  • {product}
    • + ))} +
+
+ ) +} diff --git a/evals/evals/agent-044-prefetch-url-data/app/layout.tsx b/evals/evals/agent-044-prefetch-url-data/app/layout.tsx new file mode 100644 index 000000000000..e6f0fb1729b7 --- /dev/null +++ b/evals/evals/agent-044-prefetch-url-data/app/layout.tsx @@ -0,0 +1,18 @@ +import type { Metadata } from 'next' + +export const metadata: Metadata = { + title: 'Product catalog', + description: 'Browse products by category', +} + +export default function RootLayout({ + children, +}: Readonly<{ + children: React.ReactNode +}>) { + return ( + + {children} + + ) +} diff --git a/evals/evals/agent-044-prefetch-url-data/app/page.tsx b/evals/evals/agent-044-prefetch-url-data/app/page.tsx new file mode 100644 index 000000000000..989c82c49c6a --- /dev/null +++ b/evals/evals/agent-044-prefetch-url-data/app/page.tsx @@ -0,0 +1,23 @@ +import Link from 'next/link' + +const categories = [ + { slug: 'featured', label: 'Featured products' }, + { slug: 'sale', label: 'Products on sale' }, +] + +export default function Home() { + return ( +
+

Shop by category

+
    + {categories.map((category) => ( +
  • + + {category.label} + +
    • + ))} +
+
+ ) +} diff --git a/evals/evals/agent-044-prefetch-url-data/lib/data.ts b/evals/evals/agent-044-prefetch-url-data/lib/data.ts new file mode 100644 index 000000000000..d0eb6aac8080 --- /dev/null +++ b/evals/evals/agent-044-prefetch-url-data/lib/data.ts @@ -0,0 +1,9 @@ +const productsByCategory: Record = { + featured: ['Desk Lamp', 'Wool Blanket'], + sale: ['Canvas Tote', 'Travel Mug'], +} + +export async function getProducts(category: string) { + await new Promise((resolve) => setTimeout(resolve, 800)) + return productsByCategory[category] ?? [] +} diff --git a/evals/evals/agent-044-prefetch-until-navigation/next.config.ts b/evals/evals/agent-044-prefetch-url-data/next.config.ts similarity index 100% rename from evals/evals/agent-044-prefetch-until-navigation/next.config.ts rename to evals/evals/agent-044-prefetch-url-data/next.config.ts diff --git a/evals/evals/agent-044-prefetch-until-navigation/package.json b/evals/evals/agent-044-prefetch-url-data/package.json similarity index 100% rename from evals/evals/agent-044-prefetch-until-navigation/package.json rename to evals/evals/agent-044-prefetch-url-data/package.json diff --git a/evals/evals/agent-044-prefetch-until-navigation/tsconfig.json b/evals/evals/agent-044-prefetch-url-data/tsconfig.json similarity index 100% rename from evals/evals/agent-044-prefetch-until-navigation/tsconfig.json rename to evals/evals/agent-044-prefetch-url-data/tsconfig.json diff --git a/evals/evals/agent-044-prefetch-until-navigation/EVAL.ts b/evals/evals/agent-045-prefetch-until-navigation/EVAL.ts similarity index 100% rename from evals/evals/agent-044-prefetch-until-navigation/EVAL.ts rename to evals/evals/agent-045-prefetch-until-navigation/EVAL.ts diff --git a/evals/evals/agent-044-prefetch-until-navigation/PROMPT.md b/evals/evals/agent-045-prefetch-until-navigation/PROMPT.md similarity index 100% rename from evals/evals/agent-044-prefetch-until-navigation/PROMPT.md rename to evals/evals/agent-045-prefetch-until-navigation/PROMPT.md diff --git a/evals/evals/agent-044-prefetch-until-navigation/app/layout.tsx b/evals/evals/agent-045-prefetch-until-navigation/app/layout.tsx similarity index 100% rename from evals/evals/agent-044-prefetch-until-navigation/app/layout.tsx rename to evals/evals/agent-045-prefetch-until-navigation/app/layout.tsx diff --git a/evals/evals/agent-044-prefetch-until-navigation/app/page.tsx b/evals/evals/agent-045-prefetch-until-navigation/app/page.tsx similarity index 100% rename from evals/evals/agent-044-prefetch-until-navigation/app/page.tsx rename to evals/evals/agent-045-prefetch-until-navigation/app/page.tsx diff --git a/evals/evals/agent-044-prefetch-until-navigation/app/projects/[slug]/page.tsx b/evals/evals/agent-045-prefetch-until-navigation/app/projects/[slug]/page.tsx similarity index 100% rename from evals/evals/agent-044-prefetch-until-navigation/app/projects/[slug]/page.tsx rename to evals/evals/agent-045-prefetch-until-navigation/app/projects/[slug]/page.tsx diff --git a/evals/evals/agent-044-prefetch-until-navigation/lib/data.ts b/evals/evals/agent-045-prefetch-until-navigation/lib/data.ts similarity index 100% rename from evals/evals/agent-044-prefetch-until-navigation/lib/data.ts rename to evals/evals/agent-045-prefetch-until-navigation/lib/data.ts diff --git a/evals/evals/agent-045-prefetch-until-navigation/next.config.ts b/evals/evals/agent-045-prefetch-until-navigation/next.config.ts new file mode 100644 index 000000000000..fa33c7c54f24 --- /dev/null +++ b/evals/evals/agent-045-prefetch-until-navigation/next.config.ts @@ -0,0 +1,7 @@ +import type { NextConfig } from 'next' + +const nextConfig: NextConfig = { + cacheComponents: true, +} + +export default nextConfig diff --git a/evals/evals/agent-045-prefetch-until-navigation/package.json b/evals/evals/agent-045-prefetch-until-navigation/package.json new file mode 100644 index 000000000000..97f5fdec21f9 --- /dev/null +++ b/evals/evals/agent-045-prefetch-until-navigation/package.json @@ -0,0 +1,23 @@ +{ + "private": true, + "type": "module", + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start" + }, + "dependencies": { + "next": "^16", + "react": "19.1.0", + "react-dom": "19.1.0" + }, + "devDependencies": { + "@types/node": "^20", + "@types/react": "^19", + "@types/react-dom": "^19", + "@vitejs/plugin-react": "^4.4.1", + "typescript": "^5", + "vite-tsconfig-paths": "^5.1.4", + "vitest": "^3.1.3" + } +} diff --git a/evals/evals/agent-045-prefetch-until-navigation/tsconfig.json b/evals/evals/agent-045-prefetch-until-navigation/tsconfig.json new file mode 100644 index 000000000000..8eb9f7f78971 --- /dev/null +++ b/evals/evals/agent-045-prefetch-until-navigation/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "incremental": true, + "plugins": [ + { + "name": "next" + } + ], + "paths": { + "@/*": ["./*"] + }, + "target": "ES2017" + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], + "exclude": ["node_modules", "EVAL.ts"] +} From 572ff27cd0a0747cff6968082f65fc0a80a39e9a Mon Sep 17 00:00:00 2001 From: Aurora Scharff Date: Mon, 24 Aug 2026 19:47:44 +0200 Subject: [PATCH 04/10] up --- evals/evals/agent-045-prefetch-stage/EVAL.ts | 81 +++++++++++++++++++ .../evals/agent-045-prefetch-stage/PROMPT.md | 1 + .../agent-045-prefetch-stage/app/layout.tsx | 18 +++++ .../agent-045-prefetch-stage/app/page.tsx | 10 +++ .../app/product/page.tsx | 20 +++++ .../agent-045-prefetch-stage/lib/data.ts | 7 ++ .../next.config.ts | 0 .../package.json | 0 .../tsconfig.json | 0 .../EVAL.ts | 0 .../PROMPT.md | 0 .../app/layout.tsx | 0 .../app/page.tsx | 0 .../app/projects/[slug]/page.tsx | 0 .../lib/data.ts | 0 .../next.config.ts | 7 ++ .../package.json | 23 ++++++ .../tsconfig.json | 27 +++++++ 18 files changed, 194 insertions(+) create mode 100644 evals/evals/agent-045-prefetch-stage/EVAL.ts create mode 100644 evals/evals/agent-045-prefetch-stage/PROMPT.md create mode 100644 evals/evals/agent-045-prefetch-stage/app/layout.tsx create mode 100644 evals/evals/agent-045-prefetch-stage/app/page.tsx create mode 100644 evals/evals/agent-045-prefetch-stage/app/product/page.tsx create mode 100644 evals/evals/agent-045-prefetch-stage/lib/data.ts rename evals/evals/{agent-045-prefetch-until-navigation => agent-045-prefetch-stage}/next.config.ts (100%) rename evals/evals/{agent-045-prefetch-until-navigation => agent-045-prefetch-stage}/package.json (100%) rename evals/evals/{agent-045-prefetch-until-navigation => agent-045-prefetch-stage}/tsconfig.json (100%) rename evals/evals/{agent-045-prefetch-until-navigation => agent-046-prefetch-until-navigation}/EVAL.ts (100%) rename evals/evals/{agent-045-prefetch-until-navigation => agent-046-prefetch-until-navigation}/PROMPT.md (100%) rename evals/evals/{agent-045-prefetch-until-navigation => agent-046-prefetch-until-navigation}/app/layout.tsx (100%) rename evals/evals/{agent-045-prefetch-until-navigation => agent-046-prefetch-until-navigation}/app/page.tsx (100%) rename evals/evals/{agent-045-prefetch-until-navigation => agent-046-prefetch-until-navigation}/app/projects/[slug]/page.tsx (100%) rename evals/evals/{agent-045-prefetch-until-navigation => agent-046-prefetch-until-navigation}/lib/data.ts (100%) create mode 100644 evals/evals/agent-046-prefetch-until-navigation/next.config.ts create mode 100644 evals/evals/agent-046-prefetch-until-navigation/package.json create mode 100644 evals/evals/agent-046-prefetch-until-navigation/tsconfig.json diff --git a/evals/evals/agent-045-prefetch-stage/EVAL.ts b/evals/evals/agent-045-prefetch-stage/EVAL.ts new file mode 100644 index 000000000000..4cfa4e69eea1 --- /dev/null +++ b/evals/evals/agent-045-prefetch-stage/EVAL.ts @@ -0,0 +1,81 @@ +/** + * Prefetch stage + * + * Verifies that the agent removes cached content from the reusable App Shell + * while keeping it available to a selected per-link prefetch. + */ + +import { expect, test } from 'vitest' +import { existsSync, readFileSync, readdirSync, statSync } from 'fs' +import { join } from 'path' +import { environment } from '@vercel/agent-eval/eval' + +const IGNORE_DIRS = new Set([ + '.git', + '.next', + 'node_modules', + 'dist', + 'build', + 'coverage', +]) + +const IGNORE_FILES = new Set(['EVAL.ts', 'PROMPT.md']) + +function stripComments(source: string): string { + return source.replace(/\/\*[\s\S]*?\*\//g, '').replace(/\/\/.*$/gm, '') +} + +function readSourceFiles(dir: string): string[] { + if (!existsSync(dir)) return [] + + const files: string[] = [] + for (const entry of readdirSync(dir)) { + if (IGNORE_DIRS.has(entry)) continue + + const fullPath = join(dir, entry) + const stats = statSync(fullPath) + if (stats.isDirectory()) { + files.push(...readSourceFiles(fullPath)) + } else if (!IGNORE_FILES.has(entry) && /\.(ts|tsx|js|jsx)$/.test(entry)) { + files.push(stripComments(readFileSync(fullPath, 'utf-8'))) + } + } + + return files +} + +const source = readSourceFiles(process.cwd()).join('\n---FILE---\n') +const config = stripComments( + readFileSync(join(process.cwd(), 'next.config.ts'), 'utf-8') +) + +test('enables Partial Prefetching globally or for the product route', () => { + expect( + /partialPrefetching\s*:\s*true/.test(config) || + /export\s+const\s+prefetch\s*=\s*['"]partial['"]/.test(source) + ).toBe(true) +}) + +test('opts the featured product link into per-link prefetching', () => { + expect(source).toMatch(/]*\bprefetch\s*=\s*\{true\}/) +}) + +test('uses the prefetch cache stage', () => { + const prefetchImport = source.match( + /\bunstable_prefetch(?:\s+as\s+([A-Za-z_$][\w$]*))?/ + ) + expect(prefetchImport).not.toBeNull() + + const localName = prefetchImport?.[1] ?? 'unstable_prefetch' + expect(source).toMatch(new RegExp(`await\\s+${localName}\\s*\\(`)) +}) + +test('keeps related products out of the shell but in the selected prefetch', async () => { + await expect(environment).toSatisfyCriterion( + `The implementation must keep the related-products section out of the reusable product App Shell while making it available before navigation from the featured-product Link. + +A correct solution enables Partial Prefetching globally or on the product route and adds prefetch={true} to the featured-product Link. The product title and description stay in the reusable App Shell. The related-products section is inside Suspense and rendered by an async component that awaits unstable_prefetch() before calling the cached getRelatedProducts() function. + +The unstable_prefetch() call must be outside every use cache scope; the existing cached data function belongs below it. Reject solutions that put the whole product page behind the stage, move related products into the App Shell, replace the stage with connection() or unstable_navigation(), make the related-products data uncached, or defer it until after navigation.` + ) +}) diff --git a/evals/evals/agent-045-prefetch-stage/PROMPT.md b/evals/evals/agent-045-prefetch-stage/PROMPT.md new file mode 100644 index 000000000000..49eda1fba3d1 --- /dev/null +++ b/evals/evals/agent-045-prefetch-stage/PROMPT.md @@ -0,0 +1 @@ +Keep related products out of the default product-page prefetch, but have them ready immediately when navigating from the featured-product link. diff --git a/evals/evals/agent-045-prefetch-stage/app/layout.tsx b/evals/evals/agent-045-prefetch-stage/app/layout.tsx new file mode 100644 index 000000000000..39e6ae48c2c2 --- /dev/null +++ b/evals/evals/agent-045-prefetch-stage/app/layout.tsx @@ -0,0 +1,18 @@ +import type { Metadata } from 'next' + +export const metadata: Metadata = { + title: 'Featured product', + description: 'Shop our featured product', +} + +export default function RootLayout({ + children, +}: Readonly<{ + children: React.ReactNode +}>) { + return ( + + {children} + + ) +} diff --git a/evals/evals/agent-045-prefetch-stage/app/page.tsx b/evals/evals/agent-045-prefetch-stage/app/page.tsx new file mode 100644 index 000000000000..090426fbde9a --- /dev/null +++ b/evals/evals/agent-045-prefetch-stage/app/page.tsx @@ -0,0 +1,10 @@ +import Link from 'next/link' + +export default function Home() { + return ( +
+

Featured product

+ View Premium Widget +
+ ) +} diff --git a/evals/evals/agent-045-prefetch-stage/app/product/page.tsx b/evals/evals/agent-045-prefetch-stage/app/product/page.tsx new file mode 100644 index 000000000000..7c0fb1623c4e --- /dev/null +++ b/evals/evals/agent-045-prefetch-stage/app/product/page.tsx @@ -0,0 +1,20 @@ +import { getRelatedProducts } from '@/lib/data' + +export default async function ProductPage() { + const relatedProducts = await getRelatedProducts() + + return ( +
+

Premium Widget

+

A durable widget for everyday projects.

+
+

Related products

+
    + {relatedProducts.map((product) => ( +
  • {product}
    • + ))} +
+
+
+ ) +} diff --git a/evals/evals/agent-045-prefetch-stage/lib/data.ts b/evals/evals/agent-045-prefetch-stage/lib/data.ts new file mode 100644 index 000000000000..239cf973e35f --- /dev/null +++ b/evals/evals/agent-045-prefetch-stage/lib/data.ts @@ -0,0 +1,7 @@ +import { cacheLife } from 'next/cache' + +export async function getRelatedProducts() { + 'use cache' + cacheLife('hours') + return ['Compact Widget', 'Widget Carry Case'] +} diff --git a/evals/evals/agent-045-prefetch-until-navigation/next.config.ts b/evals/evals/agent-045-prefetch-stage/next.config.ts similarity index 100% rename from evals/evals/agent-045-prefetch-until-navigation/next.config.ts rename to evals/evals/agent-045-prefetch-stage/next.config.ts diff --git a/evals/evals/agent-045-prefetch-until-navigation/package.json b/evals/evals/agent-045-prefetch-stage/package.json similarity index 100% rename from evals/evals/agent-045-prefetch-until-navigation/package.json rename to evals/evals/agent-045-prefetch-stage/package.json diff --git a/evals/evals/agent-045-prefetch-until-navigation/tsconfig.json b/evals/evals/agent-045-prefetch-stage/tsconfig.json similarity index 100% rename from evals/evals/agent-045-prefetch-until-navigation/tsconfig.json rename to evals/evals/agent-045-prefetch-stage/tsconfig.json diff --git a/evals/evals/agent-045-prefetch-until-navigation/EVAL.ts b/evals/evals/agent-046-prefetch-until-navigation/EVAL.ts similarity index 100% rename from evals/evals/agent-045-prefetch-until-navigation/EVAL.ts rename to evals/evals/agent-046-prefetch-until-navigation/EVAL.ts diff --git a/evals/evals/agent-045-prefetch-until-navigation/PROMPT.md b/evals/evals/agent-046-prefetch-until-navigation/PROMPT.md similarity index 100% rename from evals/evals/agent-045-prefetch-until-navigation/PROMPT.md rename to evals/evals/agent-046-prefetch-until-navigation/PROMPT.md diff --git a/evals/evals/agent-045-prefetch-until-navigation/app/layout.tsx b/evals/evals/agent-046-prefetch-until-navigation/app/layout.tsx similarity index 100% rename from evals/evals/agent-045-prefetch-until-navigation/app/layout.tsx rename to evals/evals/agent-046-prefetch-until-navigation/app/layout.tsx diff --git a/evals/evals/agent-045-prefetch-until-navigation/app/page.tsx b/evals/evals/agent-046-prefetch-until-navigation/app/page.tsx similarity index 100% rename from evals/evals/agent-045-prefetch-until-navigation/app/page.tsx rename to evals/evals/agent-046-prefetch-until-navigation/app/page.tsx diff --git a/evals/evals/agent-045-prefetch-until-navigation/app/projects/[slug]/page.tsx b/evals/evals/agent-046-prefetch-until-navigation/app/projects/[slug]/page.tsx similarity index 100% rename from evals/evals/agent-045-prefetch-until-navigation/app/projects/[slug]/page.tsx rename to evals/evals/agent-046-prefetch-until-navigation/app/projects/[slug]/page.tsx diff --git a/evals/evals/agent-045-prefetch-until-navigation/lib/data.ts b/evals/evals/agent-046-prefetch-until-navigation/lib/data.ts similarity index 100% rename from evals/evals/agent-045-prefetch-until-navigation/lib/data.ts rename to evals/evals/agent-046-prefetch-until-navigation/lib/data.ts diff --git a/evals/evals/agent-046-prefetch-until-navigation/next.config.ts b/evals/evals/agent-046-prefetch-until-navigation/next.config.ts new file mode 100644 index 000000000000..fa33c7c54f24 --- /dev/null +++ b/evals/evals/agent-046-prefetch-until-navigation/next.config.ts @@ -0,0 +1,7 @@ +import type { NextConfig } from 'next' + +const nextConfig: NextConfig = { + cacheComponents: true, +} + +export default nextConfig diff --git a/evals/evals/agent-046-prefetch-until-navigation/package.json b/evals/evals/agent-046-prefetch-until-navigation/package.json new file mode 100644 index 000000000000..97f5fdec21f9 --- /dev/null +++ b/evals/evals/agent-046-prefetch-until-navigation/package.json @@ -0,0 +1,23 @@ +{ + "private": true, + "type": "module", + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start" + }, + "dependencies": { + "next": "^16", + "react": "19.1.0", + "react-dom": "19.1.0" + }, + "devDependencies": { + "@types/node": "^20", + "@types/react": "^19", + "@types/react-dom": "^19", + "@vitejs/plugin-react": "^4.4.1", + "typescript": "^5", + "vite-tsconfig-paths": "^5.1.4", + "vitest": "^3.1.3" + } +} diff --git a/evals/evals/agent-046-prefetch-until-navigation/tsconfig.json b/evals/evals/agent-046-prefetch-until-navigation/tsconfig.json new file mode 100644 index 000000000000..8eb9f7f78971 --- /dev/null +++ b/evals/evals/agent-046-prefetch-until-navigation/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "incremental": true, + "plugins": [ + { + "name": "next" + } + ], + "paths": { + "@/*": ["./*"] + }, + "target": "ES2017" + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], + "exclude": ["node_modules", "EVAL.ts"] +} From 2abda5f492e217d727f7a58ab88b0971f8784ab6 Mon Sep 17 00:00:00 2001 From: Aurora Scharff Date: Tue, 25 Aug 2026 15:46:17 +0200 Subject: [PATCH 05/10] evals: align URL data prefetch criteria --- evals/evals/agent-044-prefetch-url-data/EVAL.ts | 12 ++++++++---- evals/evals/agent-044-prefetch-url-data/PROMPT.md | 2 +- evals/lib/setup.ts | 4 ++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/evals/evals/agent-044-prefetch-url-data/EVAL.ts b/evals/evals/agent-044-prefetch-url-data/EVAL.ts index ab17ec494bb2..c0de1576146b 100644 --- a/evals/evals/agent-044-prefetch-url-data/EVAL.ts +++ b/evals/evals/agent-044-prefetch-url-data/EVAL.ts @@ -56,13 +56,17 @@ test('enables Partial Prefetching globally or for the catalog route', () => { ).toBe(true) }) +test('preserves the category URL structure', () => { + expect(source).toMatch(/\/catalog\?category=/) + expect(source).toMatch(/\bsearchParams\b/) +}) + test('opts the category links into per-link prefetching', () => { - expect(source).toMatch(/]*\bprefetch\s*=\s*\{true\}/) + expect(source).toMatch(/]*\bprefetch(?:\s*=\s*\{true\})?(?:\s|>)/) }) -test('caches the category data with an explicit lifetime', () => { +test('caches the category data', () => { expect(source).toMatch(/['"]use cache['"]/) - expect(source).toMatch(/\bcacheLife\s*\(/) }) test('does not use a navigation-stage boundary', () => { @@ -73,7 +77,7 @@ test('prefetches the selected category and cached products', async () => { await expect(environment).toSatisfyCriterion( `The implementation must make the selected category heading and its cached products available from the completed per-link prefetch before navigation. -A correct solution enables Partial Prefetching globally or on the catalog route and opts the category Links into prefetch={true}. The catalog route reads the category from searchParams beneath Suspense and renders both the matching heading and getProducts(category) result in the prefetched result. The agent must add a use cache scope with an explicit cacheLife() for the category data; searchParams must remain outside that cache scope. +A correct solution preserves the existing /catalog?category=... URLs, enables Partial Prefetching globally or on the catalog route, and opts the category Links into per-link prefetching with prefetch={true} or the equivalent bare prefetch prop. The catalog route reads the category from searchParams beneath Suspense and renders both the matching heading and getProducts(category) result in the prefetched result. The category data must use a use cache scope, with either the default cache profile or an explicit cacheLife(); searchParams must remain outside that cache scope. Keep the Suspense boundary: the shared App Shell uses its fallback, while a completed per-link prefetch can resolve the cached content inside it before navigation. This is the basic URL-data inclusion case. Do not use unstable_navigation(), unstable_prefetch(), connection(), or an uncached request boundary to defer the category content. Reject solutions that only prefetch the reusable Catalog shell while leaving the selected heading or product list behind the loading fallback.` ) diff --git a/evals/evals/agent-044-prefetch-url-data/PROMPT.md b/evals/evals/agent-044-prefetch-url-data/PROMPT.md index b5bd2812273c..6389e10ed201 100644 --- a/evals/evals/agent-044-prefetch-url-data/PROMPT.md +++ b/evals/evals/agent-044-prefetch-url-data/PROMPT.md @@ -1 +1 @@ -When navigating from home to a product category, I want the selected category heading and its products to appear immediately instead of the loading state. +Keep the existing `/catalog?category=...` URLs. When navigating from home to a product category, I want the selected category heading and its products to appear immediately instead of the loading state. diff --git a/evals/lib/setup.ts b/evals/lib/setup.ts index b2fed8ea214a..a1b0759a2b2a 100644 --- a/evals/lib/setup.ts +++ b/evals/lib/setup.ts @@ -81,9 +81,9 @@ export async function writeAgentsMd(sandbox: Sandbox): Promise { const body = ` -# Next.js: ALWAYS read docs before coding +# Next.js: ALWAYS search and read docs before coding -Before any Next.js work, find and read the relevant doc in \`node_modules/next/dist/docs/\`. Your training data is outdated — the docs are the source of truth. +Before any Next.js work, search \`node_modules/next/dist/docs/\` using terms from the task and source code, then read the most relevant guides and API references. Your training data is outdated — the docs are the source of truth. ` From 0fbb54244c26cd94a875ee1774dc1a042c0b2a68 Mon Sep 17 00:00:00 2001 From: Aurora Scharff Date: Mon, 31 Aug 2026 16:31:25 +0200 Subject: [PATCH 06/10] [evals] Add focused Cache Components cases --- .../EVAL.ts | 46 +++++++++++++++++++ .../PROMPT.md | 1 + .../app/layout.tsx | 16 +++++++ .../app/news/page.tsx | 19 ++++++++ .../app/page.tsx | 3 ++ .../app/preview/page.tsx | 14 ++++++ .../next-env.d.ts | 5 ++ .../next.config.ts | 10 ++++ .../package.json | 23 ++++++++++ .../tsconfig.json | 21 +++++++++ .../EVAL.ts | 38 +++++++++++++++ .../PROMPT.md | 1 + .../app/layout.tsx | 9 ++++ .../app/page.tsx | 21 +++++++++ .../lib/projects.ts | 4 ++ .../next-env.d.ts | 5 ++ .../next.config.ts | 5 ++ .../package.json | 23 ++++++++++ .../tsconfig.json | 21 +++++++++ .../EVAL.ts | 33 +++++++++++++ .../PROMPT.md | 1 + .../app/layout.tsx | 15 ++++++ .../app/page.tsx | 10 ++++ .../app/products/[slug]/page.tsx | 30 ++++++++++++ .../lib/products.ts | 17 +++++++ .../next-env.d.ts | 5 ++ .../next.config.ts | 5 ++ .../package.json | 23 ++++++++++ .../tsconfig.json | 21 +++++++++ .../EVAL.ts | 38 +++++++++++++++ .../PROMPT.md | 1 + .../app/layout.tsx | 14 ++++++ .../app/page.tsx | 8 ++++ .../components/request-metadata.tsx | 13 ++++++ .../next-env.d.ts | 5 ++ .../next.config.ts | 5 ++ .../package.json | 23 ++++++++++ .../tsconfig.json | 21 +++++++++ 38 files changed, 573 insertions(+) create mode 100644 evals/evals/agent-048-cache-components-config-semantics/EVAL.ts create mode 100644 evals/evals/agent-048-cache-components-config-semantics/PROMPT.md create mode 100644 evals/evals/agent-048-cache-components-config-semantics/app/layout.tsx create mode 100644 evals/evals/agent-048-cache-components-config-semantics/app/news/page.tsx create mode 100644 evals/evals/agent-048-cache-components-config-semantics/app/page.tsx create mode 100644 evals/evals/agent-048-cache-components-config-semantics/app/preview/page.tsx create mode 100644 evals/evals/agent-048-cache-components-config-semantics/next-env.d.ts create mode 100644 evals/evals/agent-048-cache-components-config-semantics/next.config.ts create mode 100644 evals/evals/agent-048-cache-components-config-semantics/package.json create mode 100644 evals/evals/agent-048-cache-components-config-semantics/tsconfig.json create mode 100644 evals/evals/agent-049-cache-components-session-isolation/EVAL.ts create mode 100644 evals/evals/agent-049-cache-components-session-isolation/PROMPT.md create mode 100644 evals/evals/agent-049-cache-components-session-isolation/app/layout.tsx create mode 100644 evals/evals/agent-049-cache-components-session-isolation/app/page.tsx create mode 100644 evals/evals/agent-049-cache-components-session-isolation/lib/projects.ts create mode 100644 evals/evals/agent-049-cache-components-session-isolation/next-env.d.ts create mode 100644 evals/evals/agent-049-cache-components-session-isolation/next.config.ts create mode 100644 evals/evals/agent-049-cache-components-session-isolation/package.json create mode 100644 evals/evals/agent-049-cache-components-session-isolation/tsconfig.json create mode 100644 evals/evals/agent-050-cache-components-url-shell/EVAL.ts create mode 100644 evals/evals/agent-050-cache-components-url-shell/PROMPT.md create mode 100644 evals/evals/agent-050-cache-components-url-shell/app/layout.tsx create mode 100644 evals/evals/agent-050-cache-components-url-shell/app/page.tsx create mode 100644 evals/evals/agent-050-cache-components-url-shell/app/products/[slug]/page.tsx create mode 100644 evals/evals/agent-050-cache-components-url-shell/lib/products.ts create mode 100644 evals/evals/agent-050-cache-components-url-shell/next-env.d.ts create mode 100644 evals/evals/agent-050-cache-components-url-shell/next.config.ts create mode 100644 evals/evals/agent-050-cache-components-url-shell/package.json create mode 100644 evals/evals/agent-050-cache-components-url-shell/tsconfig.json create mode 100644 evals/evals/agent-051-cache-components-sync-io/EVAL.ts create mode 100644 evals/evals/agent-051-cache-components-sync-io/PROMPT.md create mode 100644 evals/evals/agent-051-cache-components-sync-io/app/layout.tsx create mode 100644 evals/evals/agent-051-cache-components-sync-io/app/page.tsx create mode 100644 evals/evals/agent-051-cache-components-sync-io/components/request-metadata.tsx create mode 100644 evals/evals/agent-051-cache-components-sync-io/next-env.d.ts create mode 100644 evals/evals/agent-051-cache-components-sync-io/next.config.ts create mode 100644 evals/evals/agent-051-cache-components-sync-io/package.json create mode 100644 evals/evals/agent-051-cache-components-sync-io/tsconfig.json diff --git a/evals/evals/agent-048-cache-components-config-semantics/EVAL.ts b/evals/evals/agent-048-cache-components-config-semantics/EVAL.ts new file mode 100644 index 000000000000..27879531129b --- /dev/null +++ b/evals/evals/agent-048-cache-components-config-semantics/EVAL.ts @@ -0,0 +1,46 @@ +/** + * Cache Components: route config semantics + * + * Verifies that adoption translates legacy configuration instead of deleting + * behavior or silencing validation. + */ + +import { expect, test } from 'vitest' +import { existsSync, readFileSync, readdirSync, statSync } from 'fs' +import { join } from 'path' +import { environment } from '@vercel/agent-eval/eval' + +const IGNORE_DIRS = new Set(['.git', '.next', 'node_modules', 'dist', 'build']) + +function readSource(dir: string): string[] { + if (!existsSync(dir)) return [] + return readdirSync(dir).flatMap((entry) => { + if (IGNORE_DIRS.has(entry)) return [] + const path = join(dir, entry) + if (statSync(path).isDirectory()) return readSource(path) + if (entry === 'EVAL.ts' || !/\.(ts|tsx|js|jsx)$/.test(entry)) return [] + return readFileSync(path, 'utf8') + }) +} + +const source = readSource(process.cwd()).join('\n') +const config = readFileSync(join(process.cwd(), 'next.config.ts'), 'utf8') + +test('enables the current Cache Components config only', () => { + expect(config).toMatch(/cacheComponents\s*:\s*true/) + expect(config).not.toMatch(/dynamicIO\s*:/) + expect(config).not.toMatch(/useCache\s*:/) +}) + +test('removes route opt-outs and incompatible segment config', () => { + expect(source).not.toMatch(/export\s+(?:const|var|let)\s+instant\s*=\s*false/) + expect(source).not.toMatch( + /export\s+(?:const|var|let)\s+(?:dynamic|revalidate|fetchCache)\s*=/ + ) +}) + +test('preserves the hourly feed and request-specific preview', async () => { + await expect(environment).toSatisfyCriterion( + `The newsroom feed is cached with an explicit approximately-hourly cache lifetime. A focused data-level cache is valid, and a page-level cache is also valid because the Newsroom page contains only the feed and stable framing with the same lifetime. The editor preview still reads its cookie at request time outside any public cache, is placed below Suspense or equivalent meaningful loading UI, and leaves useful stable preview framing in the static shell.` + ) +}) diff --git a/evals/evals/agent-048-cache-components-config-semantics/PROMPT.md b/evals/evals/agent-048-cache-components-config-semantics/PROMPT.md new file mode 100644 index 000000000000..97820f8fcf1b --- /dev/null +++ b/evals/evals/agent-048-cache-components-config-semantics/PROMPT.md @@ -0,0 +1 @@ +Move this publishing app to Cache Components in one branch. Keep the newsroom feed on its hourly refresh policy and keep editor preview request-specific. Finish with a production build without leaving routes opted out. diff --git a/evals/evals/agent-048-cache-components-config-semantics/app/layout.tsx b/evals/evals/agent-048-cache-components-config-semantics/app/layout.tsx new file mode 100644 index 000000000000..91cd6a8db4a9 --- /dev/null +++ b/evals/evals/agent-048-cache-components-config-semantics/app/layout.tsx @@ -0,0 +1,16 @@ +import Link from 'next/link' +import type { ReactNode } from 'react' + +export default function RootLayout({ children }: { children: ReactNode }) { + return ( + + + + {children} + + + ) +} diff --git a/evals/evals/agent-048-cache-components-config-semantics/app/news/page.tsx b/evals/evals/agent-048-cache-components-config-semantics/app/news/page.tsx new file mode 100644 index 000000000000..ef8ce563d17c --- /dev/null +++ b/evals/evals/agent-048-cache-components-config-semantics/app/news/page.tsx @@ -0,0 +1,19 @@ +export const revalidate = 3600 + +async function getStories() { + await new Promise((resolve) => setTimeout(resolve, 80)) + return ['City council approves new park', 'Night market opens Friday'] +} + +export default async function NewsPage() { + const stories = await getStories() + + return ( +
+

Newsroom

+ {stories.map((story) => ( +
{story}
+ ))} +
+ ) +} diff --git a/evals/evals/agent-048-cache-components-config-semantics/app/page.tsx b/evals/evals/agent-048-cache-components-config-semantics/app/page.tsx new file mode 100644 index 000000000000..89fbd7e29342 --- /dev/null +++ b/evals/evals/agent-048-cache-components-config-semantics/app/page.tsx @@ -0,0 +1,3 @@ +export default function HomePage() { + return

Northstar Publishing

+} diff --git a/evals/evals/agent-048-cache-components-config-semantics/app/preview/page.tsx b/evals/evals/agent-048-cache-components-config-semantics/app/preview/page.tsx new file mode 100644 index 000000000000..6644b603487e --- /dev/null +++ b/evals/evals/agent-048-cache-components-config-semantics/app/preview/page.tsx @@ -0,0 +1,14 @@ +import { cookies } from 'next/headers' + +export const dynamic = 'force-dynamic' + +export default async function PreviewPage() { + const draft = (await cookies()).get('draft-title')?.value ?? 'Untitled draft' + + return ( +
+

Editor preview

+

{draft}

+
+ ) +} diff --git a/evals/evals/agent-048-cache-components-config-semantics/next-env.d.ts b/evals/evals/agent-048-cache-components-config-semantics/next-env.d.ts new file mode 100644 index 000000000000..1b3be0840f3f --- /dev/null +++ b/evals/evals/agent-048-cache-components-config-semantics/next-env.d.ts @@ -0,0 +1,5 @@ +/// +/// + +// NOTE: This file should not be edited +// see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/evals/evals/agent-048-cache-components-config-semantics/next.config.ts b/evals/evals/agent-048-cache-components-config-semantics/next.config.ts new file mode 100644 index 000000000000..5c906c55af96 --- /dev/null +++ b/evals/evals/agent-048-cache-components-config-semantics/next.config.ts @@ -0,0 +1,10 @@ +import type { NextConfig } from 'next' + +const nextConfig: NextConfig = { + experimental: { + // @ts-expect-error -- legacy config retained for the migration exercise + dynamicIO: true, + }, +} + +export default nextConfig diff --git a/evals/evals/agent-048-cache-components-config-semantics/package.json b/evals/evals/agent-048-cache-components-config-semantics/package.json new file mode 100644 index 000000000000..7056dd9ed5f9 --- /dev/null +++ b/evals/evals/agent-048-cache-components-config-semantics/package.json @@ -0,0 +1,23 @@ +{ + "private": true, + "type": "module", + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start" + }, + "dependencies": { + "next": "^16", + "react": "19.1.0", + "react-dom": "19.1.0" + }, + "devDependencies": { + "@types/node": "^20", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^5", + "vitest": "^3.1.3", + "@vitejs/plugin-react": "^4.4.1", + "vite-tsconfig-paths": "^5.1.4" + } +} diff --git a/evals/evals/agent-048-cache-components-config-semantics/tsconfig.json b/evals/evals/agent-048-cache-components-config-semantics/tsconfig.json new file mode 100644 index 000000000000..e32fb699a1b0 --- /dev/null +++ b/evals/evals/agent-048-cache-components-config-semantics/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "ES2017", + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "react-jsx", + "incremental": true, + "plugins": [{ "name": "next" }], + "paths": { "@/*": ["./*"] } + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], + "exclude": ["node_modules", "EVAL.ts"] +} diff --git a/evals/evals/agent-049-cache-components-session-isolation/EVAL.ts b/evals/evals/agent-049-cache-components-session-isolation/EVAL.ts new file mode 100644 index 000000000000..c587888703cb --- /dev/null +++ b/evals/evals/agent-049-cache-components-session-isolation/EVAL.ts @@ -0,0 +1,38 @@ +/** Cache Components: request isolation and reusable data. */ + +import { expect, test } from 'vitest' +import { existsSync, readFileSync, readdirSync, statSync } from 'fs' +import { join } from 'path' +import { environment } from '@vercel/agent-eval/eval' + +const IGNORE_DIRS = new Set(['.git', '.next', 'node_modules', 'dist', 'build']) + +function readSource(dir: string): string[] { + if (!existsSync(dir)) return [] + return readdirSync(dir).flatMap((entry) => { + if (IGNORE_DIRS.has(entry)) return [] + const path = join(dir, entry) + if (statSync(path).isDirectory()) return readSource(path) + if (entry === 'EVAL.ts' || !/\.(ts|tsx|js|jsx)$/.test(entry)) return [] + return readFileSync(path, 'utf8') + }) +} + +const source = readSource(process.cwd()).join('\n') +const config = readFileSync(join(process.cwd(), 'next.config.ts'), 'utf8') + +test('enables Cache Components without opt-outs', () => { + expect(config).toMatch(/cacheComponents\s*:\s*true/) + expect(source).not.toMatch(/export\s+(?:const|var|let)\s+instant\s*=\s*false/) +}) + +test('introduces a bounded reusable cache', () => { + expect(source).toMatch(/['"]use cache['"]/) + expect(source).toMatch(/\bcacheLife\s*\(/) +}) + +test('keeps member data isolated and the shell meaningful', async () => { + await expect(environment).toSatisfyCriterion( + `The dashboard keeps a useful stable frame such as the Workspace heading outside a meaningful Suspense boundary. Member name, team cookie, and request-region header are read at request time in the suspended subtree and are not captured by a public cache. Shared project totals are cached in a small data function with an explicit lifetime. No page or broad request-specific component is publicly cached, so values cannot leak between members.` + ) +}) diff --git a/evals/evals/agent-049-cache-components-session-isolation/PROMPT.md b/evals/evals/agent-049-cache-components-session-isolation/PROMPT.md new file mode 100644 index 000000000000..a988a3062e4b --- /dev/null +++ b/evals/evals/agent-049-cache-components-session-isolation/PROMPT.md @@ -0,0 +1 @@ +Enable Cache Components for this workspace dashboard. Members must continue seeing their own name, team, and request region, while the shared project totals stay reusable. Keep a useful dashboard frame visible while request-specific content loads, and finish with a production build without route opt-outs. diff --git a/evals/evals/agent-049-cache-components-session-isolation/app/layout.tsx b/evals/evals/agent-049-cache-components-session-isolation/app/layout.tsx new file mode 100644 index 000000000000..07e2db241f59 --- /dev/null +++ b/evals/evals/agent-049-cache-components-session-isolation/app/layout.tsx @@ -0,0 +1,9 @@ +import type { ReactNode } from 'react' + +export default function RootLayout({ children }: { children: ReactNode }) { + return ( + + {children} + + ) +} diff --git a/evals/evals/agent-049-cache-components-session-isolation/app/page.tsx b/evals/evals/agent-049-cache-components-session-isolation/app/page.tsx new file mode 100644 index 000000000000..81dd311c1c8d --- /dev/null +++ b/evals/evals/agent-049-cache-components-session-isolation/app/page.tsx @@ -0,0 +1,21 @@ +import { cookies, headers } from 'next/headers' +import { getProjectTotals } from '@/lib/projects' + +export default async function DashboardPage() { + const cookieStore = await cookies() + const requestHeaders = await headers() + const member = cookieStore.get('member')?.value ?? 'Guest' + const team = cookieStore.get('team')?.value ?? 'Community' + const region = requestHeaders.get('x-region') ?? 'global' + const totals = await getProjectTotals() + + return ( +
+

Workspace

+

Welcome, {member}

+

Team: {team}

+

Region: {region}

+

{totals.active} active projects

+
+ ) +} diff --git a/evals/evals/agent-049-cache-components-session-isolation/lib/projects.ts b/evals/evals/agent-049-cache-components-session-isolation/lib/projects.ts new file mode 100644 index 000000000000..437d7ce00624 --- /dev/null +++ b/evals/evals/agent-049-cache-components-session-isolation/lib/projects.ts @@ -0,0 +1,4 @@ +export async function getProjectTotals() { + await new Promise((resolve) => setTimeout(resolve, 120)) + return { active: 12, archived: 4 } +} diff --git a/evals/evals/agent-049-cache-components-session-isolation/next-env.d.ts b/evals/evals/agent-049-cache-components-session-isolation/next-env.d.ts new file mode 100644 index 000000000000..1b3be0840f3f --- /dev/null +++ b/evals/evals/agent-049-cache-components-session-isolation/next-env.d.ts @@ -0,0 +1,5 @@ +/// +/// + +// NOTE: This file should not be edited +// see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/evals/evals/agent-049-cache-components-session-isolation/next.config.ts b/evals/evals/agent-049-cache-components-session-isolation/next.config.ts new file mode 100644 index 000000000000..e4f5738a310b --- /dev/null +++ b/evals/evals/agent-049-cache-components-session-isolation/next.config.ts @@ -0,0 +1,5 @@ +import type { NextConfig } from 'next' + +const nextConfig: NextConfig = {} + +export default nextConfig diff --git a/evals/evals/agent-049-cache-components-session-isolation/package.json b/evals/evals/agent-049-cache-components-session-isolation/package.json new file mode 100644 index 000000000000..7056dd9ed5f9 --- /dev/null +++ b/evals/evals/agent-049-cache-components-session-isolation/package.json @@ -0,0 +1,23 @@ +{ + "private": true, + "type": "module", + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start" + }, + "dependencies": { + "next": "^16", + "react": "19.1.0", + "react-dom": "19.1.0" + }, + "devDependencies": { + "@types/node": "^20", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^5", + "vitest": "^3.1.3", + "@vitejs/plugin-react": "^4.4.1", + "vite-tsconfig-paths": "^5.1.4" + } +} diff --git a/evals/evals/agent-049-cache-components-session-isolation/tsconfig.json b/evals/evals/agent-049-cache-components-session-isolation/tsconfig.json new file mode 100644 index 000000000000..e32fb699a1b0 --- /dev/null +++ b/evals/evals/agent-049-cache-components-session-isolation/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "ES2017", + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "react-jsx", + "incremental": true, + "plugins": [{ "name": "next" }], + "paths": { "@/*": ["./*"] } + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], + "exclude": ["node_modules", "EVAL.ts"] +} diff --git a/evals/evals/agent-050-cache-components-url-shell/EVAL.ts b/evals/evals/agent-050-cache-components-url-shell/EVAL.ts new file mode 100644 index 000000000000..0ec0cd628412 --- /dev/null +++ b/evals/evals/agent-050-cache-components-url-shell/EVAL.ts @@ -0,0 +1,33 @@ +/** Cache Components: URL data and shell boundaries. */ + +import { expect, test } from 'vitest' +import { existsSync, readFileSync, readdirSync, statSync } from 'fs' +import { join } from 'path' +import { environment } from '@vercel/agent-eval/eval' + +const IGNORE_DIRS = new Set(['.git', '.next', 'node_modules', 'dist', 'build']) + +function readSource(dir: string): string[] { + if (!existsSync(dir)) return [] + return readdirSync(dir).flatMap((entry) => { + if (IGNORE_DIRS.has(entry)) return [] + const path = join(dir, entry) + if (statSync(path).isDirectory()) return readSource(path) + if (entry === 'EVAL.ts' || !/\.(ts|tsx|js|jsx)$/.test(entry)) return [] + return readFileSync(path, 'utf8') + }) +} + +const source = readSource(process.cwd()).join('\n') +const config = readFileSync(join(process.cwd(), 'next.config.ts'), 'utf8') + +test('enables Cache Components without opt-outs', () => { + expect(config).toMatch(/cacheComponents\s*:\s*true/) + expect(source).not.toMatch(/export\s+(?:const|var|let)\s+instant\s*=\s*false/) +}) + +test('keeps URL-specific rendering below useful boundaries', async () => { + await expect(environment).toSatisfyCriterion( + `The product page returns useful route-independent shop framing before awaiting params or searchParams. The URL promises are forwarded into one or more children below meaningful Suspense fallbacks, where the slug and currency are resolved. Product lookup is cached in a small data function with every value that changes its output included in the cache key. Related products remain independently suspended and may stream. The solution does not hide the entire page behind one null or generic boundary.` + ) +}) diff --git a/evals/evals/agent-050-cache-components-url-shell/PROMPT.md b/evals/evals/agent-050-cache-components-url-shell/PROMPT.md new file mode 100644 index 000000000000..62c3e1507e07 --- /dev/null +++ b/evals/evals/agent-050-cache-components-url-shell/PROMPT.md @@ -0,0 +1 @@ +Enable Cache Components for this shop. Product URLs and their currency query must keep rendering the correct details, and product details should stay reusable across requests. Keep the shop frame useful while data loads, and let related products stream afterward. Finish with a production build without route opt-outs. diff --git a/evals/evals/agent-050-cache-components-url-shell/app/layout.tsx b/evals/evals/agent-050-cache-components-url-shell/app/layout.tsx new file mode 100644 index 000000000000..20810c070986 --- /dev/null +++ b/evals/evals/agent-050-cache-components-url-shell/app/layout.tsx @@ -0,0 +1,15 @@ +import Link from 'next/link' +import type { ReactNode } from 'react' + +export default function RootLayout({ children }: { children: ReactNode }) { + return ( + + + + {children} + + + ) +} diff --git a/evals/evals/agent-050-cache-components-url-shell/app/page.tsx b/evals/evals/agent-050-cache-components-url-shell/app/page.tsx new file mode 100644 index 000000000000..442c83196608 --- /dev/null +++ b/evals/evals/agent-050-cache-components-url-shell/app/page.tsx @@ -0,0 +1,10 @@ +import Link from 'next/link' + +export default function HomePage() { + return ( +
+

Signal Shop

+ Brass lamp +
+ ) +} diff --git a/evals/evals/agent-050-cache-components-url-shell/app/products/[slug]/page.tsx b/evals/evals/agent-050-cache-components-url-shell/app/products/[slug]/page.tsx new file mode 100644 index 000000000000..71ec832085fb --- /dev/null +++ b/evals/evals/agent-050-cache-components-url-shell/app/products/[slug]/page.tsx @@ -0,0 +1,30 @@ +import { notFound } from 'next/navigation' +import { getProduct, getRelatedProducts } from '@/lib/products' + +export default async function ProductPage({ + params, + searchParams, +}: { + params: Promise<{ slug: string }> + searchParams: Promise<{ currency?: string }> +}) { + const { slug } = await params + const { currency = 'USD' } = await searchParams + const product = await getProduct(slug, currency) + if (!product) notFound() + const related = await getRelatedProducts(slug) + + return ( +
+

Signal Shop collection

+

{product.name}

+

{product.price}

+
+

Related products

+ {related.map((item) => ( +

{item.name}

+ ))} +
+
+ ) +} diff --git a/evals/evals/agent-050-cache-components-url-shell/lib/products.ts b/evals/evals/agent-050-cache-components-url-shell/lib/products.ts new file mode 100644 index 000000000000..a63b6b77ab3b --- /dev/null +++ b/evals/evals/agent-050-cache-components-url-shell/lib/products.ts @@ -0,0 +1,17 @@ +const products = { + lamp: { slug: 'lamp', name: 'Brass lamp', usd: 120 }, + vase: { slug: 'vase', name: 'Stone vase', usd: 80 }, +} + +export async function getProduct(slug: string, currency: string) { + await new Promise((resolve) => setTimeout(resolve, 120)) + const product = products[slug as keyof typeof products] + if (!product) return null + const rate = currency === 'EUR' ? 0.92 : 1 + return { ...product, price: `${currency} ${(product.usd * rate).toFixed(2)}` } +} + +export async function getRelatedProducts(slug: string) { + await new Promise((resolve) => setTimeout(resolve, 420)) + return Object.values(products).filter((product) => product.slug !== slug) +} diff --git a/evals/evals/agent-050-cache-components-url-shell/next-env.d.ts b/evals/evals/agent-050-cache-components-url-shell/next-env.d.ts new file mode 100644 index 000000000000..1b3be0840f3f --- /dev/null +++ b/evals/evals/agent-050-cache-components-url-shell/next-env.d.ts @@ -0,0 +1,5 @@ +/// +/// + +// NOTE: This file should not be edited +// see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/evals/evals/agent-050-cache-components-url-shell/next.config.ts b/evals/evals/agent-050-cache-components-url-shell/next.config.ts new file mode 100644 index 000000000000..e4f5738a310b --- /dev/null +++ b/evals/evals/agent-050-cache-components-url-shell/next.config.ts @@ -0,0 +1,5 @@ +import type { NextConfig } from 'next' + +const nextConfig: NextConfig = {} + +export default nextConfig diff --git a/evals/evals/agent-050-cache-components-url-shell/package.json b/evals/evals/agent-050-cache-components-url-shell/package.json new file mode 100644 index 000000000000..7056dd9ed5f9 --- /dev/null +++ b/evals/evals/agent-050-cache-components-url-shell/package.json @@ -0,0 +1,23 @@ +{ + "private": true, + "type": "module", + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start" + }, + "dependencies": { + "next": "^16", + "react": "19.1.0", + "react-dom": "19.1.0" + }, + "devDependencies": { + "@types/node": "^20", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^5", + "vitest": "^3.1.3", + "@vitejs/plugin-react": "^4.4.1", + "vite-tsconfig-paths": "^5.1.4" + } +} diff --git a/evals/evals/agent-050-cache-components-url-shell/tsconfig.json b/evals/evals/agent-050-cache-components-url-shell/tsconfig.json new file mode 100644 index 000000000000..e32fb699a1b0 --- /dev/null +++ b/evals/evals/agent-050-cache-components-url-shell/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "ES2017", + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "react-jsx", + "incremental": true, + "plugins": [{ "name": "next" }], + "paths": { "@/*": ["./*"] } + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], + "exclude": ["node_modules", "EVAL.ts"] +} diff --git a/evals/evals/agent-051-cache-components-sync-io/EVAL.ts b/evals/evals/agent-051-cache-components-sync-io/EVAL.ts new file mode 100644 index 000000000000..45388a3d8e5a --- /dev/null +++ b/evals/evals/agent-051-cache-components-sync-io/EVAL.ts @@ -0,0 +1,38 @@ +/** Cache Components: synchronous request-time I/O in shared UI. */ + +import { expect, test } from 'vitest' +import { existsSync, readFileSync, readdirSync, statSync } from 'fs' +import { join } from 'path' +import { environment } from '@vercel/agent-eval/eval' + +const IGNORE_DIRS = new Set(['.git', '.next', 'node_modules', 'dist', 'build']) + +function readSource(dir: string): string[] { + if (!existsSync(dir)) return [] + return readdirSync(dir).flatMap((entry) => { + if (IGNORE_DIRS.has(entry)) return [] + const path = join(dir, entry) + if (statSync(path).isDirectory()) return readSource(path) + if (entry === 'EVAL.ts' || !/\.(ts|tsx|js|jsx)$/.test(entry)) return [] + return readFileSync(path, 'utf8') + }) +} + +const source = readSource(process.cwd()).join('\n') +const config = readFileSync(join(process.cwd(), 'next.config.ts'), 'utf8') + +test('enables Cache Components without opt-outs', () => { + expect(config).toMatch(/cacheComponents\s*:\s*true/) + expect(source).not.toMatch(/export\s+(?:const|var|let)\s+instant\s*=\s*false/) +}) + +test('uses an explicit synchronous I/O boundary', () => { + expect(source).toMatch(/\b(?:io|connection)\s*\(/) + expect(source).toMatch(/]/) +}) + +test('preserves fresh metadata without sacrificing the shared frame', async () => { + await expect(environment).toSatisfyCriterion( + `The stable Operations navigation and page frame remain outside the Suspense boundary. The request metadata is rendered in a focused async child that awaits io() or connection() before calling new Date or Date.now, Math.random, and randomUUID, so those values remain request-specific. The metadata subtree is not cached or evaluated at module scope, and it has meaningful loading UI. Prefer io() unless the implementation specifically needs to wait for a real user request.` + ) +}) diff --git a/evals/evals/agent-051-cache-components-sync-io/PROMPT.md b/evals/evals/agent-051-cache-components-sync-io/PROMPT.md new file mode 100644 index 000000000000..bfb437bc2894 --- /dev/null +++ b/evals/evals/agent-051-cache-components-sync-io/PROMPT.md @@ -0,0 +1 @@ +Enable Cache Components for this operations console. The navigation must stay immediately useful, while the request identifier, current time, and sampling value must still be generated per request. Finish with a production build and do not leave routes opted out. diff --git a/evals/evals/agent-051-cache-components-sync-io/app/layout.tsx b/evals/evals/agent-051-cache-components-sync-io/app/layout.tsx new file mode 100644 index 000000000000..03a7ce6ef914 --- /dev/null +++ b/evals/evals/agent-051-cache-components-sync-io/app/layout.tsx @@ -0,0 +1,14 @@ +import type { ReactNode } from 'react' +import { RequestMetadata } from '@/components/request-metadata' + +export default function RootLayout({ children }: { children: ReactNode }) { + return ( + + + + + {children} + + + ) +} diff --git a/evals/evals/agent-051-cache-components-sync-io/app/page.tsx b/evals/evals/agent-051-cache-components-sync-io/app/page.tsx new file mode 100644 index 000000000000..20adbd86f3fd --- /dev/null +++ b/evals/evals/agent-051-cache-components-sync-io/app/page.tsx @@ -0,0 +1,8 @@ +export default function HomePage() { + return ( +
+

Deployment health

+

All systems operational

+
+ ) +} diff --git a/evals/evals/agent-051-cache-components-sync-io/components/request-metadata.tsx b/evals/evals/agent-051-cache-components-sync-io/components/request-metadata.tsx new file mode 100644 index 000000000000..6c191a0f78da --- /dev/null +++ b/evals/evals/agent-051-cache-components-sync-io/components/request-metadata.tsx @@ -0,0 +1,13 @@ +import { randomUUID } from 'node:crypto' + +export function RequestMetadata() { + const generatedAt = new Date().toISOString() + const sample = Math.random() + const requestId = randomUUID() + + return ( + + ) +} diff --git a/evals/evals/agent-051-cache-components-sync-io/next-env.d.ts b/evals/evals/agent-051-cache-components-sync-io/next-env.d.ts new file mode 100644 index 000000000000..1b3be0840f3f --- /dev/null +++ b/evals/evals/agent-051-cache-components-sync-io/next-env.d.ts @@ -0,0 +1,5 @@ +/// +/// + +// NOTE: This file should not be edited +// see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/evals/evals/agent-051-cache-components-sync-io/next.config.ts b/evals/evals/agent-051-cache-components-sync-io/next.config.ts new file mode 100644 index 000000000000..e4f5738a310b --- /dev/null +++ b/evals/evals/agent-051-cache-components-sync-io/next.config.ts @@ -0,0 +1,5 @@ +import type { NextConfig } from 'next' + +const nextConfig: NextConfig = {} + +export default nextConfig diff --git a/evals/evals/agent-051-cache-components-sync-io/package.json b/evals/evals/agent-051-cache-components-sync-io/package.json new file mode 100644 index 000000000000..7056dd9ed5f9 --- /dev/null +++ b/evals/evals/agent-051-cache-components-sync-io/package.json @@ -0,0 +1,23 @@ +{ + "private": true, + "type": "module", + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start" + }, + "dependencies": { + "next": "^16", + "react": "19.1.0", + "react-dom": "19.1.0" + }, + "devDependencies": { + "@types/node": "^20", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^5", + "vitest": "^3.1.3", + "@vitejs/plugin-react": "^4.4.1", + "vite-tsconfig-paths": "^5.1.4" + } +} diff --git a/evals/evals/agent-051-cache-components-sync-io/tsconfig.json b/evals/evals/agent-051-cache-components-sync-io/tsconfig.json new file mode 100644 index 000000000000..e32fb699a1b0 --- /dev/null +++ b/evals/evals/agent-051-cache-components-sync-io/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "ES2017", + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "react-jsx", + "incremental": true, + "plugins": [{ "name": "next" }], + "paths": { "@/*": ["./*"] } + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], + "exclude": ["node_modules", "EVAL.ts"] +} From 21288e4fc00060bf57308ac2b620f87a72583112 Mon Sep 17 00:00:00 2001 From: Aurora Scharff Date: Mon, 31 Aug 2026 17:05:20 +0200 Subject: [PATCH 07/10] [evals] Consolidate overlapping adoption cases --- .../agent-035-connection-dynamic/EVAL.ts | 92 ++++++------------- .../agent-035-connection-dynamic/PROMPT.md | 2 +- .../app/layout.tsx | 20 ++-- .../agent-035-connection-dynamic/app/page.tsx | 6 +- .../components/request-metadata.tsx | 0 .../tsconfig.json | 2 +- evals/evals/agent-042-enable-ppr/EVAL.ts | 51 ++++++---- evals/evals/agent-042-enable-ppr/PROMPT.md | 2 +- .../agent-042-enable-ppr/app/ProductList.tsx | 18 ---- .../app/Recommendations.tsx | 19 ---- .../evals/agent-042-enable-ppr/app/layout.tsx | 22 ++--- .../app/news/page.tsx | 0 evals/evals/agent-042-enable-ppr/app/page.tsx | 18 +--- .../app/preview/page.tsx | 0 .../evals/agent-042-enable-ppr/next.config.ts | 7 +- .../evals/agent-042-enable-ppr/tsconfig.json | 2 +- .../EVAL.ts | 46 ---------- .../PROMPT.md | 1 - .../app/layout.tsx | 16 ---- .../app/page.tsx | 3 - .../next-env.d.ts | 5 - .../next.config.ts | 10 -- .../package.json | 23 ----- .../tsconfig.json | 21 ----- .../EVAL.ts | 38 -------- .../PROMPT.md | 1 - .../app/layout.tsx | 14 --- .../app/page.tsx | 8 -- .../next-env.d.ts | 5 - .../next.config.ts | 5 - .../package.json | 23 ----- .../tsconfig.json | 21 ----- 32 files changed, 93 insertions(+), 408 deletions(-) rename evals/evals/{agent-051-cache-components-sync-io => agent-035-connection-dynamic}/components/request-metadata.tsx (100%) delete mode 100644 evals/evals/agent-042-enable-ppr/app/ProductList.tsx delete mode 100644 evals/evals/agent-042-enable-ppr/app/Recommendations.tsx rename evals/evals/{agent-048-cache-components-config-semantics => agent-042-enable-ppr}/app/news/page.tsx (100%) rename evals/evals/{agent-048-cache-components-config-semantics => agent-042-enable-ppr}/app/preview/page.tsx (100%) delete mode 100644 evals/evals/agent-048-cache-components-config-semantics/EVAL.ts delete mode 100644 evals/evals/agent-048-cache-components-config-semantics/PROMPT.md delete mode 100644 evals/evals/agent-048-cache-components-config-semantics/app/layout.tsx delete mode 100644 evals/evals/agent-048-cache-components-config-semantics/app/page.tsx delete mode 100644 evals/evals/agent-048-cache-components-config-semantics/next-env.d.ts delete mode 100644 evals/evals/agent-048-cache-components-config-semantics/next.config.ts delete mode 100644 evals/evals/agent-048-cache-components-config-semantics/package.json delete mode 100644 evals/evals/agent-048-cache-components-config-semantics/tsconfig.json delete mode 100644 evals/evals/agent-051-cache-components-sync-io/EVAL.ts delete mode 100644 evals/evals/agent-051-cache-components-sync-io/PROMPT.md delete mode 100644 evals/evals/agent-051-cache-components-sync-io/app/layout.tsx delete mode 100644 evals/evals/agent-051-cache-components-sync-io/app/page.tsx delete mode 100644 evals/evals/agent-051-cache-components-sync-io/next-env.d.ts delete mode 100644 evals/evals/agent-051-cache-components-sync-io/next.config.ts delete mode 100644 evals/evals/agent-051-cache-components-sync-io/package.json delete mode 100644 evals/evals/agent-051-cache-components-sync-io/tsconfig.json diff --git a/evals/evals/agent-035-connection-dynamic/EVAL.ts b/evals/evals/agent-035-connection-dynamic/EVAL.ts index f1e42e6c9903..45388a3d8e5a 100644 --- a/evals/evals/agent-035-connection-dynamic/EVAL.ts +++ b/evals/evals/agent-035-connection-dynamic/EVAL.ts @@ -1,76 +1,38 @@ -/** - * connection() for Dynamic Rendering - * - * Tests whether the agent uses connection() from next/server to force dynamic - * rendering instead of the deprecated unstable_noStore(). - * - * Tricky because agents use unstable_noStore(), force-dynamic segment config, - * or unrelated Dynamic APIs instead of the stable connection() function. - */ +/** Cache Components: synchronous request-time I/O in shared UI. */ import { expect, test } from 'vitest' -import { readFileSync, existsSync, readdirSync } from 'fs' +import { existsSync, readFileSync, readdirSync, statSync } from 'fs' import { join } from 'path' - -function readAppFiles(): string { - const appDir = join(process.cwd(), 'app') - if (!existsSync(appDir)) return '' - const entries = readdirSync(appDir, { recursive: true }) as string[] - const files = entries.filter((f) => f.endsWith('.tsx') || f.endsWith('.ts')) - return files.map((f) => readFileSync(join(appDir, f), 'utf-8')).join('\n') +import { environment } from '@vercel/agent-eval/eval' + +const IGNORE_DIRS = new Set(['.git', '.next', 'node_modules', 'dist', 'build']) + +function readSource(dir: string): string[] { + if (!existsSync(dir)) return [] + return readdirSync(dir).flatMap((entry) => { + if (IGNORE_DIRS.has(entry)) return [] + const path = join(dir, entry) + if (statSync(path).isDirectory()) return readSource(path) + if (entry === 'EVAL.ts' || !/\.(ts|tsx|js|jsx)$/.test(entry)) return [] + return readFileSync(path, 'utf8') + }) } -test('Component imports connection from next/server', () => { - const content = readAppFiles() +const source = readSource(process.cwd()).join('\n') +const config = readFileSync(join(process.cwd(), 'next.config.ts'), 'utf8') - // Should import connection from next/server - expect(content).toMatch(/import.*connection.*from\s+['"]next\/server['"]/) +test('enables Cache Components without opt-outs', () => { + expect(config).toMatch(/cacheComponents\s*:\s*true/) + expect(source).not.toMatch(/export\s+(?:const|var|let)\s+instant\s*=\s*false/) }) -test('Component uses await connection()', () => { - const content = readAppFiles() - - // connection() returns a Promise and must be awaited - expect(content).toMatch(/await\s+connection\s*\(\s*\)/) +test('uses an explicit synchronous I/O boundary', () => { + expect(source).toMatch(/\b(?:io|connection)\s*\(/) + expect(source).toMatch(/]/) }) -test('Component is async', () => { - const content = readAppFiles() - - // Component must be async to use await - expect(content).toMatch(/async\s+function|export\s+default\s+async/) -}) - -test('Component uses Date for timestamp', () => { - const content = readAppFiles() - - // Should use Date for generating timestamp - expect(content).toMatch(/new\s+Date\s*\(|Date\.now\s*\(/) -}) - -test('Does NOT use unstable_noStore (deprecated)', () => { - const content = readAppFiles() - - // Should NOT use deprecated unstable_noStore - expect(content).not.toMatch(/unstable_noStore/) -}) - -test('Does NOT use force-dynamic segment config as primary approach', () => { - const content = readAppFiles() - - // Should use connection() as the primary approach instead of segment config - const hasForceDynamic = - content.includes("dynamic = 'force-dynamic'") || - content.includes('dynamic = "force-dynamic"') - const hasConnection = /connection\s*\(\s*\)/.test(content) - - // Enforce that connection() is the primary dynamic approach - // If force-dynamic is used, it cannot be the only dynamic mechanism - // When connection() is available/used, force-dynamic should not be redundantly added - if (hasForceDynamic && !hasConnection) { - // force-dynamic is being used as the sole dynamic mechanism (wrong) - expect.fail( - 'force-dynamic segment config should not be the primary approach - use connection() instead' - ) - } +test('preserves fresh metadata without sacrificing the shared frame', async () => { + await expect(environment).toSatisfyCriterion( + `The stable Operations navigation and page frame remain outside the Suspense boundary. The request metadata is rendered in a focused async child that awaits io() or connection() before calling new Date or Date.now, Math.random, and randomUUID, so those values remain request-specific. The metadata subtree is not cached or evaluated at module scope, and it has meaningful loading UI. Prefer io() unless the implementation specifically needs to wait for a real user request.` + ) }) diff --git a/evals/evals/agent-035-connection-dynamic/PROMPT.md b/evals/evals/agent-035-connection-dynamic/PROMPT.md index 70b64e401bda..bfb437bc2894 100644 --- a/evals/evals/agent-035-connection-dynamic/PROMPT.md +++ b/evals/evals/agent-035-connection-dynamic/PROMPT.md @@ -1 +1 @@ -Create a server component that displays the current server timestamp on each request. The timestamp should be different on every page load (not prerendered at build time). Use Next.js's recommended approach for opting out of static prerendering when you need dynamic data without using cookies/headers. +Enable Cache Components for this operations console. The navigation must stay immediately useful, while the request identifier, current time, and sampling value must still be generated per request. Finish with a production build and do not leave routes opted out. diff --git a/evals/evals/agent-035-connection-dynamic/app/layout.tsx b/evals/evals/agent-035-connection-dynamic/app/layout.tsx index 23429283cae0..03a7ce6ef914 100644 --- a/evals/evals/agent-035-connection-dynamic/app/layout.tsx +++ b/evals/evals/agent-035-connection-dynamic/app/layout.tsx @@ -1,18 +1,14 @@ -import type { Metadata } from 'next' +import type { ReactNode } from 'react' +import { RequestMetadata } from '@/components/request-metadata' -export const metadata: Metadata = { - title: 'Random Number', - description: 'Display a random number on each request', -} - -export default function RootLayout({ - children, -}: Readonly<{ - children: React.ReactNode -}>) { +export default function RootLayout({ children }: { children: ReactNode }) { return ( - {children} + + + + {children} + ) } diff --git a/evals/evals/agent-035-connection-dynamic/app/page.tsx b/evals/evals/agent-035-connection-dynamic/app/page.tsx index bc360b667d65..20adbd86f3fd 100644 --- a/evals/evals/agent-035-connection-dynamic/app/page.tsx +++ b/evals/evals/agent-035-connection-dynamic/app/page.tsx @@ -1,8 +1,8 @@ -export default function Home() { +export default function HomePage() { return (
-

Random Number Generator

-

Please implement a random number that changes on each request.

+

Deployment health

+

All systems operational

) } diff --git a/evals/evals/agent-051-cache-components-sync-io/components/request-metadata.tsx b/evals/evals/agent-035-connection-dynamic/components/request-metadata.tsx similarity index 100% rename from evals/evals/agent-051-cache-components-sync-io/components/request-metadata.tsx rename to evals/evals/agent-035-connection-dynamic/components/request-metadata.tsx diff --git a/evals/evals/agent-035-connection-dynamic/tsconfig.json b/evals/evals/agent-035-connection-dynamic/tsconfig.json index 64c21044c49f..8eb9f7f78971 100644 --- a/evals/evals/agent-035-connection-dynamic/tsconfig.json +++ b/evals/evals/agent-035-connection-dynamic/tsconfig.json @@ -23,5 +23,5 @@ "target": "ES2017" }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], - "exclude": ["node_modules"] + "exclude": ["node_modules", "EVAL.ts"] } diff --git a/evals/evals/agent-042-enable-ppr/EVAL.ts b/evals/evals/agent-042-enable-ppr/EVAL.ts index 20606268172d..27879531129b 100644 --- a/evals/evals/agent-042-enable-ppr/EVAL.ts +++ b/evals/evals/agent-042-enable-ppr/EVAL.ts @@ -1,31 +1,46 @@ /** - * Enable PPR + * Cache Components: route config semantics * - * Tests whether the agent knows that partial pre-rendering is enabled via - * `cacheComponents: true` in next.config.ts, NOT the old - * `experimental: { ppr: true }` flag. - * - * Tricky because most training data and older docs reference the experimental - * flag. The current way to enable PPR in Next.js 16 is `cacheComponents: true`. + * Verifies that adoption translates legacy configuration instead of deleting + * behavior or silencing validation. */ import { expect, test } from 'vitest' -import { readFileSync } from 'fs' +import { existsSync, readFileSync, readdirSync, statSync } from 'fs' import { join } from 'path' +import { environment } from '@vercel/agent-eval/eval' + +const IGNORE_DIRS = new Set(['.git', '.next', 'node_modules', 'dist', 'build']) + +function readSource(dir: string): string[] { + if (!existsSync(dir)) return [] + return readdirSync(dir).flatMap((entry) => { + if (IGNORE_DIRS.has(entry)) return [] + const path = join(dir, entry) + if (statSync(path).isDirectory()) return readSource(path) + if (entry === 'EVAL.ts' || !/\.(ts|tsx|js|jsx)$/.test(entry)) return [] + return readFileSync(path, 'utf8') + }) +} -test('Enables PPR via cacheComponents in next.config', () => { - const config = readFileSync(join(process.cwd(), 'next.config.ts'), 'utf-8') +const source = readSource(process.cwd()).join('\n') +const config = readFileSync(join(process.cwd(), 'next.config.ts'), 'utf8') +test('enables the current Cache Components config only', () => { expect(config).toMatch(/cacheComponents\s*:\s*true/) + expect(config).not.toMatch(/dynamicIO\s*:/) + expect(config).not.toMatch(/useCache\s*:/) }) -test('Does not use the old experimental.ppr flag', () => { - const config = readFileSync(join(process.cwd(), 'next.config.ts'), 'utf-8') - - // Strip comments to avoid false positives from explanatory comments - const stripped = config - .replace(/\/\*[\s\S]*?\*\//g, '') - .replace(/\/\/.*$/gm, '') +test('removes route opt-outs and incompatible segment config', () => { + expect(source).not.toMatch(/export\s+(?:const|var|let)\s+instant\s*=\s*false/) + expect(source).not.toMatch( + /export\s+(?:const|var|let)\s+(?:dynamic|revalidate|fetchCache)\s*=/ + ) +}) - expect(stripped).not.toMatch(/ppr\s*:\s*true/) +test('preserves the hourly feed and request-specific preview', async () => { + await expect(environment).toSatisfyCriterion( + `The newsroom feed is cached with an explicit approximately-hourly cache lifetime. A focused data-level cache is valid, and a page-level cache is also valid because the Newsroom page contains only the feed and stable framing with the same lifetime. The editor preview still reads its cookie at request time outside any public cache, is placed below Suspense or equivalent meaningful loading UI, and leaves useful stable preview framing in the static shell.` + ) }) diff --git a/evals/evals/agent-042-enable-ppr/PROMPT.md b/evals/evals/agent-042-enable-ppr/PROMPT.md index 2f9a333ba322..97820f8fcf1b 100644 --- a/evals/evals/agent-042-enable-ppr/PROMPT.md +++ b/evals/evals/agent-042-enable-ppr/PROMPT.md @@ -1 +1 @@ -Enable partial pre-rendering for this app. +Move this publishing app to Cache Components in one branch. Keep the newsroom feed on its hourly refresh policy and keep editor preview request-specific. Finish with a production build without leaving routes opted out. diff --git a/evals/evals/agent-042-enable-ppr/app/ProductList.tsx b/evals/evals/agent-042-enable-ppr/app/ProductList.tsx deleted file mode 100644 index 748a9909edfe..000000000000 --- a/evals/evals/agent-042-enable-ppr/app/ProductList.tsx +++ /dev/null @@ -1,18 +0,0 @@ -async function getProducts() { - const res = await fetch('https://api.example.com/products') - return res.json() -} - -export async function ProductList() { - const products = await getProducts() - - return ( -
    - {products.map((p: { id: string; name: string; price: number }) => ( -
  • - {p.name} - ${p.price} -
    • - ))} -
- ) -} diff --git a/evals/evals/agent-042-enable-ppr/app/Recommendations.tsx b/evals/evals/agent-042-enable-ppr/app/Recommendations.tsx deleted file mode 100644 index 61ee9575d6ad..000000000000 --- a/evals/evals/agent-042-enable-ppr/app/Recommendations.tsx +++ /dev/null @@ -1,19 +0,0 @@ -async function getRecommendations() { - const res = await fetch('https://api.example.com/recommendations') - return res.json() -} - -export async function Recommendations() { - const items = await getRecommendations() - - return ( -
-

Recommended for you

-
    - {items.map((item: { id: string; name: string }) => ( -
  • {item.name}
    • - ))} -
-
- ) -} diff --git a/evals/evals/agent-042-enable-ppr/app/layout.tsx b/evals/evals/agent-042-enable-ppr/app/layout.tsx index 6b8b4518030f..91cd6a8db4a9 100644 --- a/evals/evals/agent-042-enable-ppr/app/layout.tsx +++ b/evals/evals/agent-042-enable-ppr/app/layout.tsx @@ -1,18 +1,16 @@ -import type { Metadata } from 'next' +import Link from 'next/link' +import type { ReactNode } from 'react' -export const metadata: Metadata = { - title: 'Create Next App', - description: 'Generated by create next app', -} - -export default function RootLayout({ - children, -}: Readonly<{ - children: React.ReactNode -}>) { +export default function RootLayout({ children }: { children: ReactNode }) { return ( - {children} + + + {children} + ) } diff --git a/evals/evals/agent-048-cache-components-config-semantics/app/news/page.tsx b/evals/evals/agent-042-enable-ppr/app/news/page.tsx similarity index 100% rename from evals/evals/agent-048-cache-components-config-semantics/app/news/page.tsx rename to evals/evals/agent-042-enable-ppr/app/news/page.tsx diff --git a/evals/evals/agent-042-enable-ppr/app/page.tsx b/evals/evals/agent-042-enable-ppr/app/page.tsx index 971678328582..89fbd7e29342 100644 --- a/evals/evals/agent-042-enable-ppr/app/page.tsx +++ b/evals/evals/agent-042-enable-ppr/app/page.tsx @@ -1,17 +1,3 @@ -import { Suspense } from 'react' -import { ProductList } from './ProductList' -import { Recommendations } from './Recommendations' - -export default function Page() { - return ( -
-

Store

- Loading products...

}> - - - Loading recommendations...

}> - - -
- ) +export default function HomePage() { + return

Northstar Publishing

} diff --git a/evals/evals/agent-048-cache-components-config-semantics/app/preview/page.tsx b/evals/evals/agent-042-enable-ppr/app/preview/page.tsx similarity index 100% rename from evals/evals/agent-048-cache-components-config-semantics/app/preview/page.tsx rename to evals/evals/agent-042-enable-ppr/app/preview/page.tsx diff --git a/evals/evals/agent-042-enable-ppr/next.config.ts b/evals/evals/agent-042-enable-ppr/next.config.ts index e4f5738a310b..5c906c55af96 100644 --- a/evals/evals/agent-042-enable-ppr/next.config.ts +++ b/evals/evals/agent-042-enable-ppr/next.config.ts @@ -1,5 +1,10 @@ import type { NextConfig } from 'next' -const nextConfig: NextConfig = {} +const nextConfig: NextConfig = { + experimental: { + // @ts-expect-error -- legacy config retained for the migration exercise + dynamicIO: true, + }, +} export default nextConfig diff --git a/evals/evals/agent-042-enable-ppr/tsconfig.json b/evals/evals/agent-042-enable-ppr/tsconfig.json index 00978ef407fd..cc321ed0658b 100644 --- a/evals/evals/agent-042-enable-ppr/tsconfig.json +++ b/evals/evals/agent-042-enable-ppr/tsconfig.json @@ -23,5 +23,5 @@ } }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], - "exclude": ["node_modules"] + "exclude": ["node_modules", "EVAL.ts"] } diff --git a/evals/evals/agent-048-cache-components-config-semantics/EVAL.ts b/evals/evals/agent-048-cache-components-config-semantics/EVAL.ts deleted file mode 100644 index 27879531129b..000000000000 --- a/evals/evals/agent-048-cache-components-config-semantics/EVAL.ts +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Cache Components: route config semantics - * - * Verifies that adoption translates legacy configuration instead of deleting - * behavior or silencing validation. - */ - -import { expect, test } from 'vitest' -import { existsSync, readFileSync, readdirSync, statSync } from 'fs' -import { join } from 'path' -import { environment } from '@vercel/agent-eval/eval' - -const IGNORE_DIRS = new Set(['.git', '.next', 'node_modules', 'dist', 'build']) - -function readSource(dir: string): string[] { - if (!existsSync(dir)) return [] - return readdirSync(dir).flatMap((entry) => { - if (IGNORE_DIRS.has(entry)) return [] - const path = join(dir, entry) - if (statSync(path).isDirectory()) return readSource(path) - if (entry === 'EVAL.ts' || !/\.(ts|tsx|js|jsx)$/.test(entry)) return [] - return readFileSync(path, 'utf8') - }) -} - -const source = readSource(process.cwd()).join('\n') -const config = readFileSync(join(process.cwd(), 'next.config.ts'), 'utf8') - -test('enables the current Cache Components config only', () => { - expect(config).toMatch(/cacheComponents\s*:\s*true/) - expect(config).not.toMatch(/dynamicIO\s*:/) - expect(config).not.toMatch(/useCache\s*:/) -}) - -test('removes route opt-outs and incompatible segment config', () => { - expect(source).not.toMatch(/export\s+(?:const|var|let)\s+instant\s*=\s*false/) - expect(source).not.toMatch( - /export\s+(?:const|var|let)\s+(?:dynamic|revalidate|fetchCache)\s*=/ - ) -}) - -test('preserves the hourly feed and request-specific preview', async () => { - await expect(environment).toSatisfyCriterion( - `The newsroom feed is cached with an explicit approximately-hourly cache lifetime. A focused data-level cache is valid, and a page-level cache is also valid because the Newsroom page contains only the feed and stable framing with the same lifetime. The editor preview still reads its cookie at request time outside any public cache, is placed below Suspense or equivalent meaningful loading UI, and leaves useful stable preview framing in the static shell.` - ) -}) diff --git a/evals/evals/agent-048-cache-components-config-semantics/PROMPT.md b/evals/evals/agent-048-cache-components-config-semantics/PROMPT.md deleted file mode 100644 index 97820f8fcf1b..000000000000 --- a/evals/evals/agent-048-cache-components-config-semantics/PROMPT.md +++ /dev/null @@ -1 +0,0 @@ -Move this publishing app to Cache Components in one branch. Keep the newsroom feed on its hourly refresh policy and keep editor preview request-specific. Finish with a production build without leaving routes opted out. diff --git a/evals/evals/agent-048-cache-components-config-semantics/app/layout.tsx b/evals/evals/agent-048-cache-components-config-semantics/app/layout.tsx deleted file mode 100644 index 91cd6a8db4a9..000000000000 --- a/evals/evals/agent-048-cache-components-config-semantics/app/layout.tsx +++ /dev/null @@ -1,16 +0,0 @@ -import Link from 'next/link' -import type { ReactNode } from 'react' - -export default function RootLayout({ children }: { children: ReactNode }) { - return ( - - - - {children} - - - ) -} diff --git a/evals/evals/agent-048-cache-components-config-semantics/app/page.tsx b/evals/evals/agent-048-cache-components-config-semantics/app/page.tsx deleted file mode 100644 index 89fbd7e29342..000000000000 --- a/evals/evals/agent-048-cache-components-config-semantics/app/page.tsx +++ /dev/null @@ -1,3 +0,0 @@ -export default function HomePage() { - return

Northstar Publishing

-} diff --git a/evals/evals/agent-048-cache-components-config-semantics/next-env.d.ts b/evals/evals/agent-048-cache-components-config-semantics/next-env.d.ts deleted file mode 100644 index 1b3be0840f3f..000000000000 --- a/evals/evals/agent-048-cache-components-config-semantics/next-env.d.ts +++ /dev/null @@ -1,5 +0,0 @@ -/// -/// - -// NOTE: This file should not be edited -// see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/evals/evals/agent-048-cache-components-config-semantics/next.config.ts b/evals/evals/agent-048-cache-components-config-semantics/next.config.ts deleted file mode 100644 index 5c906c55af96..000000000000 --- a/evals/evals/agent-048-cache-components-config-semantics/next.config.ts +++ /dev/null @@ -1,10 +0,0 @@ -import type { NextConfig } from 'next' - -const nextConfig: NextConfig = { - experimental: { - // @ts-expect-error -- legacy config retained for the migration exercise - dynamicIO: true, - }, -} - -export default nextConfig diff --git a/evals/evals/agent-048-cache-components-config-semantics/package.json b/evals/evals/agent-048-cache-components-config-semantics/package.json deleted file mode 100644 index 7056dd9ed5f9..000000000000 --- a/evals/evals/agent-048-cache-components-config-semantics/package.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "private": true, - "type": "module", - "scripts": { - "dev": "next dev", - "build": "next build", - "start": "next start" - }, - "dependencies": { - "next": "^16", - "react": "19.1.0", - "react-dom": "19.1.0" - }, - "devDependencies": { - "@types/node": "^20", - "@types/react": "^19", - "@types/react-dom": "^19", - "typescript": "^5", - "vitest": "^3.1.3", - "@vitejs/plugin-react": "^4.4.1", - "vite-tsconfig-paths": "^5.1.4" - } -} diff --git a/evals/evals/agent-048-cache-components-config-semantics/tsconfig.json b/evals/evals/agent-048-cache-components-config-semantics/tsconfig.json deleted file mode 100644 index e32fb699a1b0..000000000000 --- a/evals/evals/agent-048-cache-components-config-semantics/tsconfig.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2017", - "lib": ["dom", "dom.iterable", "esnext"], - "allowJs": true, - "skipLibCheck": true, - "strict": true, - "noEmit": true, - "esModuleInterop": true, - "module": "esnext", - "moduleResolution": "bundler", - "resolveJsonModule": true, - "isolatedModules": true, - "jsx": "react-jsx", - "incremental": true, - "plugins": [{ "name": "next" }], - "paths": { "@/*": ["./*"] } - }, - "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], - "exclude": ["node_modules", "EVAL.ts"] -} diff --git a/evals/evals/agent-051-cache-components-sync-io/EVAL.ts b/evals/evals/agent-051-cache-components-sync-io/EVAL.ts deleted file mode 100644 index 45388a3d8e5a..000000000000 --- a/evals/evals/agent-051-cache-components-sync-io/EVAL.ts +++ /dev/null @@ -1,38 +0,0 @@ -/** Cache Components: synchronous request-time I/O in shared UI. */ - -import { expect, test } from 'vitest' -import { existsSync, readFileSync, readdirSync, statSync } from 'fs' -import { join } from 'path' -import { environment } from '@vercel/agent-eval/eval' - -const IGNORE_DIRS = new Set(['.git', '.next', 'node_modules', 'dist', 'build']) - -function readSource(dir: string): string[] { - if (!existsSync(dir)) return [] - return readdirSync(dir).flatMap((entry) => { - if (IGNORE_DIRS.has(entry)) return [] - const path = join(dir, entry) - if (statSync(path).isDirectory()) return readSource(path) - if (entry === 'EVAL.ts' || !/\.(ts|tsx|js|jsx)$/.test(entry)) return [] - return readFileSync(path, 'utf8') - }) -} - -const source = readSource(process.cwd()).join('\n') -const config = readFileSync(join(process.cwd(), 'next.config.ts'), 'utf8') - -test('enables Cache Components without opt-outs', () => { - expect(config).toMatch(/cacheComponents\s*:\s*true/) - expect(source).not.toMatch(/export\s+(?:const|var|let)\s+instant\s*=\s*false/) -}) - -test('uses an explicit synchronous I/O boundary', () => { - expect(source).toMatch(/\b(?:io|connection)\s*\(/) - expect(source).toMatch(/]/) -}) - -test('preserves fresh metadata without sacrificing the shared frame', async () => { - await expect(environment).toSatisfyCriterion( - `The stable Operations navigation and page frame remain outside the Suspense boundary. The request metadata is rendered in a focused async child that awaits io() or connection() before calling new Date or Date.now, Math.random, and randomUUID, so those values remain request-specific. The metadata subtree is not cached or evaluated at module scope, and it has meaningful loading UI. Prefer io() unless the implementation specifically needs to wait for a real user request.` - ) -}) diff --git a/evals/evals/agent-051-cache-components-sync-io/PROMPT.md b/evals/evals/agent-051-cache-components-sync-io/PROMPT.md deleted file mode 100644 index bfb437bc2894..000000000000 --- a/evals/evals/agent-051-cache-components-sync-io/PROMPT.md +++ /dev/null @@ -1 +0,0 @@ -Enable Cache Components for this operations console. The navigation must stay immediately useful, while the request identifier, current time, and sampling value must still be generated per request. Finish with a production build and do not leave routes opted out. diff --git a/evals/evals/agent-051-cache-components-sync-io/app/layout.tsx b/evals/evals/agent-051-cache-components-sync-io/app/layout.tsx deleted file mode 100644 index 03a7ce6ef914..000000000000 --- a/evals/evals/agent-051-cache-components-sync-io/app/layout.tsx +++ /dev/null @@ -1,14 +0,0 @@ -import type { ReactNode } from 'react' -import { RequestMetadata } from '@/components/request-metadata' - -export default function RootLayout({ children }: { children: ReactNode }) { - return ( - - - - - {children} - - - ) -} diff --git a/evals/evals/agent-051-cache-components-sync-io/app/page.tsx b/evals/evals/agent-051-cache-components-sync-io/app/page.tsx deleted file mode 100644 index 20adbd86f3fd..000000000000 --- a/evals/evals/agent-051-cache-components-sync-io/app/page.tsx +++ /dev/null @@ -1,8 +0,0 @@ -export default function HomePage() { - return ( -
-

Deployment health

-

All systems operational

-
- ) -} diff --git a/evals/evals/agent-051-cache-components-sync-io/next-env.d.ts b/evals/evals/agent-051-cache-components-sync-io/next-env.d.ts deleted file mode 100644 index 1b3be0840f3f..000000000000 --- a/evals/evals/agent-051-cache-components-sync-io/next-env.d.ts +++ /dev/null @@ -1,5 +0,0 @@ -/// -/// - -// NOTE: This file should not be edited -// see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/evals/evals/agent-051-cache-components-sync-io/next.config.ts b/evals/evals/agent-051-cache-components-sync-io/next.config.ts deleted file mode 100644 index e4f5738a310b..000000000000 --- a/evals/evals/agent-051-cache-components-sync-io/next.config.ts +++ /dev/null @@ -1,5 +0,0 @@ -import type { NextConfig } from 'next' - -const nextConfig: NextConfig = {} - -export default nextConfig diff --git a/evals/evals/agent-051-cache-components-sync-io/package.json b/evals/evals/agent-051-cache-components-sync-io/package.json deleted file mode 100644 index 7056dd9ed5f9..000000000000 --- a/evals/evals/agent-051-cache-components-sync-io/package.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "private": true, - "type": "module", - "scripts": { - "dev": "next dev", - "build": "next build", - "start": "next start" - }, - "dependencies": { - "next": "^16", - "react": "19.1.0", - "react-dom": "19.1.0" - }, - "devDependencies": { - "@types/node": "^20", - "@types/react": "^19", - "@types/react-dom": "^19", - "typescript": "^5", - "vitest": "^3.1.3", - "@vitejs/plugin-react": "^4.4.1", - "vite-tsconfig-paths": "^5.1.4" - } -} diff --git a/evals/evals/agent-051-cache-components-sync-io/tsconfig.json b/evals/evals/agent-051-cache-components-sync-io/tsconfig.json deleted file mode 100644 index e32fb699a1b0..000000000000 --- a/evals/evals/agent-051-cache-components-sync-io/tsconfig.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2017", - "lib": ["dom", "dom.iterable", "esnext"], - "allowJs": true, - "skipLibCheck": true, - "strict": true, - "noEmit": true, - "esModuleInterop": true, - "module": "esnext", - "moduleResolution": "bundler", - "resolveJsonModule": true, - "isolatedModules": true, - "jsx": "react-jsx", - "incremental": true, - "plugins": [{ "name": "next" }], - "paths": { "@/*": ["./*"] } - }, - "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], - "exclude": ["node_modules", "EVAL.ts"] -} From d60f307f8a9b1f131b892fda654a93dacd8a7ae9 Mon Sep 17 00:00:00 2001 From: Aurora Scharff Date: Mon, 31 Aug 2026 21:06:23 +0200 Subject: [PATCH 08/10] evals: split focused success criteria --- .../agent-035-connection-dynamic/EVAL.ts | 16 +++++- evals/evals/agent-040-instant/EVAL.ts | 30 +++++++++-- evals/evals/agent-042-enable-ppr/EVAL.ts | 10 +++- .../EVAL.ts | 0 .../PROMPT.md | 0 .../app/catalog/page.tsx | 0 .../app/layout.tsx | 0 .../app/page.tsx | 0 .../lib/data.ts | 0 .../next.config.ts | 0 .../package.json | 0 .../tsconfig.json | 0 .../EVAL.ts | 14 +++-- .../PROMPT.md | 0 .../app/layout.tsx | 0 .../app/page.tsx | 0 .../app/product/page.tsx | 0 .../lib/data.ts | 0 .../next.config.ts | 0 .../package.json | 0 .../tsconfig.json | 0 .../EVAL.ts | 33 ------------ .../EVAL.ts | 18 +++++-- .../PROMPT.md | 0 .../app/layout.tsx | 0 .../app/page.tsx | 0 .../app/projects/[slug]/page.tsx | 0 .../lib/data.ts | 0 .../next.config.ts | 0 .../package.json | 0 .../tsconfig.json | 0 .../EVAL.ts | 16 +++++- .../PROMPT.md | 0 .../app/layout.tsx | 0 .../app/page.tsx | 0 .../lib/projects.ts | 0 .../next-env.d.ts | 0 .../next.config.ts | 0 .../package.json | 0 .../tsconfig.json | 0 .../EVAL.ts | 51 +++++++++++++++++++ .../PROMPT.md | 0 .../app/layout.tsx | 0 .../app/page.tsx | 0 .../app/products/[slug]/page.tsx | 0 .../lib/products.ts | 0 .../next-env.d.ts | 0 .../next.config.ts | 0 .../package.json | 0 .../tsconfig.json | 0 50 files changed, 136 insertions(+), 52 deletions(-) rename evals/evals/{agent-044-prefetch-url-data => agent-048-prefetch-url-data}/EVAL.ts (100%) rename evals/evals/{agent-044-prefetch-url-data => agent-048-prefetch-url-data}/PROMPT.md (100%) rename evals/evals/{agent-044-prefetch-url-data => agent-048-prefetch-url-data}/app/catalog/page.tsx (100%) rename evals/evals/{agent-044-prefetch-url-data => agent-048-prefetch-url-data}/app/layout.tsx (100%) rename evals/evals/{agent-044-prefetch-url-data => agent-048-prefetch-url-data}/app/page.tsx (100%) rename evals/evals/{agent-044-prefetch-url-data => agent-048-prefetch-url-data}/lib/data.ts (100%) rename evals/evals/{agent-044-prefetch-url-data => agent-048-prefetch-url-data}/next.config.ts (100%) rename evals/evals/{agent-044-prefetch-url-data => agent-048-prefetch-url-data}/package.json (100%) rename evals/evals/{agent-044-prefetch-url-data => agent-048-prefetch-url-data}/tsconfig.json (100%) rename evals/evals/{agent-045-prefetch-stage => agent-049-prefetch-stage}/EVAL.ts (63%) rename evals/evals/{agent-045-prefetch-stage => agent-049-prefetch-stage}/PROMPT.md (100%) rename evals/evals/{agent-045-prefetch-stage => agent-049-prefetch-stage}/app/layout.tsx (100%) rename evals/evals/{agent-045-prefetch-stage => agent-049-prefetch-stage}/app/page.tsx (100%) rename evals/evals/{agent-045-prefetch-stage => agent-049-prefetch-stage}/app/product/page.tsx (100%) rename evals/evals/{agent-045-prefetch-stage => agent-049-prefetch-stage}/lib/data.ts (100%) rename evals/evals/{agent-045-prefetch-stage => agent-049-prefetch-stage}/next.config.ts (100%) rename evals/evals/{agent-045-prefetch-stage => agent-049-prefetch-stage}/package.json (100%) rename evals/evals/{agent-045-prefetch-stage => agent-049-prefetch-stage}/tsconfig.json (100%) delete mode 100644 evals/evals/agent-050-cache-components-url-shell/EVAL.ts rename evals/evals/{agent-046-prefetch-until-navigation => agent-051-prefetch-until-navigation}/EVAL.ts (59%) rename evals/evals/{agent-046-prefetch-until-navigation => agent-051-prefetch-until-navigation}/PROMPT.md (100%) rename evals/evals/{agent-046-prefetch-until-navigation => agent-051-prefetch-until-navigation}/app/layout.tsx (100%) rename evals/evals/{agent-046-prefetch-until-navigation => agent-051-prefetch-until-navigation}/app/page.tsx (100%) rename evals/evals/{agent-046-prefetch-until-navigation => agent-051-prefetch-until-navigation}/app/projects/[slug]/page.tsx (100%) rename evals/evals/{agent-046-prefetch-until-navigation => agent-051-prefetch-until-navigation}/lib/data.ts (100%) rename evals/evals/{agent-046-prefetch-until-navigation => agent-051-prefetch-until-navigation}/next.config.ts (100%) rename evals/evals/{agent-046-prefetch-until-navigation => agent-051-prefetch-until-navigation}/package.json (100%) rename evals/evals/{agent-046-prefetch-until-navigation => agent-051-prefetch-until-navigation}/tsconfig.json (100%) rename evals/evals/{agent-049-cache-components-session-isolation => agent-052-cache-components-session-isolation}/EVAL.ts (50%) rename evals/evals/{agent-049-cache-components-session-isolation => agent-052-cache-components-session-isolation}/PROMPT.md (100%) rename evals/evals/{agent-049-cache-components-session-isolation => agent-052-cache-components-session-isolation}/app/layout.tsx (100%) rename evals/evals/{agent-049-cache-components-session-isolation => agent-052-cache-components-session-isolation}/app/page.tsx (100%) rename evals/evals/{agent-049-cache-components-session-isolation => agent-052-cache-components-session-isolation}/lib/projects.ts (100%) rename evals/evals/{agent-049-cache-components-session-isolation => agent-052-cache-components-session-isolation}/next-env.d.ts (100%) rename evals/evals/{agent-049-cache-components-session-isolation => agent-052-cache-components-session-isolation}/next.config.ts (100%) rename evals/evals/{agent-049-cache-components-session-isolation => agent-052-cache-components-session-isolation}/package.json (100%) rename evals/evals/{agent-049-cache-components-session-isolation => agent-052-cache-components-session-isolation}/tsconfig.json (100%) create mode 100644 evals/evals/agent-053-cache-components-url-shell/EVAL.ts rename evals/evals/{agent-050-cache-components-url-shell => agent-053-cache-components-url-shell}/PROMPT.md (100%) rename evals/evals/{agent-050-cache-components-url-shell => agent-053-cache-components-url-shell}/app/layout.tsx (100%) rename evals/evals/{agent-050-cache-components-url-shell => agent-053-cache-components-url-shell}/app/page.tsx (100%) rename evals/evals/{agent-050-cache-components-url-shell => agent-053-cache-components-url-shell}/app/products/[slug]/page.tsx (100%) rename evals/evals/{agent-050-cache-components-url-shell => agent-053-cache-components-url-shell}/lib/products.ts (100%) rename evals/evals/{agent-050-cache-components-url-shell => agent-053-cache-components-url-shell}/next-env.d.ts (100%) rename evals/evals/{agent-050-cache-components-url-shell => agent-053-cache-components-url-shell}/next.config.ts (100%) rename evals/evals/{agent-050-cache-components-url-shell => agent-053-cache-components-url-shell}/package.json (100%) rename evals/evals/{agent-050-cache-components-url-shell => agent-053-cache-components-url-shell}/tsconfig.json (100%) diff --git a/evals/evals/agent-035-connection-dynamic/EVAL.ts b/evals/evals/agent-035-connection-dynamic/EVAL.ts index 45388a3d8e5a..bce8d5893893 100644 --- a/evals/evals/agent-035-connection-dynamic/EVAL.ts +++ b/evals/evals/agent-035-connection-dynamic/EVAL.ts @@ -31,8 +31,20 @@ test('uses an explicit synchronous I/O boundary', () => { expect(source).toMatch(/]/) }) -test('preserves fresh metadata without sacrificing the shared frame', async () => { +test('keeps the operations frame useful while metadata loads', async () => { await expect(environment).toSatisfyCriterion( - `The stable Operations navigation and page frame remain outside the Suspense boundary. The request metadata is rendered in a focused async child that awaits io() or connection() before calling new Date or Date.now, Math.random, and randomUUID, so those values remain request-specific. The metadata subtree is not cached or evaluated at module scope, and it has meaningful loading UI. Prefer io() unless the implementation specifically needs to wait for a real user request.` + `The stable Operations navigation and page frame must remain outside the Suspense boundary used for request metadata. The boundary must provide visible, meaningful loading UI rather than a null or empty fallback. Accept equivalent component and file organization.` + ) +}) + +test('keeps request metadata fresh', async () => { + await expect(environment).toSatisfyCriterion( + `The request identifier, current time, and sampling value must remain request-specific. Render them in a focused async child that awaits io() or connection() before calling randomUUID, new Date or Date.now, and Math.random. Prefer io() unless the implementation specifically needs to wait for a real user request. Accept equivalent component and file organization.` + ) +}) + +test('does not reuse request metadata', async () => { + await expect(environment).toSatisfyCriterion( + `The request identifier, current time, and sampling value must not be cached, evaluated at module scope, or replaced with static values. Reject any solution that can reuse those values across requests. Accept equivalent component and file organization.` ) }) diff --git a/evals/evals/agent-040-instant/EVAL.ts b/evals/evals/agent-040-instant/EVAL.ts index a6ca887659e3..aceef5bb71d6 100644 --- a/evals/evals/agent-040-instant/EVAL.ts +++ b/evals/evals/agent-040-instant/EVAL.ts @@ -10,12 +10,34 @@ import { expect, test } from 'vitest' import { environment } from '@vercel/agent-eval/eval' -test('keeps the title in the shell while live inventory streams', async () => { +test('keeps the product title in the static shell', async () => { await expect(environment).toSatisfyCriterion( - `The product route must produce meaningful instant UI with Cache Components, without relying on Partial Prefetching. + `The product title "Premium Widget" must render in the product page's static shell. It must remain outside the Suspense boundary and async work used for the live inventory, so the title is available before the inventory resolves. -The product title "Premium Widget" must render in the product page's static shell, outside the Suspense boundary that contains the live inventory. The inventory count and price must render from an async child beneath that Suspense boundary, so the page returns its title and fallback without waiting for getInventory(). The runtime boundary in getInventory(), such as connection(), must be preserved; do not cache the randomized inventory or replace it with static values. Keep the framework's default validation enabled; do not add instant = false to suppress the insight. +Accept equivalent component and file organization. Reject solutions where the page awaits the inventory before returning or places the title behind the inventory boundary.` + ) +}) + +test('streams inventory behind a meaningful fallback', async () => { + await expect(environment).toSatisfyCriterion( + `The inventory count and price must render from an async child beneath a Suspense boundary with a visible, meaningful fallback. The product page must be able to return its title and fallback without waiting for getInventory(). + +Accept equivalent component and file organization. Reject a null or empty fallback, or a solution where the page awaits inventory before returning.` + ) +}) + +test('keeps inventory live and uncached', async () => { + await expect(environment).toSatisfyCriterion( + `The randomized inventory must remain request-time data. Preserve the runtime boundary in getInventory(), such as connection(), and do not cache the inventory or replace its count and price with static values. + +Accept equivalent component and file organization. Reject any solution that makes the randomized inventory reusable across requests.` + ) +}) + +test('keeps default instant validation enabled', async () => { + await expect(environment).toSatisfyCriterion( + `The solution must keep the framework's default instant validation enabled and solve the static-shell problem directly. It must not add instant = false, disable Cache Components, or use Partial Prefetching as a substitute for a valid product shell. -Accept equivalent component and file organization. Reject solutions where the page awaits inventory before returning, the title is hidden behind the inventory boundary, the live inventory is cached, or Partial Prefetching is used as a substitute for a valid instant shell.` +Accept equivalent component and file organization.` ) }) diff --git a/evals/evals/agent-042-enable-ppr/EVAL.ts b/evals/evals/agent-042-enable-ppr/EVAL.ts index 27879531129b..ff209bf284f9 100644 --- a/evals/evals/agent-042-enable-ppr/EVAL.ts +++ b/evals/evals/agent-042-enable-ppr/EVAL.ts @@ -39,8 +39,14 @@ test('removes route opt-outs and incompatible segment config', () => { ) }) -test('preserves the hourly feed and request-specific preview', async () => { +test('preserves the hourly newsroom feed', async () => { await expect(environment).toSatisfyCriterion( - `The newsroom feed is cached with an explicit approximately-hourly cache lifetime. A focused data-level cache is valid, and a page-level cache is also valid because the Newsroom page contains only the feed and stable framing with the same lifetime. The editor preview still reads its cookie at request time outside any public cache, is placed below Suspense or equivalent meaningful loading UI, and leaves useful stable preview framing in the static shell.` + `The newsroom feed must remain cached with an explicit approximately-hourly cache lifetime. A focused data-level cache is valid, and a page-level cache is also valid because the Newsroom page contains only the feed and stable framing with the same lifetime. Reject solutions that remove or materially change the feed's hourly refresh behavior.` + ) +}) + +test('keeps editor preview request-specific', async () => { + await expect(environment).toSatisfyCriterion( + `The editor preview must still read its cookie at request time outside every public cache. Place the request-specific preview below Suspense or equivalent meaningful loading UI while leaving useful stable preview framing in the static shell. Reject solutions that cache the cookie value, hide the entire preview page behind one fallback, or remove the request-specific behavior.` ) }) diff --git a/evals/evals/agent-044-prefetch-url-data/EVAL.ts b/evals/evals/agent-048-prefetch-url-data/EVAL.ts similarity index 100% rename from evals/evals/agent-044-prefetch-url-data/EVAL.ts rename to evals/evals/agent-048-prefetch-url-data/EVAL.ts diff --git a/evals/evals/agent-044-prefetch-url-data/PROMPT.md b/evals/evals/agent-048-prefetch-url-data/PROMPT.md similarity index 100% rename from evals/evals/agent-044-prefetch-url-data/PROMPT.md rename to evals/evals/agent-048-prefetch-url-data/PROMPT.md diff --git a/evals/evals/agent-044-prefetch-url-data/app/catalog/page.tsx b/evals/evals/agent-048-prefetch-url-data/app/catalog/page.tsx similarity index 100% rename from evals/evals/agent-044-prefetch-url-data/app/catalog/page.tsx rename to evals/evals/agent-048-prefetch-url-data/app/catalog/page.tsx diff --git a/evals/evals/agent-044-prefetch-url-data/app/layout.tsx b/evals/evals/agent-048-prefetch-url-data/app/layout.tsx similarity index 100% rename from evals/evals/agent-044-prefetch-url-data/app/layout.tsx rename to evals/evals/agent-048-prefetch-url-data/app/layout.tsx diff --git a/evals/evals/agent-044-prefetch-url-data/app/page.tsx b/evals/evals/agent-048-prefetch-url-data/app/page.tsx similarity index 100% rename from evals/evals/agent-044-prefetch-url-data/app/page.tsx rename to evals/evals/agent-048-prefetch-url-data/app/page.tsx diff --git a/evals/evals/agent-044-prefetch-url-data/lib/data.ts b/evals/evals/agent-048-prefetch-url-data/lib/data.ts similarity index 100% rename from evals/evals/agent-044-prefetch-url-data/lib/data.ts rename to evals/evals/agent-048-prefetch-url-data/lib/data.ts diff --git a/evals/evals/agent-044-prefetch-url-data/next.config.ts b/evals/evals/agent-048-prefetch-url-data/next.config.ts similarity index 100% rename from evals/evals/agent-044-prefetch-url-data/next.config.ts rename to evals/evals/agent-048-prefetch-url-data/next.config.ts diff --git a/evals/evals/agent-044-prefetch-url-data/package.json b/evals/evals/agent-048-prefetch-url-data/package.json similarity index 100% rename from evals/evals/agent-044-prefetch-url-data/package.json rename to evals/evals/agent-048-prefetch-url-data/package.json diff --git a/evals/evals/agent-044-prefetch-url-data/tsconfig.json b/evals/evals/agent-048-prefetch-url-data/tsconfig.json similarity index 100% rename from evals/evals/agent-044-prefetch-url-data/tsconfig.json rename to evals/evals/agent-048-prefetch-url-data/tsconfig.json diff --git a/evals/evals/agent-045-prefetch-stage/EVAL.ts b/evals/evals/agent-049-prefetch-stage/EVAL.ts similarity index 63% rename from evals/evals/agent-045-prefetch-stage/EVAL.ts rename to evals/evals/agent-049-prefetch-stage/EVAL.ts index 4cfa4e69eea1..e70d0d4a5018 100644 --- a/evals/evals/agent-045-prefetch-stage/EVAL.ts +++ b/evals/evals/agent-049-prefetch-stage/EVAL.ts @@ -70,12 +70,18 @@ test('uses the prefetch cache stage', () => { expect(source).toMatch(new RegExp(`await\\s+${localName}\\s*\\(`)) }) -test('keeps related products out of the shell but in the selected prefetch', async () => { +test('keeps stable product content in the reusable shell', async () => { await expect(environment).toSatisfyCriterion( - `The implementation must keep the related-products section out of the reusable product App Shell while making it available before navigation from the featured-product Link. + `The product title and description must remain in the reusable product App Shell. The related-products section must remain outside that shell, beneath its own Suspense boundary. Reject solutions that put the whole product page behind the prefetch stage or move related products into the App Shell. Accept equivalent component and file organization.` + ) +}) + +test('includes related products only in the selected prefetch', async () => { + await expect(environment).toSatisfyCriterion( + `The related-products section must be available before navigation from the featured-product Link without joining the default product App Shell. -A correct solution enables Partial Prefetching globally or on the product route and adds prefetch={true} to the featured-product Link. The product title and description stay in the reusable App Shell. The related-products section is inside Suspense and rendered by an async component that awaits unstable_prefetch() before calling the cached getRelatedProducts() function. +Render related products through an async component that awaits unstable_prefetch() before calling the cached getRelatedProducts() function. The unstable_prefetch() call must be outside every use cache scope, while the existing cached data function stays below it. -The unstable_prefetch() call must be outside every use cache scope; the existing cached data function belongs below it. Reject solutions that put the whole product page behind the stage, move related products into the App Shell, replace the stage with connection() or unstable_navigation(), make the related-products data uncached, or defer it until after navigation.` +Reject solutions that replace the stage with connection() or unstable_navigation(), make the related-products data uncached, or defer it until after navigation. Accept equivalent component and file organization.` ) }) diff --git a/evals/evals/agent-045-prefetch-stage/PROMPT.md b/evals/evals/agent-049-prefetch-stage/PROMPT.md similarity index 100% rename from evals/evals/agent-045-prefetch-stage/PROMPT.md rename to evals/evals/agent-049-prefetch-stage/PROMPT.md diff --git a/evals/evals/agent-045-prefetch-stage/app/layout.tsx b/evals/evals/agent-049-prefetch-stage/app/layout.tsx similarity index 100% rename from evals/evals/agent-045-prefetch-stage/app/layout.tsx rename to evals/evals/agent-049-prefetch-stage/app/layout.tsx diff --git a/evals/evals/agent-045-prefetch-stage/app/page.tsx b/evals/evals/agent-049-prefetch-stage/app/page.tsx similarity index 100% rename from evals/evals/agent-045-prefetch-stage/app/page.tsx rename to evals/evals/agent-049-prefetch-stage/app/page.tsx diff --git a/evals/evals/agent-045-prefetch-stage/app/product/page.tsx b/evals/evals/agent-049-prefetch-stage/app/product/page.tsx similarity index 100% rename from evals/evals/agent-045-prefetch-stage/app/product/page.tsx rename to evals/evals/agent-049-prefetch-stage/app/product/page.tsx diff --git a/evals/evals/agent-045-prefetch-stage/lib/data.ts b/evals/evals/agent-049-prefetch-stage/lib/data.ts similarity index 100% rename from evals/evals/agent-045-prefetch-stage/lib/data.ts rename to evals/evals/agent-049-prefetch-stage/lib/data.ts diff --git a/evals/evals/agent-045-prefetch-stage/next.config.ts b/evals/evals/agent-049-prefetch-stage/next.config.ts similarity index 100% rename from evals/evals/agent-045-prefetch-stage/next.config.ts rename to evals/evals/agent-049-prefetch-stage/next.config.ts diff --git a/evals/evals/agent-045-prefetch-stage/package.json b/evals/evals/agent-049-prefetch-stage/package.json similarity index 100% rename from evals/evals/agent-045-prefetch-stage/package.json rename to evals/evals/agent-049-prefetch-stage/package.json diff --git a/evals/evals/agent-045-prefetch-stage/tsconfig.json b/evals/evals/agent-049-prefetch-stage/tsconfig.json similarity index 100% rename from evals/evals/agent-045-prefetch-stage/tsconfig.json rename to evals/evals/agent-049-prefetch-stage/tsconfig.json diff --git a/evals/evals/agent-050-cache-components-url-shell/EVAL.ts b/evals/evals/agent-050-cache-components-url-shell/EVAL.ts deleted file mode 100644 index 0ec0cd628412..000000000000 --- a/evals/evals/agent-050-cache-components-url-shell/EVAL.ts +++ /dev/null @@ -1,33 +0,0 @@ -/** Cache Components: URL data and shell boundaries. */ - -import { expect, test } from 'vitest' -import { existsSync, readFileSync, readdirSync, statSync } from 'fs' -import { join } from 'path' -import { environment } from '@vercel/agent-eval/eval' - -const IGNORE_DIRS = new Set(['.git', '.next', 'node_modules', 'dist', 'build']) - -function readSource(dir: string): string[] { - if (!existsSync(dir)) return [] - return readdirSync(dir).flatMap((entry) => { - if (IGNORE_DIRS.has(entry)) return [] - const path = join(dir, entry) - if (statSync(path).isDirectory()) return readSource(path) - if (entry === 'EVAL.ts' || !/\.(ts|tsx|js|jsx)$/.test(entry)) return [] - return readFileSync(path, 'utf8') - }) -} - -const source = readSource(process.cwd()).join('\n') -const config = readFileSync(join(process.cwd(), 'next.config.ts'), 'utf8') - -test('enables Cache Components without opt-outs', () => { - expect(config).toMatch(/cacheComponents\s*:\s*true/) - expect(source).not.toMatch(/export\s+(?:const|var|let)\s+instant\s*=\s*false/) -}) - -test('keeps URL-specific rendering below useful boundaries', async () => { - await expect(environment).toSatisfyCriterion( - `The product page returns useful route-independent shop framing before awaiting params or searchParams. The URL promises are forwarded into one or more children below meaningful Suspense fallbacks, where the slug and currency are resolved. Product lookup is cached in a small data function with every value that changes its output included in the cache key. Related products remain independently suspended and may stream. The solution does not hide the entire page behind one null or generic boundary.` - ) -}) diff --git a/evals/evals/agent-046-prefetch-until-navigation/EVAL.ts b/evals/evals/agent-051-prefetch-until-navigation/EVAL.ts similarity index 59% rename from evals/evals/agent-046-prefetch-until-navigation/EVAL.ts rename to evals/evals/agent-051-prefetch-until-navigation/EVAL.ts index 922e1394fa3c..4b4937202f39 100644 --- a/evals/evals/agent-046-prefetch-until-navigation/EVAL.ts +++ b/evals/evals/agent-051-prefetch-until-navigation/EVAL.ts @@ -57,6 +57,10 @@ test('enables Partial Prefetching globally or for the project route', () => { ).toBe(true) }) +test('opts the project link into per-link prefetching', () => { + expect(source).toMatch(/]*\bprefetch\s*=\s*\{true\}/) +}) + test('uses the navigation cache stage', () => { const navigationImport = source.match( /\bunstable_navigation(?:\s+as\s+([A-Za-z_$][\w$]*))?/ @@ -67,14 +71,18 @@ test('uses the navigation cache stage', () => { expect(source).toMatch(new RegExp(`await\\s+${localName}\\s*\\(`)) }) -test('the title is prefetched and project details wait for navigation', async () => { +test('includes the project title in the selected prefetch', async () => { await expect(environment).toSatisfyCriterion( - `The implementation must make the selected project's title available before the click without prefetching the rest of the project page. + `The selected project's title must be available before the click from the dashboard's per-link prefetch. Keep the params-dependent title inside Suspense so the reusable App Shell can show a fallback while the selected prefetch resolves the title. The title must render before the nested boundary that waits for navigation. -A correct solution uses Partial Prefetching globally or on the project destination, and opts the dashboard's project Link into a per-link prefetch with prefetch={true}. The project route keeps its params-dependent title inside Suspense so the reusable App Shell can show a fallback while the per-link prefetch resolves the title. +Accept equivalent component and file organization. Reject solutions that leave the title behind the reusable shell fallback, put it behind unstable_navigation(), disable prefetching, or prefetch the entire page.` + ) +}) -The rendered project title must appear before a nested Suspense boundary. Inside that boundary, an async component must await unstable_navigation() before it loads or renders the project activity, deployments, description, or other detail content. The unstable_navigation() call must be outside any use cache scope; cached data functions may be called after it. +test('waits until navigation for project details', async () => { + await expect(environment).toSatisfyCriterion( + `Project activity, deployments, description, and other detail content must not join the prefetch. Render those details inside a nested Suspense boundary through an async component that awaits unstable_navigation() before loading or rendering them. -Accept equivalent component and file organization. Reject solutions that put the title behind unstable_navigation(), fetch project details before reaching unstable_navigation(), replace unstable_navigation() with connection() or an uncached fetch, disable prefetching, or prefetch the entire page.` +The unstable_navigation() call must remain outside every use cache scope; cached data functions may be called after it. Reject solutions that fetch project details before reaching unstable_navigation(), replace it with connection() or an uncached fetch, or prefetch the entire page. Accept equivalent component and file organization.` ) }) diff --git a/evals/evals/agent-046-prefetch-until-navigation/PROMPT.md b/evals/evals/agent-051-prefetch-until-navigation/PROMPT.md similarity index 100% rename from evals/evals/agent-046-prefetch-until-navigation/PROMPT.md rename to evals/evals/agent-051-prefetch-until-navigation/PROMPT.md diff --git a/evals/evals/agent-046-prefetch-until-navigation/app/layout.tsx b/evals/evals/agent-051-prefetch-until-navigation/app/layout.tsx similarity index 100% rename from evals/evals/agent-046-prefetch-until-navigation/app/layout.tsx rename to evals/evals/agent-051-prefetch-until-navigation/app/layout.tsx diff --git a/evals/evals/agent-046-prefetch-until-navigation/app/page.tsx b/evals/evals/agent-051-prefetch-until-navigation/app/page.tsx similarity index 100% rename from evals/evals/agent-046-prefetch-until-navigation/app/page.tsx rename to evals/evals/agent-051-prefetch-until-navigation/app/page.tsx diff --git a/evals/evals/agent-046-prefetch-until-navigation/app/projects/[slug]/page.tsx b/evals/evals/agent-051-prefetch-until-navigation/app/projects/[slug]/page.tsx similarity index 100% rename from evals/evals/agent-046-prefetch-until-navigation/app/projects/[slug]/page.tsx rename to evals/evals/agent-051-prefetch-until-navigation/app/projects/[slug]/page.tsx diff --git a/evals/evals/agent-046-prefetch-until-navigation/lib/data.ts b/evals/evals/agent-051-prefetch-until-navigation/lib/data.ts similarity index 100% rename from evals/evals/agent-046-prefetch-until-navigation/lib/data.ts rename to evals/evals/agent-051-prefetch-until-navigation/lib/data.ts diff --git a/evals/evals/agent-046-prefetch-until-navigation/next.config.ts b/evals/evals/agent-051-prefetch-until-navigation/next.config.ts similarity index 100% rename from evals/evals/agent-046-prefetch-until-navigation/next.config.ts rename to evals/evals/agent-051-prefetch-until-navigation/next.config.ts diff --git a/evals/evals/agent-046-prefetch-until-navigation/package.json b/evals/evals/agent-051-prefetch-until-navigation/package.json similarity index 100% rename from evals/evals/agent-046-prefetch-until-navigation/package.json rename to evals/evals/agent-051-prefetch-until-navigation/package.json diff --git a/evals/evals/agent-046-prefetch-until-navigation/tsconfig.json b/evals/evals/agent-051-prefetch-until-navigation/tsconfig.json similarity index 100% rename from evals/evals/agent-046-prefetch-until-navigation/tsconfig.json rename to evals/evals/agent-051-prefetch-until-navigation/tsconfig.json diff --git a/evals/evals/agent-049-cache-components-session-isolation/EVAL.ts b/evals/evals/agent-052-cache-components-session-isolation/EVAL.ts similarity index 50% rename from evals/evals/agent-049-cache-components-session-isolation/EVAL.ts rename to evals/evals/agent-052-cache-components-session-isolation/EVAL.ts index c587888703cb..f86f4c850e14 100644 --- a/evals/evals/agent-049-cache-components-session-isolation/EVAL.ts +++ b/evals/evals/agent-052-cache-components-session-isolation/EVAL.ts @@ -31,8 +31,20 @@ test('introduces a bounded reusable cache', () => { expect(source).toMatch(/\bcacheLife\s*\(/) }) -test('keeps member data isolated and the shell meaningful', async () => { +test('keeps a useful dashboard frame in the shell', async () => { await expect(environment).toSatisfyCriterion( - `The dashboard keeps a useful stable frame such as the Workspace heading outside a meaningful Suspense boundary. Member name, team cookie, and request-region header are read at request time in the suspended subtree and are not captured by a public cache. Shared project totals are cached in a small data function with an explicit lifetime. No page or broad request-specific component is publicly cached, so values cannot leak between members.` + `The dashboard must keep a useful stable frame, such as the Workspace heading, outside the Suspense boundary used for request-specific member content. The boundary must provide visible, meaningful loading UI rather than a null or empty fallback. Accept equivalent component and file organization.` + ) +}) + +test('keeps member data isolated per request', async () => { + await expect(environment).toSatisfyCriterion( + `Member name, team cookie, and request-region header must be read at request time in the suspended subtree and must not be captured by a public cache. No page or broad request-specific component may be publicly cached. Reject any solution that can reuse one member's values for another member. Accept equivalent component and file organization.` + ) +}) + +test('caches only the shared project totals', async () => { + await expect(environment).toSatisfyCriterion( + `Shared project totals must be cached in a focused reusable data function with an explicit cache lifetime. The cache must not include member name, cookies, headers, or other request-specific values. Accept equivalent component and file organization.` ) }) diff --git a/evals/evals/agent-049-cache-components-session-isolation/PROMPT.md b/evals/evals/agent-052-cache-components-session-isolation/PROMPT.md similarity index 100% rename from evals/evals/agent-049-cache-components-session-isolation/PROMPT.md rename to evals/evals/agent-052-cache-components-session-isolation/PROMPT.md diff --git a/evals/evals/agent-049-cache-components-session-isolation/app/layout.tsx b/evals/evals/agent-052-cache-components-session-isolation/app/layout.tsx similarity index 100% rename from evals/evals/agent-049-cache-components-session-isolation/app/layout.tsx rename to evals/evals/agent-052-cache-components-session-isolation/app/layout.tsx diff --git a/evals/evals/agent-049-cache-components-session-isolation/app/page.tsx b/evals/evals/agent-052-cache-components-session-isolation/app/page.tsx similarity index 100% rename from evals/evals/agent-049-cache-components-session-isolation/app/page.tsx rename to evals/evals/agent-052-cache-components-session-isolation/app/page.tsx diff --git a/evals/evals/agent-049-cache-components-session-isolation/lib/projects.ts b/evals/evals/agent-052-cache-components-session-isolation/lib/projects.ts similarity index 100% rename from evals/evals/agent-049-cache-components-session-isolation/lib/projects.ts rename to evals/evals/agent-052-cache-components-session-isolation/lib/projects.ts diff --git a/evals/evals/agent-049-cache-components-session-isolation/next-env.d.ts b/evals/evals/agent-052-cache-components-session-isolation/next-env.d.ts similarity index 100% rename from evals/evals/agent-049-cache-components-session-isolation/next-env.d.ts rename to evals/evals/agent-052-cache-components-session-isolation/next-env.d.ts diff --git a/evals/evals/agent-049-cache-components-session-isolation/next.config.ts b/evals/evals/agent-052-cache-components-session-isolation/next.config.ts similarity index 100% rename from evals/evals/agent-049-cache-components-session-isolation/next.config.ts rename to evals/evals/agent-052-cache-components-session-isolation/next.config.ts diff --git a/evals/evals/agent-049-cache-components-session-isolation/package.json b/evals/evals/agent-052-cache-components-session-isolation/package.json similarity index 100% rename from evals/evals/agent-049-cache-components-session-isolation/package.json rename to evals/evals/agent-052-cache-components-session-isolation/package.json diff --git a/evals/evals/agent-049-cache-components-session-isolation/tsconfig.json b/evals/evals/agent-052-cache-components-session-isolation/tsconfig.json similarity index 100% rename from evals/evals/agent-049-cache-components-session-isolation/tsconfig.json rename to evals/evals/agent-052-cache-components-session-isolation/tsconfig.json diff --git a/evals/evals/agent-053-cache-components-url-shell/EVAL.ts b/evals/evals/agent-053-cache-components-url-shell/EVAL.ts new file mode 100644 index 000000000000..0705e5c0e342 --- /dev/null +++ b/evals/evals/agent-053-cache-components-url-shell/EVAL.ts @@ -0,0 +1,51 @@ +/** Cache Components: URL data and shell boundaries. */ + +import { expect, test } from 'vitest' +import { existsSync, readFileSync, readdirSync, statSync } from 'fs' +import { join } from 'path' +import { environment } from '@vercel/agent-eval/eval' + +const IGNORE_DIRS = new Set(['.git', '.next', 'node_modules', 'dist', 'build']) + +function readSource(dir: string): string[] { + if (!existsSync(dir)) return [] + return readdirSync(dir).flatMap((entry) => { + if (IGNORE_DIRS.has(entry)) return [] + const path = join(dir, entry) + if (statSync(path).isDirectory()) return readSource(path) + if (entry === 'EVAL.ts' || !/\.(ts|tsx|js|jsx)$/.test(entry)) return [] + return readFileSync(path, 'utf8') + }) +} + +const source = readSource(process.cwd()).join('\n') +const config = readFileSync(join(process.cwd(), 'next.config.ts'), 'utf8') + +test('enables Cache Components without opt-outs', () => { + expect(config).toMatch(/cacheComponents\s*:\s*true/) + expect(source).not.toMatch(/export\s+(?:const|var|let)\s+instant\s*=\s*false/) +}) + +test('keeps a useful shop frame in the static shell', async () => { + await expect(environment).toSatisfyCriterion( + `The product page must return useful route-independent shop framing before awaiting params or searchParams. The solution must not hide the entire page behind one null, empty, or generic Suspense boundary. Accept equivalent component and file organization.` + ) +}) + +test('resolves URL data below meaningful boundaries', async () => { + await expect(environment).toSatisfyCriterion( + `The product page must forward the params and searchParams promises into one or more children below meaningful Suspense fallbacks, where the product slug and currency are resolved. Reject solutions that await either URL promise before returning the useful shop frame. Accept equivalent component and file organization.` + ) +}) + +test('keys reusable product details by every input', async () => { + await expect(environment).toSatisfyCriterion( + `Product lookup must be cached in a focused reusable data function. Every value that can change its output, including the product slug and currency, must be passed into the cached function so it participates in the cache key. Reject broad page caching or a cache that can return one product or currency for another. Accept equivalent component and file organization.` + ) +}) + +test('streams related products independently', async () => { + await expect(environment).toSatisfyCriterion( + `Related products must remain behind their own meaningful Suspense boundary so they can stream independently after the primary product details. Reject solutions that block the primary details on related products or combine the whole page under one fallback. Accept equivalent component and file organization.` + ) +}) diff --git a/evals/evals/agent-050-cache-components-url-shell/PROMPT.md b/evals/evals/agent-053-cache-components-url-shell/PROMPT.md similarity index 100% rename from evals/evals/agent-050-cache-components-url-shell/PROMPT.md rename to evals/evals/agent-053-cache-components-url-shell/PROMPT.md diff --git a/evals/evals/agent-050-cache-components-url-shell/app/layout.tsx b/evals/evals/agent-053-cache-components-url-shell/app/layout.tsx similarity index 100% rename from evals/evals/agent-050-cache-components-url-shell/app/layout.tsx rename to evals/evals/agent-053-cache-components-url-shell/app/layout.tsx diff --git a/evals/evals/agent-050-cache-components-url-shell/app/page.tsx b/evals/evals/agent-053-cache-components-url-shell/app/page.tsx similarity index 100% rename from evals/evals/agent-050-cache-components-url-shell/app/page.tsx rename to evals/evals/agent-053-cache-components-url-shell/app/page.tsx diff --git a/evals/evals/agent-050-cache-components-url-shell/app/products/[slug]/page.tsx b/evals/evals/agent-053-cache-components-url-shell/app/products/[slug]/page.tsx similarity index 100% rename from evals/evals/agent-050-cache-components-url-shell/app/products/[slug]/page.tsx rename to evals/evals/agent-053-cache-components-url-shell/app/products/[slug]/page.tsx diff --git a/evals/evals/agent-050-cache-components-url-shell/lib/products.ts b/evals/evals/agent-053-cache-components-url-shell/lib/products.ts similarity index 100% rename from evals/evals/agent-050-cache-components-url-shell/lib/products.ts rename to evals/evals/agent-053-cache-components-url-shell/lib/products.ts diff --git a/evals/evals/agent-050-cache-components-url-shell/next-env.d.ts b/evals/evals/agent-053-cache-components-url-shell/next-env.d.ts similarity index 100% rename from evals/evals/agent-050-cache-components-url-shell/next-env.d.ts rename to evals/evals/agent-053-cache-components-url-shell/next-env.d.ts diff --git a/evals/evals/agent-050-cache-components-url-shell/next.config.ts b/evals/evals/agent-053-cache-components-url-shell/next.config.ts similarity index 100% rename from evals/evals/agent-050-cache-components-url-shell/next.config.ts rename to evals/evals/agent-053-cache-components-url-shell/next.config.ts diff --git a/evals/evals/agent-050-cache-components-url-shell/package.json b/evals/evals/agent-053-cache-components-url-shell/package.json similarity index 100% rename from evals/evals/agent-050-cache-components-url-shell/package.json rename to evals/evals/agent-053-cache-components-url-shell/package.json diff --git a/evals/evals/agent-050-cache-components-url-shell/tsconfig.json b/evals/evals/agent-053-cache-components-url-shell/tsconfig.json similarity index 100% rename from evals/evals/agent-050-cache-components-url-shell/tsconfig.json rename to evals/evals/agent-053-cache-components-url-shell/tsconfig.json From e65fdddca7ce5099c556031f4cd165fd9af6ea3e Mon Sep 17 00:00:00 2001 From: Aurora Scharff Date: Wed, 9 Sep 2026 01:03:51 +0200 Subject: [PATCH 09/10] evals: refine prefetching comparisons --- .../01-getting-started/04-linking-and-navigating.mdx | 3 ++- evals/evals/agent-035-connection-dynamic/EVAL.ts | 2 +- evals/evals/agent-048-prefetch-url-data/PROMPT.md | 2 +- .../agent-052-cache-components-session-isolation/EVAL.ts | 7 +++---- evals/lib/setup.ts | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/01-app/01-getting-started/04-linking-and-navigating.mdx b/docs/01-app/01-getting-started/04-linking-and-navigating.mdx index 08a04ec62454..e94e7d965e2a 100644 --- a/docs/01-app/01-getting-started/04-linking-and-navigating.mdx +++ b/docs/01-app/01-getting-started/04-linking-and-navigating.mdx @@ -5,6 +5,7 @@ related: links: - app/api-reference/components/link - app/api-reference/file-conventions/loading + - app/guides/optimizing-prefetching - app/guides/prefetching --- @@ -95,7 +96,7 @@ By skipping or partially prefetching dynamic routes, Next.js avoids unnecessary height="748" /> -> **Good to know:** See the [Prefetching guide](/docs/app/guides/prefetching) for the full behavior, including how to control prefetching per link and how it changes when you adopt [Partial Prefetching](/docs/app/guides/adopting-partial-prefetching). +> **Good to know:** See the [Prefetching guide](/docs/app/guides/prefetching) for the full behavior, including how to control prefetching per link and how it changes when you adopt [Partial Prefetching](/docs/app/guides/adopting-partial-prefetching). To include URL-dependent content in a link's prefetch, see [Optimizing prefetching](/docs/app/guides/optimizing-prefetching). To improve the navigation experience to dynamic routes, you can use [streaming](#streaming). diff --git a/evals/evals/agent-035-connection-dynamic/EVAL.ts b/evals/evals/agent-035-connection-dynamic/EVAL.ts index bce8d5893893..d795003af2eb 100644 --- a/evals/evals/agent-035-connection-dynamic/EVAL.ts +++ b/evals/evals/agent-035-connection-dynamic/EVAL.ts @@ -33,7 +33,7 @@ test('uses an explicit synchronous I/O boundary', () => { test('keeps the operations frame useful while metadata loads', async () => { await expect(environment).toSatisfyCriterion( - `The stable Operations navigation and page frame must remain outside the Suspense boundary used for request metadata. The boundary must provide visible, meaningful loading UI rather than a null or empty fallback. Accept equivalent component and file organization.` + `The stable Operations navigation and page frame must remain outside the Suspense boundary used for request metadata. Accept equivalent component and file organization.` ) }) diff --git a/evals/evals/agent-048-prefetch-url-data/PROMPT.md b/evals/evals/agent-048-prefetch-url-data/PROMPT.md index 6389e10ed201..6c8e5d0309c1 100644 --- a/evals/evals/agent-048-prefetch-url-data/PROMPT.md +++ b/evals/evals/agent-048-prefetch-url-data/PROMPT.md @@ -1 +1 @@ -Keep the existing `/catalog?category=...` URLs. When navigating from home to a product category, I want the selected category heading and its products to appear immediately instead of the loading state. +Keep the existing `/catalog?category=...` URLs. Prepare each category link's URL-specific heading and products before the click so selecting it can navigate immediately without showing the loading state. Do not make the click wait for a full server render. diff --git a/evals/evals/agent-052-cache-components-session-isolation/EVAL.ts b/evals/evals/agent-052-cache-components-session-isolation/EVAL.ts index f86f4c850e14..93e0d33a1be6 100644 --- a/evals/evals/agent-052-cache-components-session-isolation/EVAL.ts +++ b/evals/evals/agent-052-cache-components-session-isolation/EVAL.ts @@ -26,9 +26,8 @@ test('enables Cache Components without opt-outs', () => { expect(source).not.toMatch(/export\s+(?:const|var|let)\s+instant\s*=\s*false/) }) -test('introduces a bounded reusable cache', () => { +test('introduces a reusable cache', () => { expect(source).toMatch(/['"]use cache['"]/) - expect(source).toMatch(/\bcacheLife\s*\(/) }) test('keeps a useful dashboard frame in the shell', async () => { @@ -39,12 +38,12 @@ test('keeps a useful dashboard frame in the shell', async () => { test('keeps member data isolated per request', async () => { await expect(environment).toSatisfyCriterion( - `Member name, team cookie, and request-region header must be read at request time in the suspended subtree and must not be captured by a public cache. No page or broad request-specific component may be publicly cached. Reject any solution that can reuse one member's values for another member. Accept equivalent component and file organization.` + `Member name, team cookie, and request-region header must be read at request time in the suspended subtree and must not be captured by a public cache. No page or broad request-specific component may be publicly cached. A component that directly calls cookies() or headers() without a use cache directive remains request-specific and needs no additional cache opt-out. Reject any solution that can reuse one member's values for another member. Accept equivalent component and file organization.` ) }) test('caches only the shared project totals', async () => { await expect(environment).toSatisfyCriterion( - `Shared project totals must be cached in a focused reusable data function with an explicit cache lifetime. The cache must not include member name, cookies, headers, or other request-specific values. Accept equivalent component and file organization.` + `Shared project totals must be cached in a focused reusable data function. The cache must not include member name, cookies, headers, or other request-specific values. Accept equivalent component and file organization.` ) }) diff --git a/evals/lib/setup.ts b/evals/lib/setup.ts index a1b0759a2b2a..b2fed8ea214a 100644 --- a/evals/lib/setup.ts +++ b/evals/lib/setup.ts @@ -81,9 +81,9 @@ export async function writeAgentsMd(sandbox: Sandbox): Promise { const body = ` -# Next.js: ALWAYS search and read docs before coding +# Next.js: ALWAYS read docs before coding -Before any Next.js work, search \`node_modules/next/dist/docs/\` using terms from the task and source code, then read the most relevant guides and API references. Your training data is outdated — the docs are the source of truth. +Before any Next.js work, find and read the relevant doc in \`node_modules/next/dist/docs/\`. Your training data is outdated — the docs are the source of truth. ` From b5231024c99e2bf6adfebbfa54c5f28c00167358 Mon Sep 17 00:00:00 2001 From: Aurora Scharff Date: Wed, 9 Sep 2026 11:10:29 +0200 Subject: [PATCH 10/10] test: accept bare prefetch prop in navigation evals --- evals/evals/agent-049-prefetch-stage/EVAL.ts | 2 +- evals/evals/agent-051-prefetch-until-navigation/EVAL.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/evals/evals/agent-049-prefetch-stage/EVAL.ts b/evals/evals/agent-049-prefetch-stage/EVAL.ts index e70d0d4a5018..6ebb997222ff 100644 --- a/evals/evals/agent-049-prefetch-stage/EVAL.ts +++ b/evals/evals/agent-049-prefetch-stage/EVAL.ts @@ -57,7 +57,7 @@ test('enables Partial Prefetching globally or for the product route', () => { }) test('opts the featured product link into per-link prefetching', () => { - expect(source).toMatch(/]*\bprefetch\s*=\s*\{true\}/) + expect(source).toMatch(/]*\bprefetch(?:\s*=\s*\{true\})?(?:\s|>)/) }) test('uses the prefetch cache stage', () => { diff --git a/evals/evals/agent-051-prefetch-until-navigation/EVAL.ts b/evals/evals/agent-051-prefetch-until-navigation/EVAL.ts index 4b4937202f39..1db0dec18e34 100644 --- a/evals/evals/agent-051-prefetch-until-navigation/EVAL.ts +++ b/evals/evals/agent-051-prefetch-until-navigation/EVAL.ts @@ -58,7 +58,7 @@ test('enables Partial Prefetching globally or for the project route', () => { }) test('opts the project link into per-link prefetching', () => { - expect(source).toMatch(/]*\bprefetch\s*=\s*\{true\}/) + expect(source).toMatch(/]*\bprefetch(?:\s*=\s*\{true\})?(?:\s|>)/) }) test('uses the navigation cache stage', () => {