From 727a33c1e41684a76dea65c1f3667dc059cf8ac8 Mon Sep 17 00:00:00 2001 From: Robert Wagner Date: Sun, 23 Aug 2026 10:55:29 -0400 Subject: [PATCH 1/3] Serve markdown twins via Accept: text/markdown negotiation on Vercel Co-Authored-By: Claude Fable 5 --- package.json | 2 +- scripts/vercel-md-negotiation.mjs | 186 +++++++++++++++++++++++ tests/unit/vercel-md-negotiation.test.ts | 133 ++++++++++++++++ 3 files changed, 320 insertions(+), 1 deletion(-) create mode 100644 scripts/vercel-md-negotiation.mjs create mode 100644 tests/unit/vercel-md-negotiation.test.ts diff --git a/package.json b/package.json index da229ce..948d2c8 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "license": "MIT", "scripts": { "astro": "astro", - "build": "astro check && astro build", + "build": "astro check && astro build && node scripts/vercel-md-negotiation.mjs", "db:push": "drizzle-kit push", "db:seed": "tsx db/seed.ts", "db:studio": "drizzle-kit studio", diff --git a/scripts/vercel-md-negotiation.mjs b/scripts/vercel-md-negotiation.mjs new file mode 100644 index 0000000..2e4d088 --- /dev/null +++ b/scripts/vercel-md-negotiation.mjs @@ -0,0 +1,186 @@ +/** + * Patches `.vercel/output/config.json` after `astro build` so that pages with + * a prerendered markdown twin (`{path}.html.md`) serve that twin when a client + * asks for it with `Accept: text/markdown`, per https://acceptmarkdown.com. + * + * Both variants of a negotiated URL are stamped with `Vary: Accept` so CDNs + * never serve a cached HTML response to an agent asking for markdown (or vice + * versa). + * + * This has to happen post-build because Vercel checks the filesystem before + * applying `vercel.json` rewrites, so an Accept-based rewrite there would + * never run for prerendered pages. Routes injected before the `filesystem` + * handler in the Build Output API config do run first. + * + * Runs as part of `pnpm build`. No-ops when there is no Vercel build output. + */ + +import { existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join, relative, sep } from 'node:path'; +import process from 'node:process'; +import { pathToFileURL } from 'node:url'; + +const MD_SUFFIX = '.html.md'; + +// Vercel route `has` condition matching any Accept header that mentions +// text/markdown. +const ACCEPT_MARKDOWN = [ + { type: 'header', key: 'accept', value: '.*text/markdown.*' } +]; + +// How many slugs to pack into a single route regex alternation. +const CHUNK_SIZE = 50; + +/** + * Find every prerendered markdown twin in the static output directory and + * return the negotiated URL paths they belong to ('/index' for the homepage). + */ +export function collectMarkdownPaths(staticDir) { + const paths = []; + + const walk = (dir) => { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const full = join(dir, entry.name); + if (entry.isDirectory()) { + walk(full); + } else if (entry.name.endsWith(MD_SUFFIX)) { + const rel = relative(staticDir, full).split(sep).join('/'); + paths.push('/' + rel.slice(0, -MD_SUFFIX.length)); + } + } + }; + + walk(staticDir); + return paths.sort(); +} + +const escapeRegex = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + +const chunk = (items, size) => { + const chunks = []; + for (let i = 0; i < items.length; i += size) { + chunks.push(items.slice(i, i + size)); + } + return chunks; +}; + +/** + * Build the routes that implement the negotiation for the given markdown + * paths. Order matters: Vary stamps first (they `continue`), then the + * Accept-conditional rewrites to the markdown twins. + */ +export function buildNegotiationRoutes(mdPaths) { + const routes = []; + const hasHome = mdPaths.includes('/index'); + const slugChunks = chunk( + mdPaths.filter((p) => p !== '/index').map((p) => escapeRegex(p.slice(1))), + CHUNK_SIZE + ); + + // Stamp Vary: Accept on every negotiated URL, whichever variant ends up + // being served. + if (hasHome) { + routes.push({ src: '^/$', headers: { vary: 'Accept' }, continue: true }); + } + for (const slugs of slugChunks) { + routes.push({ + src: `^/(?:${slugs.join('|')})$`, + headers: { vary: 'Accept' }, + continue: true + }); + } + + // Rewrite to the markdown twin when the client asks for markdown. + if (hasHome) { + routes.push({ + src: '^/$', + has: ACCEPT_MARKDOWN, + dest: '/index.html.md' + }); + } + for (const slugs of slugChunks) { + routes.push({ + src: `^/(${slugs.join('|')})$`, + has: ACCEPT_MARKDOWN, + dest: '/$1.html.md' + }); + } + + return routes; +} + +/** + * Return a copy of the Vercel Build Output config with the negotiation routes + * inserted ahead of the `filesystem` handler. Idempotent: an already patched + * config is returned unchanged. + */ +export function patchConfig(config, mdPaths) { + if (!Array.isArray(config.routes)) { + throw new Error('config.json has no routes array'); + } + + const alreadyPatched = config.routes.some( + (route) => + typeof route.dest === 'string' && + route.dest.endsWith(MD_SUFFIX) && + Array.isArray(route.has) + ); + if (alreadyPatched) { + return { config, inserted: 0 }; + } + + const negotiationRoutes = buildNegotiationRoutes(mdPaths); + if (negotiationRoutes.length === 0) { + return { config, inserted: 0 }; + } + + const filesystemIndex = config.routes.findIndex( + (route) => route.handle === 'filesystem' + ); + if (filesystemIndex === -1) { + throw new Error( + 'config.json has no `handle: "filesystem"` route; the Vercel build output format may have changed' + ); + } + + const routes = [ + ...config.routes.slice(0, filesystemIndex), + ...negotiationRoutes, + ...config.routes.slice(filesystemIndex) + ]; + + return { config: { ...config, routes }, inserted: negotiationRoutes.length }; +} + +export function main(outputDir = '.vercel/output') { + const configPath = join(outputDir, 'config.json'); + const staticDir = join(outputDir, 'static'); + + if (!existsSync(configPath) || !existsSync(staticDir)) { + console.log( + `[md-negotiation] No Vercel build output at ${outputDir}, skipping` + ); + return; + } + + const config = JSON.parse(readFileSync(configPath, 'utf-8')); + const mdPaths = collectMarkdownPaths(staticDir); + const { config: patched, inserted } = patchConfig(config, mdPaths); + + if (inserted === 0) { + console.log('[md-negotiation] Nothing to patch'); + return; + } + + writeFileSync(configPath, JSON.stringify(patched, null, 2)); + console.log( + `[md-negotiation] Added ${inserted} routes negotiating markdown for ${mdPaths.length} pages` + ); +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + main(process.argv[2]); +} diff --git a/tests/unit/vercel-md-negotiation.test.ts b/tests/unit/vercel-md-negotiation.test.ts new file mode 100644 index 0000000..b58fad8 --- /dev/null +++ b/tests/unit/vercel-md-negotiation.test.ts @@ -0,0 +1,133 @@ +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, describe, expect, it } from 'vitest'; + +import { + buildNegotiationRoutes, + collectMarkdownPaths, + patchConfig +} from '../../scripts/vercel-md-negotiation.mjs'; + +const ACCEPT_MD = [{ type: 'header', key: 'accept', value: '.*text/markdown.*' }]; + +describe('vercel-md-negotiation', () => { + describe('collectMarkdownPaths', () => { + const dir = mkdtempSync(join(tmpdir(), 'starpod-md-')); + + afterAll(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it('finds every .html.md twin, including nested ones', () => { + writeFileSync(join(dir, 'index.html.md'), '# home'); + writeFileSync(join(dir, 'about.html.md'), '# about'); + writeFileSync(join(dir, 'about.html'), ''); + writeFileSync(join(dir, 'llms.txt'), 'llms'); + mkdirSync(join(dir, 'nested')); + writeFileSync(join(dir, 'nested', 'page.html.md'), '# nested'); + + expect(collectMarkdownPaths(dir)).toEqual([ + '/about', + '/index', + '/nested/page' + ]); + }); + }); + + describe('buildNegotiationRoutes', () => { + it('emits Vary stamps and Accept-conditional rewrites', () => { + const routes = buildNegotiationRoutes(['/index', '/about', '/contact']); + + // Vary stamps come first and continue to later routes. + expect(routes[0]).toEqual({ + src: '^/$', + headers: { vary: 'Accept' }, + continue: true + }); + expect(routes[1]).toEqual({ + src: '^/(?:about|contact)$', + headers: { vary: 'Accept' }, + continue: true + }); + + // Rewrites only fire for markdown-accepting clients. + expect(routes[2]).toEqual({ + src: '^/$', + has: ACCEPT_MD, + dest: '/index.html.md' + }); + expect(routes[3]).toEqual({ + src: '^/(about|contact)$', + has: ACCEPT_MD, + dest: '/$1.html.md' + }); + }); + + it('escapes regex metacharacters in slugs', () => { + const routes = buildNegotiationRoutes(['/what+is.this']); + expect(routes[0].src).toBe('^/(?:what\\+is\\.this)$'); + }); + + it('chunks large slug lists into multiple routes', () => { + const paths = Array.from({ length: 120 }, (_, i) => `/episode-${i}`); + const routes = buildNegotiationRoutes(paths); + + // 3 chunks of Vary stamps + 3 chunks of rewrites, no homepage. + expect(routes).toHaveLength(6); + expect(routes.every((r) => r.src.startsWith('^/'))).toBe(true); + }); + + it('returns no routes when there are no markdown twins', () => { + expect(buildNegotiationRoutes([])).toEqual([]); + }); + }); + + describe('patchConfig', () => { + const baseConfig = () => ({ + version: 3, + routes: [ + { src: '^/old-path$', headers: { Location: '/new-path' }, status: 308 }, + { handle: 'filesystem' }, + { src: '^/.*$', dest: '_render' } + ] + }); + + it('inserts negotiation routes before the filesystem handler', () => { + const { config, inserted } = patchConfig(baseConfig(), [ + '/index', + '/about' + ]); + + expect(inserted).toBe(4); + const filesystemIndex = config.routes.findIndex( + (r: { handle?: string }) => r.handle === 'filesystem' + ); + const rewriteIndex = config.routes.findIndex( + (r: { dest?: string }) => r.dest === '/about.html.md' || r.dest === '/$1.html.md' + ); + + // Redirects stay first, negotiation routes go before filesystem. + expect(config.routes[0].status).toBe(308); + expect(rewriteIndex).toBeGreaterThan(0); + expect(rewriteIndex).toBeLessThan(filesystemIndex); + }); + + it('is idempotent', () => { + const { config } = patchConfig(baseConfig(), ['/index', '/about']); + const { config: again, inserted } = patchConfig(config, [ + '/index', + '/about' + ]); + + expect(inserted).toBe(0); + expect(again.routes).toHaveLength(config.routes.length); + }); + + it('throws when the filesystem handler is missing', () => { + expect(() => + patchConfig({ version: 3, routes: [] }, ['/about']) + ).toThrow(/filesystem/); + }); + }); +}); From 6a30488e2bd00cd6bcf83f970abe5903f5c5a832 Mon Sep 17 00:00:00 2001 From: Robert Wagner Date: Sun, 23 Aug 2026 22:37:55 -0400 Subject: [PATCH 2/3] Make idempotency check exact against the generated route set An unrelated user-added Accept-conditional markdown route no longer suppresses patching; skip only when every generated route is already present. Co-Authored-By: Claude Fable 5 --- scripts/vercel-md-negotiation.mjs | 37 ++++++++++++++++++------ tests/unit/vercel-md-negotiation.test.ts | 21 ++++++++++++++ 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/scripts/vercel-md-negotiation.mjs b/scripts/vercel-md-negotiation.mjs index 2e4d088..ca18f9b 100644 --- a/scripts/vercel-md-negotiation.mjs +++ b/scripts/vercel-md-negotiation.mjs @@ -114,23 +114,42 @@ export function buildNegotiationRoutes(mdPaths) { * inserted ahead of the `filesystem` handler. Idempotent: an already patched * config is returned unchanged. */ +// Canonical JSON encoding (sorted object keys) so routes can be compared for +// exact equality regardless of key order. +const canonical = (value) => { + if (Array.isArray(value)) { + return value.map(canonical); + } + if (value && typeof value === 'object') { + return Object.fromEntries( + Object.keys(value) + .sort() + .map((key) => [key, canonical(value[key])]) + ); + } + return value; +}; + +const routeKey = (route) => JSON.stringify(canonical(route)); + export function patchConfig(config, mdPaths) { if (!Array.isArray(config.routes)) { throw new Error('config.json has no routes array'); } - const alreadyPatched = config.routes.some( - (route) => - typeof route.dest === 'string' && - route.dest.endsWith(MD_SUFFIX) && - Array.isArray(route.has) - ); - if (alreadyPatched) { + const negotiationRoutes = buildNegotiationRoutes(mdPaths); + if (negotiationRoutes.length === 0) { return { config, inserted: 0 }; } - const negotiationRoutes = buildNegotiationRoutes(mdPaths); - if (negotiationRoutes.length === 0) { + // Idempotency: only skip when every generated route is already present + // exactly. A user-added conditional markdown route must not suppress the + // generated set. + const existingRoutes = new Set(config.routes.map(routeKey)); + const alreadyPatched = negotiationRoutes.every((route) => + existingRoutes.has(routeKey(route)) + ); + if (alreadyPatched) { return { config, inserted: 0 }; } diff --git a/tests/unit/vercel-md-negotiation.test.ts b/tests/unit/vercel-md-negotiation.test.ts index b58fad8..a39ba2f 100644 --- a/tests/unit/vercel-md-negotiation.test.ts +++ b/tests/unit/vercel-md-negotiation.test.ts @@ -124,6 +124,27 @@ describe('vercel-md-negotiation', () => { expect(again.routes).toHaveLength(config.routes.length); }); + it('still patches when an unrelated conditional markdown route exists', () => { + const config = baseConfig(); + // A user-added Accept-conditional route pointing at a markdown file must + // not suppress the generated negotiation set. + config.routes.unshift({ + src: '^/custom$', + has: [{ type: 'header', key: 'accept', value: '.*text/markdown.*' }], + dest: '/custom-page.html.md' + }); + + const { config: patched, inserted } = patchConfig(config, [ + '/index', + '/about' + ]); + + expect(inserted).toBe(4); + expect( + patched.routes.some((r: { dest?: string }) => r.dest === '/index.html.md') + ).toBe(true); + }); + it('throws when the filesystem handler is missing', () => { expect(() => patchConfig({ version: 3, routes: [] }, ['/about']) From a51cb2b3e457a1d7df9b82037e39540ab7c4ff1c Mon Sep 17 00:00:00 2001 From: Robert Wagner Date: Sun, 23 Aug 2026 22:40:03 -0400 Subject: [PATCH 3/3] Fix astro check type error in patchConfig test astro check type-checks test files; baseConfig's inferred routes type did not allow the 'has' property used by the new regression test. Co-Authored-By: Claude Fable 5 --- tests/unit/vercel-md-negotiation.test.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/unit/vercel-md-negotiation.test.ts b/tests/unit/vercel-md-negotiation.test.ts index a39ba2f..b1d7bd5 100644 --- a/tests/unit/vercel-md-negotiation.test.ts +++ b/tests/unit/vercel-md-negotiation.test.ts @@ -84,7 +84,17 @@ describe('vercel-md-negotiation', () => { }); describe('patchConfig', () => { - const baseConfig = () => ({ + type Route = { + src?: string; + dest?: string; + headers?: Record; + status?: number; + handle?: string; + has?: Array<{ type: string; key: string; value: string }>; + continue?: boolean; + }; + + const baseConfig = (): { version: number; routes: Route[] } => ({ version: 3, routes: [ { src: '^/old-path$', headers: { Location: '/new-path' }, status: 308 },