diff --git a/package.json b/package.json index da229ce..948d2c8 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "license": "MIT", "scripts": { "astro": "astro", - "build": "astro check && astro build", + "build": "astro check && astro build && node scripts/vercel-md-negotiation.mjs", "db:push": "drizzle-kit push", "db:seed": "tsx db/seed.ts", "db:studio": "drizzle-kit studio", diff --git a/scripts/vercel-md-negotiation.mjs b/scripts/vercel-md-negotiation.mjs new file mode 100644 index 0000000..ca18f9b --- /dev/null +++ b/scripts/vercel-md-negotiation.mjs @@ -0,0 +1,205 @@ +/** + * Patches `.vercel/output/config.json` after `astro build` so that pages with + * a prerendered markdown twin (`{path}.html.md`) serve that twin when a client + * asks for it with `Accept: text/markdown`, per https://acceptmarkdown.com. + * + * Both variants of a negotiated URL are stamped with `Vary: Accept` so CDNs + * never serve a cached HTML response to an agent asking for markdown (or vice + * versa). + * + * This has to happen post-build because Vercel checks the filesystem before + * applying `vercel.json` rewrites, so an Accept-based rewrite there would + * never run for prerendered pages. Routes injected before the `filesystem` + * handler in the Build Output API config do run first. + * + * Runs as part of `pnpm build`. No-ops when there is no Vercel build output. + */ + +import { existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join, relative, sep } from 'node:path'; +import process from 'node:process'; +import { pathToFileURL } from 'node:url'; + +const MD_SUFFIX = '.html.md'; + +// Vercel route `has` condition matching any Accept header that mentions +// text/markdown. +const ACCEPT_MARKDOWN = [ + { type: 'header', key: 'accept', value: '.*text/markdown.*' } +]; + +// How many slugs to pack into a single route regex alternation. +const CHUNK_SIZE = 50; + +/** + * Find every prerendered markdown twin in the static output directory and + * return the negotiated URL paths they belong to ('/index' for the homepage). + */ +export function collectMarkdownPaths(staticDir) { + const paths = []; + + const walk = (dir) => { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const full = join(dir, entry.name); + if (entry.isDirectory()) { + walk(full); + } else if (entry.name.endsWith(MD_SUFFIX)) { + const rel = relative(staticDir, full).split(sep).join('/'); + paths.push('/' + rel.slice(0, -MD_SUFFIX.length)); + } + } + }; + + walk(staticDir); + return paths.sort(); +} + +const escapeRegex = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + +const chunk = (items, size) => { + const chunks = []; + for (let i = 0; i < items.length; i += size) { + chunks.push(items.slice(i, i + size)); + } + return chunks; +}; + +/** + * Build the routes that implement the negotiation for the given markdown + * paths. Order matters: Vary stamps first (they `continue`), then the + * Accept-conditional rewrites to the markdown twins. + */ +export function buildNegotiationRoutes(mdPaths) { + const routes = []; + const hasHome = mdPaths.includes('/index'); + const slugChunks = chunk( + mdPaths.filter((p) => p !== '/index').map((p) => escapeRegex(p.slice(1))), + CHUNK_SIZE + ); + + // Stamp Vary: Accept on every negotiated URL, whichever variant ends up + // being served. + if (hasHome) { + routes.push({ src: '^/$', headers: { vary: 'Accept' }, continue: true }); + } + for (const slugs of slugChunks) { + routes.push({ + src: `^/(?:${slugs.join('|')})$`, + headers: { vary: 'Accept' }, + continue: true + }); + } + + // Rewrite to the markdown twin when the client asks for markdown. + if (hasHome) { + routes.push({ + src: '^/$', + has: ACCEPT_MARKDOWN, + dest: '/index.html.md' + }); + } + for (const slugs of slugChunks) { + routes.push({ + src: `^/(${slugs.join('|')})$`, + has: ACCEPT_MARKDOWN, + dest: '/$1.html.md' + }); + } + + return routes; +} + +/** + * Return a copy of the Vercel Build Output config with the negotiation routes + * inserted ahead of the `filesystem` handler. Idempotent: an already patched + * config is returned unchanged. + */ +// Canonical JSON encoding (sorted object keys) so routes can be compared for +// exact equality regardless of key order. +const canonical = (value) => { + if (Array.isArray(value)) { + return value.map(canonical); + } + if (value && typeof value === 'object') { + return Object.fromEntries( + Object.keys(value) + .sort() + .map((key) => [key, canonical(value[key])]) + ); + } + return value; +}; + +const routeKey = (route) => JSON.stringify(canonical(route)); + +export function patchConfig(config, mdPaths) { + if (!Array.isArray(config.routes)) { + throw new Error('config.json has no routes array'); + } + + const negotiationRoutes = buildNegotiationRoutes(mdPaths); + if (negotiationRoutes.length === 0) { + return { config, inserted: 0 }; + } + + // Idempotency: only skip when every generated route is already present + // exactly. A user-added conditional markdown route must not suppress the + // generated set. + const existingRoutes = new Set(config.routes.map(routeKey)); + const alreadyPatched = negotiationRoutes.every((route) => + existingRoutes.has(routeKey(route)) + ); + if (alreadyPatched) { + return { config, inserted: 0 }; + } + + const filesystemIndex = config.routes.findIndex( + (route) => route.handle === 'filesystem' + ); + if (filesystemIndex === -1) { + throw new Error( + 'config.json has no `handle: "filesystem"` route; the Vercel build output format may have changed' + ); + } + + const routes = [ + ...config.routes.slice(0, filesystemIndex), + ...negotiationRoutes, + ...config.routes.slice(filesystemIndex) + ]; + + return { config: { ...config, routes }, inserted: negotiationRoutes.length }; +} + +export function main(outputDir = '.vercel/output') { + const configPath = join(outputDir, 'config.json'); + const staticDir = join(outputDir, 'static'); + + if (!existsSync(configPath) || !existsSync(staticDir)) { + console.log( + `[md-negotiation] No Vercel build output at ${outputDir}, skipping` + ); + return; + } + + const config = JSON.parse(readFileSync(configPath, 'utf-8')); + const mdPaths = collectMarkdownPaths(staticDir); + const { config: patched, inserted } = patchConfig(config, mdPaths); + + if (inserted === 0) { + console.log('[md-negotiation] Nothing to patch'); + return; + } + + writeFileSync(configPath, JSON.stringify(patched, null, 2)); + console.log( + `[md-negotiation] Added ${inserted} routes negotiating markdown for ${mdPaths.length} pages` + ); +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + main(process.argv[2]); +} diff --git a/tests/unit/vercel-md-negotiation.test.ts b/tests/unit/vercel-md-negotiation.test.ts new file mode 100644 index 0000000..b1d7bd5 --- /dev/null +++ b/tests/unit/vercel-md-negotiation.test.ts @@ -0,0 +1,164 @@ +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, describe, expect, it } from 'vitest'; + +import { + buildNegotiationRoutes, + collectMarkdownPaths, + patchConfig +} from '../../scripts/vercel-md-negotiation.mjs'; + +const ACCEPT_MD = [{ type: 'header', key: 'accept', value: '.*text/markdown.*' }]; + +describe('vercel-md-negotiation', () => { + describe('collectMarkdownPaths', () => { + const dir = mkdtempSync(join(tmpdir(), 'starpod-md-')); + + afterAll(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it('finds every .html.md twin, including nested ones', () => { + writeFileSync(join(dir, 'index.html.md'), '# home'); + writeFileSync(join(dir, 'about.html.md'), '# about'); + writeFileSync(join(dir, 'about.html'), ''); + writeFileSync(join(dir, 'llms.txt'), 'llms'); + mkdirSync(join(dir, 'nested')); + writeFileSync(join(dir, 'nested', 'page.html.md'), '# nested'); + + expect(collectMarkdownPaths(dir)).toEqual([ + '/about', + '/index', + '/nested/page' + ]); + }); + }); + + describe('buildNegotiationRoutes', () => { + it('emits Vary stamps and Accept-conditional rewrites', () => { + const routes = buildNegotiationRoutes(['/index', '/about', '/contact']); + + // Vary stamps come first and continue to later routes. + expect(routes[0]).toEqual({ + src: '^/$', + headers: { vary: 'Accept' }, + continue: true + }); + expect(routes[1]).toEqual({ + src: '^/(?:about|contact)$', + headers: { vary: 'Accept' }, + continue: true + }); + + // Rewrites only fire for markdown-accepting clients. + expect(routes[2]).toEqual({ + src: '^/$', + has: ACCEPT_MD, + dest: '/index.html.md' + }); + expect(routes[3]).toEqual({ + src: '^/(about|contact)$', + has: ACCEPT_MD, + dest: '/$1.html.md' + }); + }); + + it('escapes regex metacharacters in slugs', () => { + const routes = buildNegotiationRoutes(['/what+is.this']); + expect(routes[0].src).toBe('^/(?:what\\+is\\.this)$'); + }); + + it('chunks large slug lists into multiple routes', () => { + const paths = Array.from({ length: 120 }, (_, i) => `/episode-${i}`); + const routes = buildNegotiationRoutes(paths); + + // 3 chunks of Vary stamps + 3 chunks of rewrites, no homepage. + expect(routes).toHaveLength(6); + expect(routes.every((r) => r.src.startsWith('^/'))).toBe(true); + }); + + it('returns no routes when there are no markdown twins', () => { + expect(buildNegotiationRoutes([])).toEqual([]); + }); + }); + + describe('patchConfig', () => { + type Route = { + src?: string; + dest?: string; + headers?: Record; + status?: number; + handle?: string; + has?: Array<{ type: string; key: string; value: string }>; + continue?: boolean; + }; + + const baseConfig = (): { version: number; routes: Route[] } => ({ + version: 3, + routes: [ + { src: '^/old-path$', headers: { Location: '/new-path' }, status: 308 }, + { handle: 'filesystem' }, + { src: '^/.*$', dest: '_render' } + ] + }); + + it('inserts negotiation routes before the filesystem handler', () => { + const { config, inserted } = patchConfig(baseConfig(), [ + '/index', + '/about' + ]); + + expect(inserted).toBe(4); + const filesystemIndex = config.routes.findIndex( + (r: { handle?: string }) => r.handle === 'filesystem' + ); + const rewriteIndex = config.routes.findIndex( + (r: { dest?: string }) => r.dest === '/about.html.md' || r.dest === '/$1.html.md' + ); + + // Redirects stay first, negotiation routes go before filesystem. + expect(config.routes[0].status).toBe(308); + expect(rewriteIndex).toBeGreaterThan(0); + expect(rewriteIndex).toBeLessThan(filesystemIndex); + }); + + it('is idempotent', () => { + const { config } = patchConfig(baseConfig(), ['/index', '/about']); + const { config: again, inserted } = patchConfig(config, [ + '/index', + '/about' + ]); + + expect(inserted).toBe(0); + expect(again.routes).toHaveLength(config.routes.length); + }); + + it('still patches when an unrelated conditional markdown route exists', () => { + const config = baseConfig(); + // A user-added Accept-conditional route pointing at a markdown file must + // not suppress the generated negotiation set. + config.routes.unshift({ + src: '^/custom$', + has: [{ type: 'header', key: 'accept', value: '.*text/markdown.*' }], + dest: '/custom-page.html.md' + }); + + const { config: patched, inserted } = patchConfig(config, [ + '/index', + '/about' + ]); + + expect(inserted).toBe(4); + expect( + patched.routes.some((r: { dest?: string }) => r.dest === '/index.html.md') + ).toBe(true); + }); + + it('throws when the filesystem handler is missing', () => { + expect(() => + patchConfig({ version: 3, routes: [] }, ['/about']) + ).toThrow(/filesystem/); + }); + }); +});