Serve markdown twins via Accept: text/markdown negotiation on Vercel

package.json

@@ -9,7 +9,7 @@
   "license": "MIT",
   "scripts": {
     "astro": "astro",
-    "build": "astro check && astro build",
+    "build": "astro check && astro build && node scripts/vercel-md-negotiation.mjs",
     "db:push": "drizzle-kit push",
     "db:seed": "tsx db/seed.ts",
     "db:studio": "drizzle-kit studio",

scripts/vercel-md-negotiation.mjs

@@ -0,0 +1,205 @@
+/**
+ * Patches `.vercel/output/config.json` after `astro build` so that pages with
+ * a prerendered markdown twin (`{path}.html.md`) serve that twin when a client
+ * asks for it with `Accept: text/markdown`, per https://acceptmarkdown.com.
+ *
+ * Both variants of a negotiated URL are stamped with `Vary: Accept` so CDNs
+ * never serve a cached HTML response to an agent asking for markdown (or vice
+ * versa).
+ *
+ * This has to happen post-build because Vercel checks the filesystem before
+ * applying `vercel.json` rewrites, so an Accept-based rewrite there would
+ * never run for prerendered pages. Routes injected before the `filesystem`
+ * handler in the Build Output API config do run first.
+ *
+ * Runs as part of `pnpm build`. No-ops when there is no Vercel build output.
+ */
+
+import { existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { join, relative, sep } from 'node:path';
+import process from 'node:process';
+import { pathToFileURL } from 'node:url';
+
+const MD_SUFFIX = '.html.md';
+
+// Vercel route `has` condition matching any Accept header that mentions
+// text/markdown.
+const ACCEPT_MARKDOWN = [
+  { type: 'header', key: 'accept', value: '.*text/markdown.*' }
+];
+
+// How many slugs to pack into a single route regex alternation.
+const CHUNK_SIZE = 50;
+
+/**
+ * Find every prerendered markdown twin in the static output directory and
+ * return the negotiated URL paths they belong to ('/index' for the homepage).
+ */
+export function collectMarkdownPaths(staticDir) {
+  const paths = [];
+
+  const walk = (dir) => {
+    for (const entry of readdirSync(dir, { withFileTypes: true })) {
+      const full = join(dir, entry.name);
+      if (entry.isDirectory()) {
+        walk(full);
+      } else if (entry.name.endsWith(MD_SUFFIX)) {
+        const rel = relative(staticDir, full).split(sep).join('/');
+        paths.push('/' + rel.slice(0, -MD_SUFFIX.length));
+      }
+    }
+  };
+
+  walk(staticDir);
+  return paths.sort();
+}
+
+const escapeRegex = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+
+const chunk = (items, size) => {
+  const chunks = [];
+  for (let i = 0; i < items.length; i += size) {
+    chunks.push(items.slice(i, i + size));
+  }
+  return chunks;
+};
+
+/**
+ * Build the routes that implement the negotiation for the given markdown
+ * paths. Order matters: Vary stamps first (they `continue`), then the
+ * Accept-conditional rewrites to the markdown twins.
+ */
+export function buildNegotiationRoutes(mdPaths) {
+  const routes = [];
+  const hasHome = mdPaths.includes('/index');
+  const slugChunks = chunk(
+    mdPaths.filter((p) => p !== '/index').map((p) => escapeRegex(p.slice(1))),
+    CHUNK_SIZE
+  );
+
+  // Stamp Vary: Accept on every negotiated URL, whichever variant ends up
+  // being served.
+  if (hasHome) {
+    routes.push({ src: '^/$', headers: { vary: 'Accept' }, continue: true });
+  }
+  for (const slugs of slugChunks) {
+    routes.push({
+      src: `^/(?:${slugs.join('|')})$`,
+      headers: { vary: 'Accept' },
+      continue: true
+    });
+  }
+
+  // Rewrite to the markdown twin when the client asks for markdown.
+  if (hasHome) {
+    routes.push({
+      src: '^/$',
+      has: ACCEPT_MARKDOWN,
+      dest: '/index.html.md'
+    });
+  }
+  for (const slugs of slugChunks) {
+    routes.push({
+      src: `^/(${slugs.join('|')})$`,
+      has: ACCEPT_MARKDOWN,
+      dest: '/$1.html.md'
+    });
+  }
+
+  return routes;
+}
+
+/**
+ * Return a copy of the Vercel Build Output config with the negotiation routes
+ * inserted ahead of the `filesystem` handler. Idempotent: an already patched
+ * config is returned unchanged.
+ */
+// Canonical JSON encoding (sorted object keys) so routes can be compared for
+// exact equality regardless of key order.
+const canonical = (value) => {
+  if (Array.isArray(value)) {
+    return value.map(canonical);
+  }
+  if (value && typeof value === 'object') {
+    return Object.fromEntries(
+      Object.keys(value)
+        .sort()
+        .map((key) => [key, canonical(value[key])])
+    );
+  }
+  return value;
+};
+
+const routeKey = (route) => JSON.stringify(canonical(route));
+
+export function patchConfig(config, mdPaths) {
+  if (!Array.isArray(config.routes)) {
+    throw new Error('config.json has no routes array');
+  }
+
+  const negotiationRoutes = buildNegotiationRoutes(mdPaths);
+  if (negotiationRoutes.length === 0) {
+    return { config, inserted: 0 };
+  }
+
+  // Idempotency: only skip when every generated route is already present
+  // exactly. A user-added conditional markdown route must not suppress the
+  // generated set.
+  const existingRoutes = new Set(config.routes.map(routeKey));
+  const alreadyPatched = negotiationRoutes.every((route) =>
+    existingRoutes.has(routeKey(route))
+  );
+  if (alreadyPatched) {
+    return { config, inserted: 0 };
+  }
+
+  const filesystemIndex = config.routes.findIndex(
+    (route) => route.handle === 'filesystem'
+  );
+  if (filesystemIndex === -1) {
+    throw new Error(
+      'config.json has no `handle: "filesystem"` route; the Vercel build output format may have changed'
+    );
+  }
+
+  const routes = [
+    ...config.routes.slice(0, filesystemIndex),
+    ...negotiationRoutes,
+    ...config.routes.slice(filesystemIndex)
+  ];
+
+  return { config: { ...config, routes }, inserted: negotiationRoutes.length };
+}
+
+export function main(outputDir = '.vercel/output') {
+  const configPath = join(outputDir, 'config.json');
+  const staticDir = join(outputDir, 'static');
+
+  if (!existsSync(configPath) || !existsSync(staticDir)) {
+    console.log(
+      `[md-negotiation] No Vercel build output at ${outputDir}, skipping`
+    );
+    return;
+  }
+
+  const config = JSON.parse(readFileSync(configPath, 'utf-8'));
+  const mdPaths = collectMarkdownPaths(staticDir);
+  const { config: patched, inserted } = patchConfig(config, mdPaths);
+
+  if (inserted === 0) {
+    console.log('[md-negotiation] Nothing to patch');
+    return;
+  }
+
+  writeFileSync(configPath, JSON.stringify(patched, null, 2));
+  console.log(
+    `[md-negotiation] Added ${inserted} routes negotiating markdown for ${mdPaths.length} pages`
+  );
+}
+
+if (
+  process.argv[1] &&
+  import.meta.url === pathToFileURL(process.argv[1]).href
+) {
+  main(process.argv[2]);
+}

tests/unit/vercel-md-negotiation.test.ts

@@ -0,0 +1,164 @@
+import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterAll, describe, expect, it } from 'vitest';
+
+import {
+  buildNegotiationRoutes,
+  collectMarkdownPaths,
+  patchConfig
+} from '../../scripts/vercel-md-negotiation.mjs';
+
+const ACCEPT_MD = [{ type: 'header', key: 'accept', value: '.*text/markdown.*' }];
+
+describe('vercel-md-negotiation', () => {
+  describe('collectMarkdownPaths', () => {
+    const dir = mkdtempSync(join(tmpdir(), 'starpod-md-'));
+
+    afterAll(() => {
+      rmSync(dir, { recursive: true, force: true });
+    });
+
+    it('finds every .html.md twin, including nested ones', () => {
+      writeFileSync(join(dir, 'index.html.md'), '# home');
+      writeFileSync(join(dir, 'about.html.md'), '# about');
+      writeFileSync(join(dir, 'about.html'), '<html>');
+      writeFileSync(join(dir, 'llms.txt'), 'llms');
+      mkdirSync(join(dir, 'nested'));
+      writeFileSync(join(dir, 'nested', 'page.html.md'), '# nested');
+
+      expect(collectMarkdownPaths(dir)).toEqual([
+        '/about',
+        '/index',
+        '/nested/page'
+      ]);
+    });
+  });
+
+  describe('buildNegotiationRoutes', () => {
+    it('emits Vary stamps and Accept-conditional rewrites', () => {
+      const routes = buildNegotiationRoutes(['/index', '/about', '/contact']);
+
+      // Vary stamps come first and continue to later routes.
+      expect(routes[0]).toEqual({
+        src: '^/$',
+        headers: { vary: 'Accept' },
+        continue: true
+      });
+      expect(routes[1]).toEqual({
+        src: '^/(?:about|contact)$',
+        headers: { vary: 'Accept' },
+        continue: true
+      });
+
+      // Rewrites only fire for markdown-accepting clients.
+      expect(routes[2]).toEqual({
+        src: '^/$',
+        has: ACCEPT_MD,
+        dest: '/index.html.md'
+      });
+      expect(routes[3]).toEqual({
+        src: '^/(about|contact)$',
+        has: ACCEPT_MD,
+        dest: '/$1.html.md'
+      });
+    });
+
+    it('escapes regex metacharacters in slugs', () => {
+      const routes = buildNegotiationRoutes(['/what+is.this']);
+      expect(routes[0].src).toBe('^/(?:what\\+is\\.this)$');
+    });
+
+    it('chunks large slug lists into multiple routes', () => {
+      const paths = Array.from({ length: 120 }, (_, i) => `/episode-${i}`);
+      const routes = buildNegotiationRoutes(paths);
+
+      // 3 chunks of Vary stamps + 3 chunks of rewrites, no homepage.
+      expect(routes).toHaveLength(6);
+      expect(routes.every((r) => r.src.startsWith('^/'))).toBe(true);
+    });
+
+    it('returns no routes when there are no markdown twins', () => {
+      expect(buildNegotiationRoutes([])).toEqual([]);
+    });
+  });
+
+  describe('patchConfig', () => {
+    type Route = {
+      src?: string;
+      dest?: string;
+      headers?: Record<string, string>;
+      status?: number;
+      handle?: string;
+      has?: Array<{ type: string; key: string; value: string }>;
+      continue?: boolean;
+    };
+
+    const baseConfig = (): { version: number; routes: Route[] } => ({
+      version: 3,
+      routes: [
+        { src: '^/old-path$', headers: { Location: '/new-path' }, status: 308 },
+        { handle: 'filesystem' },
+        { src: '^/.*$', dest: '_render' }
+      ]
+    });
+
+    it('inserts negotiation routes before the filesystem handler', () => {
+      const { config, inserted } = patchConfig(baseConfig(), [
+        '/index',
+        '/about'
+      ]);
+
+      expect(inserted).toBe(4);
+      const filesystemIndex = config.routes.findIndex(
+        (r: { handle?: string }) => r.handle === 'filesystem'
+      );
+      const rewriteIndex = config.routes.findIndex(
+        (r: { dest?: string }) => r.dest === '/about.html.md' || r.dest === '/$1.html.md'
+      );
+
+      // Redirects stay first, negotiation routes go before filesystem.
+      expect(config.routes[0].status).toBe(308);
+      expect(rewriteIndex).toBeGreaterThan(0);
+      expect(rewriteIndex).toBeLessThan(filesystemIndex);
+    });
+
+    it('is idempotent', () => {
+      const { config } = patchConfig(baseConfig(), ['/index', '/about']);
+      const { config: again, inserted } = patchConfig(config, [
+        '/index',
+        '/about'
+      ]);
+
+      expect(inserted).toBe(0);
+      expect(again.routes).toHaveLength(config.routes.length);
+    });
+
+    it('still patches when an unrelated conditional markdown route exists', () => {
+      const config = baseConfig();
+      // A user-added Accept-conditional route pointing at a markdown file must
+      // not suppress the generated negotiation set.
+      config.routes.unshift({
+        src: '^/custom$',
+        has: [{ type: 'header', key: 'accept', value: '.*text/markdown.*' }],
+        dest: '/custom-page.html.md'
+      });
+
+      const { config: patched, inserted } = patchConfig(config, [
+        '/index',
+        '/about'
+      ]);
+
+      expect(inserted).toBe(4);
+      expect(
+        patched.routes.some((r: { dest?: string }) => r.dest === '/index.html.md')
+      ).toBe(true);
+    });
+
+    it('throws when the filesystem handler is missing', () => {
+      expect(() =>
+        patchConfig({ version: 3, routes: [] }, ['/about'])
+      ).toThrow(/filesystem/);
+    });
+  });
+});