feat: give the site a sitemap, a robots file and canonical URLs

src/app/blog/page.tsx

@@ -7,6 +7,7 @@ import { documented } from '@/lib/services/packages.service';
 export const metadata: Metadata = {
   title: 'Blog',
   description: 'Writing about the Axonpack libraries. Release notes live in the changelog.',
+  alternates: { canonical: '/blog/' },
 };
 
 export default function BlogPage() {

src/app/docs/[[...slug]]/page.tsx

@@ -12,7 +12,7 @@ import { getMDXComponents } from '@/components/mdx';
 import type { Metadata } from 'next';
 import { createRelativeLink } from 'fumadocs-ui/mdx';
 import { findNeighbour } from 'fumadocs-core/page-tree';
-import { pageSourceUrl } from '@/lib/shared';
+import { appLongName, canonicalPath, pageSourceUrl } from '@/lib/shared';
 
 export default async function Page(props: PageProps<'/docs/[[...slug]]'>) {
   const params = await props.params;
@@ -64,7 +64,13 @@ export async function generateMetadata(props: PageProps<'/docs/[[...slug]]'>): P
   return {
     title: page.data.title,
     description: page.data.description,
+    alternates: { canonical: canonicalPath(page.url) },
+    // Next replaces the parent's `openGraph` rather than merging into it, so the site name has to
+    // be restated here or a docs page's card loses it.
     openGraph: {
+      type: 'article',
+      siteName: appLongName,
+      url: canonicalPath(page.url),
       images: getPageImageUrl(page).url,
     },
   };

src/app/layout.tsx

@@ -3,7 +3,7 @@ import { SiteFooter } from '@/components/site-footer';
 import './global.css';
 import type { Metadata } from 'next';
 import { Chakra_Petch, JetBrains_Mono } from 'next/font/google';
-import { appLongName, appName, appTagline } from '@/lib/shared';
+import { appLongName, appName, appTagline, siteUrl } from '@/lib/shared';
 
 // Chakra Petch has no variable cut, so every weight the site uses has to be asked for by name.
 const sans = Chakra_Petch({ subsets: ['latin'], weight: ['300', '400', '500', '600', '700'] });
@@ -16,11 +16,18 @@ export const metadata: Metadata = {
   // Every absolute URL in the metadata (OG and Twitter images, canonicals) is resolved against
   // this. It has to be the deployed origin, not localhost, or a shared link previews nothing.
   // The override is for a preview deploy on some other origin.
-  metadataBase: new URL(process.env.NEXT_PUBLIC_SITE_URL ?? 'https://axonpack.github.io'),
+  metadataBase: new URL(siteUrl),
   // Long name when the site names itself, short as a suffix on a page that already has a title.
   title: { default: appLongName, template: `%s — ${appName}` },
   description: appTagline,
   icons: { icon: '/logo.png' },
+  // Inherited by every page, so a page only states what differs. `openGraph.title` is left out on
+  // purpose: Next fills it from the page's own resolved title, and naming it here would freeze the
+  // site name onto every card.
+  openGraph: { type: 'website', siteName: appLongName, locale: 'en_US' },
+  // Without this a card unfurls as the small square thumbnail, which crops the 1200x630 image to
+  // an unreadable centre strip.
+  twitter: { card: 'summary_large_image' },
 };
 
 export default function Layout({ children }: LayoutProps<'/'>) {

src/app/opengraph-image.alt.txt

@@ -0,0 +1 @@
+Axonpack Open Source

src/app/page.tsx

@@ -11,6 +11,7 @@ export const metadata: Metadata = {
   // The headline names the project itself now, so prefixing it with the long name said it twice.
   title: { absolute: `${content.hero.title} ${content.hero.titleAccent}` },
   description: content.hero.lead,
+  alternates: { canonical: '/' },
 };
 
 /** Composes only. Each section is its own component, per the monorepo's CONVENTIONS.md. */

src/app/robots.ts

@@ -0,0 +1,19 @@
+import type { MetadataRoute } from 'next';
+import { siteUrl } from '@/lib/shared';
+
+/**
+ * Open, apart from the search index. `/api/search.json` is one big blob of every page's text, so a
+ * crawler that reads it gets the whole site again with none of the structure.
+ *
+ * `/llms.txt` and `/llms.mdx/` stay allowed on purpose: they exist so an agent's crawler can read
+ * the docs, and disallowing them would shut out the readers they were written for.
+ */
+// `output: 'export'` refuses to build a metadata route that has not said it is static.
+export const dynamic = 'force-static';
+
+export default function robots(): MetadataRoute.Robots {
+  return {
+    rules: { userAgent: '*', allow: '/', disallow: '/api/' },
+    sitemap: new URL('/sitemap.xml', siteUrl).href,
+  };
+}

src/app/sitemap.ts

@@ -0,0 +1,25 @@
+import type { MetadataRoute } from 'next';
+import { source } from '@/lib/source';
+import { canonicalPath, siteUrl } from '@/lib/shared';
+
+/**
+ * Every static page on the site. The docs half comes from fumadocs' own source rather than a
+ * hand-kept list, so writing a page is the only step: there is no second place to forget.
+ *
+ * `output: 'export'` is on, so this runs once at build time and lands in `out/sitemap.xml`.
+ */
+// `output: 'export'` refuses to build a metadata route that has not said it is static.
+export const dynamic = 'force-static';
+
+export default function sitemap(): MetadataRoute.Sitemap {
+  const lastModified = new Date();
+
+  // The docs index is one of `getPages()`, so listing it here as well put it in twice.
+  const fixed = ['/', '/blog'];
+  const docs = source.getPages().map((page) => page.url);
+
+  return [...fixed, ...docs].map((path) => ({
+    url: new URL(canonicalPath(path), siteUrl).href,
+    lastModified,
+  }));
+}

src/lib/shared.ts

@@ -33,3 +33,17 @@ export const docsRepo = {
 
 export const pageSourceUrl = (path: string) =>
   `https://github.com/${docsRepo.user}/${docsRepo.repo}/blob/${docsRepo.branch}/content/docs/${path}`;
+
+/**
+ * The deployed origin. `metadataBase` resolves every relative URL in the metadata against it, and
+ * the sitemap and robots routes need it spelled out, since neither is metadata and neither sees
+ * `metadataBase`. The override is for a preview deploy on some other origin.
+ */
+export const siteUrl = process.env.NEXT_PUBLIC_SITE_URL ?? 'https://axonpack.github.io';
+
+/**
+ * `next.config.mjs` sets `trailingSlash`, so /docs/foo is a redirect and /docs/foo/ is the page.
+ * A canonical or a sitemap entry pointing at the redirect asks a crawler to follow a hop to find
+ * the page it was already told about, so both go through here.
+ */
+export const canonicalPath = (path: string) => (path.endsWith('/') ? path : `${path}/`);