fix(seo): serve static robots.txt pointing to /sitemap.xml

src/app/robots.test.ts

@@ -48,17 +48,11 @@ describe("robots.txt", () => {
     expect(disallow.join(" ")).not.toContain("/reputation");
   });
 
-  it("advertises the chunked sitemap files", async () => {
-    mockFetchHandles.mockResolvedValue({
-      data: Array.from({ length: 60_000 }, (_, i) => `c-${i}`),
-      source: "mock",
-    });
-
-    const result = await robots();
+  it("points to the stable sitemap URL without awaiting backend calls (#715)", () => {
+    const result = robots();
 
-    expect(result.sitemap).toEqual([
-      expect.stringMatching(/\/sitemaps\/0\.xml$/),
-      expect.stringMatching(/\/sitemaps\/1\.xml$/),
-    ]);
+    expect(result.sitemap).toMatch(/\/sitemap\.xml$/);
+    expect(mockFetchBounties).not.toHaveBeenCalled();
+    expect(mockFetchHandles).not.toHaveBeenCalled();
   });
 });

src/app/robots.ts

@@ -1,31 +1,25 @@
 import type { MetadataRoute } from "next";
-import { sitemapUrls } from "@/lib/sitemap";
+import { SITE_URL } from "@/lib/sitemap";
 
 /**
- * robots.txt — the third surface that has to agree with the indexability
- * policy in src/lib/seo-policy.ts (the other two being the sitemap and the
- * per-page `robots` metadata on /reputation/[handle]).
+ * robots.txt — static crawler instructions for MergeFi.
  *
- * Note what is deliberately NOT here: no `Disallow: /reputation/`. Profiles
- * are public and linkable; the ones that haven't opted into indexing carry
- * `noindex, nofollow` in their own page metadata. Blocking them in robots.txt
- * would prevent crawlers from ever fetching the page, which means they would
- * never see that noindex tag — leaving already-indexed URLs in the index
- * indefinitely. Letting crawlers read the page is what makes de-indexing work.
+ * Points to the stable canonical sitemap URL (`/sitemap.xml`) rather than
+ * dynamically awaiting backend-fetched sitemap chunks (#715). This guarantees
+ * crawlers receive robots.txt instantly with zero database or API latency,
+ * even when backend services are down or cold. The /sitemap.xml route and child
+ * chunks handle dynamic generation and caching independently.
  *
- * Dashboard/auth routes stay disallowed: they are per-user surfaces with no
- * value in a search index, and /auth/callback carries a short-lived token in
- * its query string.
+ * Authenticated surfaces (/dashboard/, /auth/callback) remain disallowed.
  */
-export default async function robots(): Promise<MetadataRoute.Robots> {
+export default function robots(): MetadataRoute.Robots {
   return {
     rules: {
       userAgent: "*",
       allow: "/",
       disallow: ["/dashboard/", "/auth/callback"],
     },
-    // One entry per generated child sitemap, so this stays correct as the
-    // entry count grows past a single 50,000-URL file.
-    sitemap: await sitemapUrls(),
+    sitemap: `${SITE_URL}/sitemap.xml`,
   };
 }
+