diff --git a/docs/deployment-runbook.md b/docs/deployment-runbook.md index 8e2a06d03..bd3be77f6 100644 --- a/docs/deployment-runbook.md +++ b/docs/deployment-runbook.md @@ -243,8 +243,15 @@ Worker route remains sufficient rollback. The narrow canary route is more specific than `/doc/*` and would otherwise keep intercepting that version. -The production documentation routes are `/doc`, `/doc/*`, `/doc-latest`, -`/doc-latest/*` and `/robots.txt`. The Worker serves the same checked-in robots +The production route patterns are `www.gecode.dev/doc*` and +`www.gecode.dev/robots.txt*`. Cloudflare matches query strings too, so the +trailing wildcards cover bare entry points with queries. The Worker handles +only `/doc`, `/doc/...`, `/doc-latest`, `/doc-latest/...` and `/robots.txt`; +it passes other matching website paths through to the origin unchanged. +`/doc` redirects to `/documentation.html`, which works during the classic-site +soak and through the Astro fallback afterward. + +The Worker serves the same checked-in robots file as Astro so the indexing policy takes effect before the website cutover. It selects aliases through `LATEST_DOC_VERSION`; it does not copy alias objects. Only `https://www.gecode.dev/doc/latest/...` is indexable. Versioned URLs remain diff --git a/docs/static-documentation-hosting.md b/docs/static-documentation-hosting.md index a4cc6af08..c1234b02c 100644 --- a/docs/static-documentation-hosting.md +++ b/docs/static-documentation-hosting.md @@ -29,8 +29,8 @@ The active Astro site remains free on GitHub Pages. ```text www.gecode.dev/* GitHub Pages (Astro) -www.gecode.dev/doc/* Cloudflare Worker route -www.gecode.dev/doc-latest/* Cloudflare Worker route +www.gecode.dev/doc* Cloudflare Worker route +www.gecode.dev/robots.txt* Cloudflare Worker route | v private R2 bucket @@ -221,7 +221,9 @@ versioned key in R2. - Proxy the `www` record while retaining its GitHub Pages CNAME origin. - Test the staging custom domain, then deploy the checked-in canary environment. - After the canary passes, install the production routes for - `www.gecode.dev/doc/*` and `www.gecode.dev/doc-latest/*`. + `www.gecode.dev/doc*` and `www.gecode.dev/robots.txt*`. The trailing wildcard + includes query strings; the Worker passes similarly prefixed website paths + outside its documentation namespace to the origin unchanged. - Remove the canary after the production smoke test so its more-specific route no longer intercepts the selected version. - Compare status, body digest, MIME type, cache headers, and range behavior diff --git a/scripts/docs/smoke-worker.mjs b/scripts/docs/smoke-worker.mjs index a7ef83e6b..e42e5c6d6 100644 --- a/scripts/docs/smoke-worker.mjs +++ b/scripts/docs/smoke-worker.mjs @@ -46,8 +46,20 @@ for (const prefix of immutableOnly ? [`/doc/${version}`] : [`/doc/${version}`, " if (!immutableOnly) { await check("/doc?smoke=1", 308, {}, (response) => { - assert.equal(response.headers.get("location"), `${origin}/documentation/?smoke=1`); + assert.equal(response.headers.get("location"), `${origin}/documentation.html?smoke=1`); }); + await check("/robots.txt?smoke=1", 200, {}, async (response) => { + assert.match(response.headers.get("content-type"), /text\/plain/); + const body = await response.text(); + assert.doesNotMatch(body, /^Disallow:\s*\/doc(?:\/latest|-latest)/m); + assert.match(body, /^Sitemap: https:\/\/www\.gecode\.dev\/doc\/sitemap\.xml$/m); + }); + if (production) { + await check("/documentation.html?smoke=1", 200, { redirect: "follow" }, (response) => { + assertNoindex(response, false); + assert.equal(response.headers.get("x-gecode-documentation-version"), null); + }); + } } await check(`/doc/${version}/reference/doxygen.css`, 200, {}, (response) => { assert.match(response.headers.get("content-type"), /text\/css/); diff --git a/workers/docs/README.md b/workers/docs/README.md index 1e82d95d8..dc7ff4e82 100644 --- a/workers/docs/README.md +++ b/workers/docs/README.md @@ -21,6 +21,13 @@ version URLs, including PDFs, carry `X-Robots-Tag: noindex`. The `noindex`; staging documentation is also `noindex`. These paths remain crawlable so search engines can read the indexing headers. +Production routes use `doc*` and `robots.txt*` because Cloudflare matches query +strings against route patterns. Requests outside the exact documentation +namespaces, such as `/documentation.html`, pass through to the production +origin without documentation indexing headers. Unknown staging paths return +404, avoiding a fetch back into the custom-domain Worker. The `/doc` landing +redirect uses `/documentation.html`, which exists before and after Astro. + ## Local validation Run all Worker integration tests and compile the production configuration: diff --git a/workers/docs/src/index.test.ts b/workers/docs/src/index.test.ts index 56bffd66d..93eff3b79 100644 --- a/workers/docs/src/index.test.ts +++ b/workers/docs/src/index.test.ts @@ -317,7 +317,53 @@ describe("documentation worker", () => { expect(response.headers.get("location")).toBe(`${base}${path}/?view=1`); } expect((await request("/doc/6.4.0/missing-directory")).status).toBe(404); - expect((await request("/doc")).headers.get("location")).toBe(`${base}/documentation/`); + for (const path of ["/doc", "/doc/", "/doc?smoke=1", "/doc/?smoke=1"]) { + const response = await request(path); + expect(response.status).toBe(308); + expect(response.headers.get("location")).toBe(`${base}/documentation.html${path.includes("?") ? "?smoke=1" : ""}`); + } + }); + + it("passes neighboring website paths through without changing requests or responses", async () => { + const originFetch = vi.spyOn(globalThis, "fetch"); + try { + for (const [path, method, status] of [ + ["/documentation.html?smoke=1", "GET", 200], + ["/documentation/", "HEAD", 404], + ["/documents/submit?draft=1", "POST", 201], + ["/doc-latest-news.html", "GET", 200], + ["/robots.txt.bak?download=1", "GET", 404], + ] as const) { + const incoming = new Request(`${base}${path}`, { + method, headers: { "X-Request-Test": "preserved" }, + body: method === "POST" ? "submission bytes" : undefined, + }); + const upstream = new Response(method === "HEAD" ? null : "origin content", { + status, + headers: { + "Content-Type": "text/html", + "X-Robots-Tag": "index, follow", + Link: '; rel="canonical"', + }, + }); + originFetch.mockResolvedValueOnce(upstream); + const context = createExecutionContext(); + const response = await worker.fetch(incoming, env, context); + await waitOnExecutionContext(context); + expect(originFetch).toHaveBeenLastCalledWith(incoming); + expect(response).toBe(upstream); + expect(response.status).toBe(status); + expect(response.headers.get("x-robots-tag")).toBe("index, follow"); + expect(response.headers.get("link")).toBe('; rel="canonical"'); + if (method === "POST") expect(await incoming.text()).toBe("submission bytes"); + } + const staging = await request("https://docs-staging.gecode.dev/documentation.html"); + expect(staging.status).toBe(404); + expect(staging.headers.get("x-robots-tag")).toBe("noindex"); + expect(originFetch).toHaveBeenCalledTimes(5); + } finally { + originFetch.mockRestore(); + } }); it("returns explicit errors", async () => { @@ -325,7 +371,7 @@ describe("documentation worker", () => { const method = await request("/doc/6.4.0/index.html", { method: "POST" }); expect(method.status).toBe(405); expect(method.headers.get("allow")).toBe("GET, HEAD"); - expect((await request("/doc/%2e%2e/secret")).status).toBe(400); + expect((await request("/doc/6.4.0/%")).status).toBe(400); expect((await request("/doc/6.4.0/%252e%252e/secret")).status).toBe(400); expect((await request("/doc/6.4.0/reference%2fPageChange.html")).status).toBe(400); }); diff --git a/workers/docs/src/index.ts b/workers/docs/src/index.ts index c72cfcd3f..66ab90ff9 100644 --- a/workers/docs/src/index.ts +++ b/workers/docs/src/index.ts @@ -185,7 +185,7 @@ async function serve(request: Request, env: Env, context: ExecutionContext): Pro destination.pathname = pathname; return Response.redirect(destination.href, 308); }; - if (url.pathname === "/doc" || url.pathname === "/doc/") return redirect("/documentation/"); + if (url.pathname === "/doc" || url.pathname === "/doc/") return redirect("/documentation.html"); const resolved = resolvePath(url.pathname, env.LATEST_DOC_VERSION); if (!resolved) return errorResponse(400, "Invalid documentation path"); @@ -285,6 +285,18 @@ async function serve(request: Request, env: Env, context: ExecutionContext): Pro export default { async fetch(request: Request, env: Env, context: ExecutionContext): Promise { + const url = new URL(request.url); + const pathname = url.pathname; + const ownsPath = pathname === "/robots.txt" + || pathname === "/doc" || pathname.startsWith("/doc/") + || pathname === "/doc-latest" || pathname.startsWith("/doc-latest/"); + // Wildcard routes also receive /documentation.html and similarly named + // website paths. Leave their origin response and indexing headers intact. + if (!ownsPath) { + if (url.hostname === "www.gecode.dev") return fetch(request); + return applyIndexingPolicy(request, errorResponse(404, "Page not found"), env); + } + let response: Response; try { response = await serve(request, env, context); diff --git a/workers/docs/wrangler.jsonc b/workers/docs/wrangler.jsonc index 48fa789df..663e893fc 100644 --- a/workers/docs/wrangler.jsonc +++ b/workers/docs/wrangler.jsonc @@ -51,23 +51,11 @@ "observability": { "enabled": true }, "routes": [ { - "pattern": "www.gecode.dev/robots.txt", + "pattern": "www.gecode.dev/robots.txt*", "zone_name": "gecode.dev" }, { - "pattern": "www.gecode.dev/doc", - "zone_name": "gecode.dev" - }, - { - "pattern": "www.gecode.dev/doc/*", - "zone_name": "gecode.dev" - }, - { - "pattern": "www.gecode.dev/doc-latest", - "zone_name": "gecode.dev" - }, - { - "pattern": "www.gecode.dev/doc-latest/*", + "pattern": "www.gecode.dev/doc*", "zone_name": "gecode.dev" } ],