diff --git a/docs/deployment-runbook.md b/docs/deployment-runbook.md
index 8e2a06d03..bd3be77f6 100644
--- a/docs/deployment-runbook.md
+++ b/docs/deployment-runbook.md
@@ -243,8 +243,15 @@ Worker route remains sufficient rollback.
The narrow canary route is more specific than `/doc/*` and would otherwise
keep intercepting that version.
-The production documentation routes are `/doc`, `/doc/*`, `/doc-latest`,
-`/doc-latest/*` and `/robots.txt`. The Worker serves the same checked-in robots
+The production route patterns are `www.gecode.dev/doc*` and
+`www.gecode.dev/robots.txt*`. Cloudflare matches query strings too, so the
+trailing wildcards cover bare entry points with queries. The Worker handles
+only `/doc`, `/doc/...`, `/doc-latest`, `/doc-latest/...` and `/robots.txt`;
+it passes other matching website paths through to the origin unchanged.
+`/doc` redirects to `/documentation.html`, which works during the classic-site
+soak and through the Astro fallback afterward.
+
+The Worker serves the same checked-in robots
file as Astro so the indexing policy takes effect before the website cutover.
It selects aliases through `LATEST_DOC_VERSION`; it does not copy alias objects.
Only `https://www.gecode.dev/doc/latest/...` is indexable. Versioned URLs remain
diff --git a/docs/static-documentation-hosting.md b/docs/static-documentation-hosting.md
index a4cc6af08..c1234b02c 100644
--- a/docs/static-documentation-hosting.md
+++ b/docs/static-documentation-hosting.md
@@ -29,8 +29,8 @@ The active Astro site remains free on GitHub Pages.
```text
www.gecode.dev/* GitHub Pages (Astro)
-www.gecode.dev/doc/* Cloudflare Worker route
-www.gecode.dev/doc-latest/* Cloudflare Worker route
+www.gecode.dev/doc* Cloudflare Worker route
+www.gecode.dev/robots.txt* Cloudflare Worker route
|
v
private R2 bucket
@@ -221,7 +221,9 @@ versioned key in R2.
- Proxy the `www` record while retaining its GitHub Pages CNAME origin.
- Test the staging custom domain, then deploy the checked-in canary environment.
- After the canary passes, install the production routes for
- `www.gecode.dev/doc/*` and `www.gecode.dev/doc-latest/*`.
+ `www.gecode.dev/doc*` and `www.gecode.dev/robots.txt*`. The trailing wildcard
+ includes query strings; the Worker passes similarly prefixed website paths
+ outside its documentation namespace to the origin unchanged.
- Remove the canary after the production smoke test so its more-specific route
no longer intercepts the selected version.
- Compare status, body digest, MIME type, cache headers, and range behavior
diff --git a/scripts/docs/smoke-worker.mjs b/scripts/docs/smoke-worker.mjs
index a7ef83e6b..e42e5c6d6 100644
--- a/scripts/docs/smoke-worker.mjs
+++ b/scripts/docs/smoke-worker.mjs
@@ -46,8 +46,20 @@ for (const prefix of immutableOnly ? [`/doc/${version}`] : [`/doc/${version}`, "
if (!immutableOnly) {
await check("/doc?smoke=1", 308, {}, (response) => {
- assert.equal(response.headers.get("location"), `${origin}/documentation/?smoke=1`);
+ assert.equal(response.headers.get("location"), `${origin}/documentation.html?smoke=1`);
});
+ await check("/robots.txt?smoke=1", 200, {}, async (response) => {
+ assert.match(response.headers.get("content-type"), /text\/plain/);
+ const body = await response.text();
+ assert.doesNotMatch(body, /^Disallow:\s*\/doc(?:\/latest|-latest)/m);
+ assert.match(body, /^Sitemap: https:\/\/www\.gecode\.dev\/doc\/sitemap\.xml$/m);
+ });
+ if (production) {
+ await check("/documentation.html?smoke=1", 200, { redirect: "follow" }, (response) => {
+ assertNoindex(response, false);
+ assert.equal(response.headers.get("x-gecode-documentation-version"), null);
+ });
+ }
}
await check(`/doc/${version}/reference/doxygen.css`, 200, {}, (response) => {
assert.match(response.headers.get("content-type"), /text\/css/);
diff --git a/workers/docs/README.md b/workers/docs/README.md
index 1e82d95d8..dc7ff4e82 100644
--- a/workers/docs/README.md
+++ b/workers/docs/README.md
@@ -21,6 +21,13 @@ version URLs, including PDFs, carry `X-Robots-Tag: noindex`. The
`noindex`; staging documentation is also `noindex`. These paths remain
crawlable so search engines can read the indexing headers.
+Production routes use `doc*` and `robots.txt*` because Cloudflare matches query
+strings against route patterns. Requests outside the exact documentation
+namespaces, such as `/documentation.html`, pass through to the production
+origin without documentation indexing headers. Unknown staging paths return
+404, avoiding a fetch back into the custom-domain Worker. The `/doc` landing
+redirect uses `/documentation.html`, which exists before and after Astro.
+
## Local validation
Run all Worker integration tests and compile the production configuration:
diff --git a/workers/docs/src/index.test.ts b/workers/docs/src/index.test.ts
index 56bffd66d..93eff3b79 100644
--- a/workers/docs/src/index.test.ts
+++ b/workers/docs/src/index.test.ts
@@ -317,7 +317,53 @@ describe("documentation worker", () => {
expect(response.headers.get("location")).toBe(`${base}${path}/?view=1`);
}
expect((await request("/doc/6.4.0/missing-directory")).status).toBe(404);
- expect((await request("/doc")).headers.get("location")).toBe(`${base}/documentation/`);
+ for (const path of ["/doc", "/doc/", "/doc?smoke=1", "/doc/?smoke=1"]) {
+ const response = await request(path);
+ expect(response.status).toBe(308);
+ expect(response.headers.get("location")).toBe(`${base}/documentation.html${path.includes("?") ? "?smoke=1" : ""}`);
+ }
+ });
+
+ it("passes neighboring website paths through without changing requests or responses", async () => {
+ const originFetch = vi.spyOn(globalThis, "fetch");
+ try {
+ for (const [path, method, status] of [
+ ["/documentation.html?smoke=1", "GET", 200],
+ ["/documentation/", "HEAD", 404],
+ ["/documents/submit?draft=1", "POST", 201],
+ ["/doc-latest-news.html", "GET", 200],
+ ["/robots.txt.bak?download=1", "GET", 404],
+ ] as const) {
+ const incoming = new Request(`${base}${path}`, {
+ method, headers: { "X-Request-Test": "preserved" },
+ body: method === "POST" ? "submission bytes" : undefined,
+ });
+ const upstream = new Response(method === "HEAD" ? null : "origin content", {
+ status,
+ headers: {
+ "Content-Type": "text/html",
+ "X-Robots-Tag": "index, follow",
+ Link: '; rel="canonical"',
+ },
+ });
+ originFetch.mockResolvedValueOnce(upstream);
+ const context = createExecutionContext();
+ const response = await worker.fetch(incoming, env, context);
+ await waitOnExecutionContext(context);
+ expect(originFetch).toHaveBeenLastCalledWith(incoming);
+ expect(response).toBe(upstream);
+ expect(response.status).toBe(status);
+ expect(response.headers.get("x-robots-tag")).toBe("index, follow");
+ expect(response.headers.get("link")).toBe('; rel="canonical"');
+ if (method === "POST") expect(await incoming.text()).toBe("submission bytes");
+ }
+ const staging = await request("https://docs-staging.gecode.dev/documentation.html");
+ expect(staging.status).toBe(404);
+ expect(staging.headers.get("x-robots-tag")).toBe("noindex");
+ expect(originFetch).toHaveBeenCalledTimes(5);
+ } finally {
+ originFetch.mockRestore();
+ }
});
it("returns explicit errors", async () => {
@@ -325,7 +371,7 @@ describe("documentation worker", () => {
const method = await request("/doc/6.4.0/index.html", { method: "POST" });
expect(method.status).toBe(405);
expect(method.headers.get("allow")).toBe("GET, HEAD");
- expect((await request("/doc/%2e%2e/secret")).status).toBe(400);
+ expect((await request("/doc/6.4.0/%")).status).toBe(400);
expect((await request("/doc/6.4.0/%252e%252e/secret")).status).toBe(400);
expect((await request("/doc/6.4.0/reference%2fPageChange.html")).status).toBe(400);
});
diff --git a/workers/docs/src/index.ts b/workers/docs/src/index.ts
index c72cfcd3f..66ab90ff9 100644
--- a/workers/docs/src/index.ts
+++ b/workers/docs/src/index.ts
@@ -185,7 +185,7 @@ async function serve(request: Request, env: Env, context: ExecutionContext): Pro
destination.pathname = pathname;
return Response.redirect(destination.href, 308);
};
- if (url.pathname === "/doc" || url.pathname === "/doc/") return redirect("/documentation/");
+ if (url.pathname === "/doc" || url.pathname === "/doc/") return redirect("/documentation.html");
const resolved = resolvePath(url.pathname, env.LATEST_DOC_VERSION);
if (!resolved) return errorResponse(400, "Invalid documentation path");
@@ -285,6 +285,18 @@ async function serve(request: Request, env: Env, context: ExecutionContext): Pro
export default {
async fetch(request: Request, env: Env, context: ExecutionContext): Promise {
+ const url = new URL(request.url);
+ const pathname = url.pathname;
+ const ownsPath = pathname === "/robots.txt"
+ || pathname === "/doc" || pathname.startsWith("/doc/")
+ || pathname === "/doc-latest" || pathname.startsWith("/doc-latest/");
+ // Wildcard routes also receive /documentation.html and similarly named
+ // website paths. Leave their origin response and indexing headers intact.
+ if (!ownsPath) {
+ if (url.hostname === "www.gecode.dev") return fetch(request);
+ return applyIndexingPolicy(request, errorResponse(404, "Page not found"), env);
+ }
+
let response: Response;
try {
response = await serve(request, env, context);
diff --git a/workers/docs/wrangler.jsonc b/workers/docs/wrangler.jsonc
index 48fa789df..663e893fc 100644
--- a/workers/docs/wrangler.jsonc
+++ b/workers/docs/wrangler.jsonc
@@ -51,23 +51,11 @@
"observability": { "enabled": true },
"routes": [
{
- "pattern": "www.gecode.dev/robots.txt",
+ "pattern": "www.gecode.dev/robots.txt*",
"zone_name": "gecode.dev"
},
{
- "pattern": "www.gecode.dev/doc",
- "zone_name": "gecode.dev"
- },
- {
- "pattern": "www.gecode.dev/doc/*",
- "zone_name": "gecode.dev"
- },
- {
- "pattern": "www.gecode.dev/doc-latest",
- "zone_name": "gecode.dev"
- },
- {
- "pattern": "www.gecode.dev/doc-latest/*",
+ "pattern": "www.gecode.dev/doc*",
"zone_name": "gecode.dev"
}
],