From e07513dc6515d96f15c55a537dd0d8282b7f87c8 Mon Sep 17 00:00:00 2001 From: Mikael Zayenz Lagerkvist Date: Sat, 5 Sep 2026 15:15:16 +0200 Subject: [PATCH 1/3] Correct historical map manifests and verify production documentation deployments --- .github/workflows/workers.yml | 11 +++++++++++ scripts/docs/lib.mjs | 2 +- scripts/docs/lib.test.mjs | 11 ++++++++++- scripts/docs/promotion.test.mjs | 30 ++++++++++++++++++++++++++++++ scripts/docs/remote-lib.mjs | 31 +++++++++++++++++++++++++++---- workers/docs/wrangler.jsonc | 3 +++ 6 files changed, 82 insertions(+), 6 deletions(-) diff --git a/.github/workflows/workers.yml b/.github/workflows/workers.yml index 8057e67bf..0a2335649 100644 --- a/.github/workflows/workers.yml +++ b/.github/workflows/workers.yml @@ -120,6 +120,17 @@ jobs: if: inputs.operation == 'deploy' && inputs.environment == 'canary' && (inputs.worker == 'both' || inputs.worker == 'documentation') run: npx wrangler deploy --env canary --config workers/docs/wrangler.jsonc + - name: Verify production documentation + if: inputs.operation == 'deploy' && inputs.environment == 'production' && (inputs.worker == 'both' || inputs.worker == 'documentation') + run: | + version=$(node -p "JSON.parse(require('fs').readFileSync('workers/docs/wrangler.jsonc', 'utf8')).env.production.vars.LATEST_DOC_VERSION") + for attempt in 1 2 3 4 5 6; do + if node scripts/docs/smoke-worker.mjs https://www.gecode.dev "$version"; then exit 0; fi + if [ "$attempt" -eq 6 ]; then exit 1; fi + echo "Waiting for the production deployment to propagate (attempt $attempt)." + sleep 10 + done + - name: Verify documentation canary if: inputs.operation == 'deploy' && inputs.environment == 'canary' && inputs.worker == 'documentation' run: | diff --git a/scripts/docs/lib.mjs b/scripts/docs/lib.mjs index 830ce853f..7fb82130f 100644 --- a/scripts/docs/lib.mjs +++ b/scripts/docs/lib.mjs @@ -12,7 +12,6 @@ const contentTypes = new Map([ [".jpg", "image/jpeg"], [".js", "text/javascript; charset=utf-8"], [".json", "application/json; charset=utf-8"], - [".map", "application/json; charset=utf-8"], [".pdf", "application/pdf"], [".png", "image/png"], [".svg", "image/svg+xml"], @@ -22,6 +21,7 @@ const contentTypes = new Map([ ]); export function contentType(filename) { + if (/\.(?:[cm]?js|css)\.map$/i.test(filename)) return "application/json; charset=utf-8"; return contentTypes.get(path.extname(filename).toLowerCase()) ?? "application/octet-stream"; } diff --git a/scripts/docs/lib.test.mjs b/scripts/docs/lib.test.mjs index cc12e69bd..380c83c8c 100644 --- a/scripts/docs/lib.test.mjs +++ b/scripts/docs/lib.test.mjs @@ -3,10 +3,19 @@ import { mkdtemp, mkdir, readFile, symlink, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import test from "node:test"; -import { createManifest, sitemapDocuments, validateVersion } from "./lib.mjs"; +import { contentType, createManifest, sitemapDocuments, validateVersion } from "./lib.mjs"; import { loadAndVerifyManifest, validateBuildId, validateRemote } from "./remote-lib.mjs"; import { patchDoxygenHtml } from "./patch-doxygen-html.mjs"; +test("distinguishes JavaScript and CSS source maps from Graphviz image maps", () => { + for (const filename of ["app.js.map", "app.mjs.map", "app.cjs.map", "style.css.map", "APP.JS.MAP"]) { + assert.equal(contentType(filename), "application/json; charset=utf-8"); + } + for (const filename of ["classGecode.map", "directory.MAP", "unknown.map"]) { + assert.equal(contentType(filename), "application/octet-stream"); + } +}); + test("creates a sorted, content-addressed version manifest", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "gecode-manifest-")); await mkdir(path.join(root, "reference")); diff --git a/scripts/docs/promotion.test.mjs b/scripts/docs/promotion.test.mjs index 62da05a5c..86236b0ef 100644 --- a/scripts/docs/promotion.test.mjs +++ b/scripts/docs/promotion.test.mjs @@ -5,10 +5,40 @@ import os from "node:os"; import path from "node:path"; import test from "node:test"; import { createManifest } from "./lib.mjs"; +import { verifyRemoteManifest } from "./remote-lib.mjs"; let hasRclone = false; try { execFileSync("rclone", ["version"], { stdio: "ignore" }); hasRclone = true; } catch {} +test("verifies legacy Graphviz map manifests without accepting arbitrary bytes or corrupted hashes", { skip: !hasRclone && "rclone is not installed" }, async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "gecode-legacy-map-")); + try { + const mapPath = path.join(root, "graph.map"); + const legacyManifest = async (body) => { + await writeFile(mapPath, body); + const manifest = await createManifest(root, "1.3.1"); + manifest.files[0].contentType = "application/json; charset=utf-8"; + return manifest; + }; + for (const body of [ + '\n\n\n', + '\n', + "base referer\nrect class.html 7,8 85,56\n", + ]) { + const manifest = await legacyManifest(body); + await verifyRemoteManifest(root, manifest); + assert.equal(manifest.files[0].contentType, "application/json; charset=utf-8"); + const corrupt = structuredClone(manifest); + corrupt.files[0].sha256 = "0".repeat(64); + await assert.rejects(verifyRemoteManifest(root, corrupt), /SHA-256 manifest/); + } + const arbitrary = await legacyManifest("arbitrary data masquerading as a map\n"); + await assert.rejects(verifyRemoteManifest(root, arbitrary), /historical image-map format/); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + test("promotion preserves completed versions and resumes only matching partial uploads", { skip: !hasRclone && "rclone is not installed" }, async () => { const root = await mkdtemp(path.join(os.tmpdir(), "gecode-promotion-")); try { diff --git a/scripts/docs/remote-lib.mjs b/scripts/docs/remote-lib.mjs index 1f0d673b1..3f1f33f66 100644 --- a/scripts/docs/remote-lib.mjs +++ b/scripts/docs/remote-lib.mjs @@ -2,7 +2,7 @@ import { spawn } from "node:child_process"; import { mkdtemp, readFile, rm } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { createManifest } from "./lib.mjs"; +import { contentType, createManifest } from "./lib.mjs"; export function validateRemote(remote) { if (!/^[A-Za-z0-9_-]+:[^/].*$/.test(remote)) { @@ -73,10 +73,12 @@ export async function verifyRemoteManifest(remotePath, expectedManifest, { allow "--hash", ], { allowMissing: allowPartial }) ?? "[]"); const expectedByPath = new Map(expectedManifest.files.map((file) => [file.path, file])); + const legacyMaps = new Set(); + const mimeBase = (value) => value?.split(";", 1)[0].trim().toLowerCase(); const compatibleContentType = (actual, expected) => { if (expected === "application/octet-stream") return Boolean(actual); - const actualBase = actual?.split(";", 1)[0].trim().toLowerCase(); - const expectedBase = expected.split(";", 1)[0].trim().toLowerCase(); + const actualBase = mimeBase(actual); + const expectedBase = mimeBase(expected); if (actualBase === expectedBase) return true; return new Set([actualBase, expectedBase]).size === 2 && [actualBase, expectedBase].every((value) => value === "text/javascript" || value === "application/javascript"); @@ -87,7 +89,13 @@ export async function verifyRemoteManifest(remotePath, expectedManifest, { allow if (!expected) throw new Error(`Unexpected remote object: ${remoteObject.Path}`); if (remoteObject.Size !== expected.bytes) throw new Error(`Remote size differs: ${remoteObject.Path}`); if (!compatibleContentType(remoteObject.MimeType, expected.contentType)) { - throw new Error(`Remote content type differs for ${remoteObject.Path}: ${remoteObject.MimeType ?? "missing"}`); + if (/\.map$/i.test(expected.path) && contentType(expected.path) === "application/octet-stream" + && mimeBase(expected.contentType) === "application/json" + && mimeBase(remoteObject.MimeType) === "application/octet-stream") { + legacyMaps.add(expected.path); + } else { + throw new Error(`Remote content type differs for ${remoteObject.Path}: ${remoteObject.MimeType ?? "missing"}`); + } } } @@ -102,6 +110,21 @@ export async function verifyRemoteManifest(remotePath, expectedManifest, { allow try { await runRclone(["copy", remotePath, temporaryDirectory, "--checksum", "--fast-list", "--metadata", "--progress"]); const actual = await createManifest(temporaryDirectory, expectedManifest.documentationVersion); + // Historical manifests mislabeled Graphviz image maps as JSON. Accept only + // recognizable map content whose downloaded bytes still match the manifest. + for (const file of actual.files) { + if (!legacyMaps.has(file.path)) continue; + const body = await readFile(path.join(temporaryDirectory, file.path), "utf8"); + const expected = expectedByPath.get(file.path); + if (!/^(?:<(?:map|area)(?:\s|>)|base referer\r?\nrect\s)/.test(body) || file.sha256 !== expected.sha256) { + throw new Error(`Remote tree ${remotePath} does not match the SHA-256 manifest or historical image-map format: ${file.path}`); + } + } + expectedManifest = { + ...expectedManifest, + files: expectedManifest.files.map((file) => legacyMaps.has(file.path) + ? { ...file, contentType: "application/octet-stream" } : file), + }; if (JSON.stringify(actual) !== JSON.stringify(expectedManifest)) { throw new Error(`Remote tree ${remotePath} does not match the SHA-256 manifest`); } diff --git a/workers/docs/wrangler.jsonc b/workers/docs/wrangler.jsonc index 0e4f9ddce..6ba56b536 100644 --- a/workers/docs/wrangler.jsonc +++ b/workers/docs/wrangler.jsonc @@ -4,6 +4,7 @@ "main": "src/index.ts", "compatibility_date": "2026-08-08", "workers_dev": true, + "observability": { "enabled": true }, "routes": [ { "pattern": "docs-staging.gecode.dev", @@ -23,6 +24,7 @@ "canary": { "name": "gecode-documentation-canary", "workers_dev": false, + "observability": { "enabled": true }, "routes": [ { "pattern": "www.gecode.dev/doc/6.4.0", @@ -46,6 +48,7 @@ "production": { "name": "gecode-documentation", "workers_dev": false, + "observability": { "enabled": true }, "routes": [ { "pattern": "www.gecode.dev/doc", From f0d3683a6468012d695895451dc65c3a0d569c2f Mon Sep 17 00:00:00 2001 From: Mikael Zayenz Lagerkvist Date: Sat, 5 Sep 2026 15:22:11 +0200 Subject: [PATCH 2/3] Finish canary cleanup and record verified documentation cutover readiness --- .github/workflows/workers.yml | 8 +- docs/deployment-runbook.md | 23 +++-- docs/migration-readiness-review-2026-09-05.md | 97 ++++++++++++------- package-lock.json | 6 +- scripts/docs/remove-canary.mjs | 23 +++++ 5 files changed, 108 insertions(+), 49 deletions(-) create mode 100644 scripts/docs/remove-canary.mjs diff --git a/.github/workflows/workers.yml b/.github/workflows/workers.yml index 0a2335649..4075e25de 100644 --- a/.github/workflows/workers.yml +++ b/.github/workflows/workers.yml @@ -124,11 +124,11 @@ jobs: if: inputs.operation == 'deploy' && inputs.environment == 'production' && (inputs.worker == 'both' || inputs.worker == 'documentation') run: | version=$(node -p "JSON.parse(require('fs').readFileSync('workers/docs/wrangler.jsonc', 'utf8')).env.production.vars.LATEST_DOC_VERSION") - for attempt in 1 2 3 4 5 6; do + for attempt in 1 2 3 4 5 6 7 8 9 10 11 12; do if node scripts/docs/smoke-worker.mjs https://www.gecode.dev "$version"; then exit 0; fi - if [ "$attempt" -eq 6 ]; then exit 1; fi + if [ "$attempt" -eq 12 ]; then exit 1; fi echo "Waiting for the production deployment to propagate (attempt $attempt)." - sleep 10 + sleep 30 done - name: Verify documentation canary @@ -144,7 +144,7 @@ jobs: - name: Remove documentation canary if: inputs.operation == 'remove-canary' - run: npx wrangler delete --env canary --config workers/docs/wrangler.jsonc --force + run: node scripts/docs/remove-canary.mjs - name: Deploy redirect Worker to staging if: inputs.operation == 'deploy' && inputs.environment == 'staging' && (inputs.worker == 'both' || inputs.worker == 'redirects') diff --git a/docs/deployment-runbook.md b/docs/deployment-runbook.md index 97fcd3701..8ac548bc3 100644 --- a/docs/deployment-runbook.md +++ b/docs/deployment-runbook.md @@ -15,15 +15,16 @@ small Pages fallback files. As checked on 5 September 2026, Cloudflare is authoritative for `gecode.dev`. Cloudflare proxies the apex GitHub Pages A records and the `www` CNAME to `gecode.github.io` with Full (strict) TLS and Always Use HTTPS. GitHub Pages -still serves the production website and documentation; the Astro artifact has -not been deployed. +still serves the active website. The 6.4.0 documentation canary now uses R2; +other versions and aliases await production rollout. Astro has not been deployed. Cloudflare Email Routing is ready. Its managed MX, SPF, and DKIM records are authoritative, both forwarding destinations are verified, and the catch-all rule sends mail through the checked-in `gecode-email-routing` Worker. The private R2 documentation archive and `docs-staging.gecode.dev` Worker custom -domain are live and pass the phase 4 smoke tests. No canary or production -documentation Worker routes exist yet. +domain are live and pass the phase 4 smoke tests. Full R2 verification passed +for all nine versions (52,385 files / 1,138,898,740 bytes). The 6.4.0 canary +passes live checks and its two routes are fail-closed. The `cloudflare-staging`, `cloudflare-canary`, and `cloudflare-production` GitHub environments contain the stable Cloudflare account ID and the dedicated @@ -36,7 +37,8 @@ The [5 September readiness review](migration-readiness-review-2026-09-05.md) records the local fixes, rollback build, verification limits and remaining work. Phases 2–4 below describe the original migration sequence; DNS delegation and infrastructure setup are largely complete. Staging deployment through -GitHub now passes. Continue with clean-checkout website CI and phase 5 checks. +GitHub and clean-checkout website CI pass. The website migration is merged +in PR #7; Pages remains manual. Continue with phase 5 production rollout. ## Cutover overview @@ -116,12 +118,14 @@ The successful [staging run](https://github.com/Gecode/gecode.github.io/actions/ used commit `5ec53f3b111a46d92c42f8b996d79b86dc39cd80` on `codex/verify-docs-staging-20260905`. That branch contains a limited Worker snapshot with a staging-only push workflow and suppresses the old Pages -workflow for its branch. Land the complete website migration separately. +workflow for its branch. The complete website migration was subsequently +merged in [PR #7](https://github.com/Gecode/gecode.github.io/pull/7). Worker tests, dry-run builds, deployment and all HTTP smoke checks passed. The deployed `gecode-documentation-staging` version is `f9295ce3-1641-4727-b3d9-8f0c7f31b016`. The pre-rehearsal version was -`8bfa5cfb-1c96-4cb6-8f57-d5d2bb85080c`. Production and canary zone routes remain -absent, and the classic production pages still return 200. +`8bfa5cfb-1c96-4cb6-8f57-d5d2bb85080c`. The later +[canary run](https://github.com/Gecode/gecode.github.io/actions/runs/33967976199) +passed from merged `main`. Classic production pages still return 200. To replace the token, create a new token with those permissions, set `CLOUDFLARE_API_TOKEN` in all three environments, verify a staging deployment, @@ -359,7 +363,8 @@ Run `python3 scripts/build-classic-rollback.py` from the website checkout with the existing Bundler dependencies installed. It creates `dist/classic-rollback/` and `dist/classic-rollback.tar.gz`, refusing to replace an existing output directory. The 5 September rehearsal produced 91,168,795 -uncompressed bytes. Keep the tarball outside this checkout before cutover; +uncompressed bytes. A verified copy is retained at +`/Users/zayenz/gecode/website-rollback/classic-site-2026-07-15.tar.gz`; restore its contents as the Pages artifact when needed. Before Astro, removing the documentation routes restores the existing origin diff --git a/docs/migration-readiness-review-2026-09-05.md b/docs/migration-readiness-review-2026-09-05.md index 6a71ce1db..f53ceb9d5 100644 --- a/docs/migration-readiness-review-2026-09-05.md +++ b/docs/migration-readiness-review-2026-09-05.md @@ -1,19 +1,17 @@ # Website migration and release readiness -Reviewed and updated on 5 September 2026 in `explore/astro-rework`, based on -`72760fbcf`. Three subagents reviewed deployment, documentation serving and -publication, and the Gecode/MPG release integration. Fixes are in the local -website, MPG and release-support working trees. Existing uncommitted work was -preserved. A limited documentation Worker snapshot was subsequently committed -and pushed on an isolated staging branch, then deployed through GitHub Actions. -The migration branch and production website remain unchanged by that deployment. - -The DNS move is complete. Production still serves Jekyll, and documentation -still comes from the old origin. The immediate code defects have been fixed -locally. Deployment access is now provisioned. The remaining cutover needs -clean-checkout website CI and staged production verification. Future coordinated -releases have a working local preparation slice; their publication coordinator -is unfinished. +Reviewed and updated on 5 September 2026. Three subagents reviewed deployment, +documentation serving and publication, and Gecode/MPG release integration. +The website migration was merged in [PR #7](https://github.com/Gecode/gecode.github.io/pull/7) +as `30ccd9636fb7eabed4b14ed5fea09602652ac0be` after clean-checkout CI passed. +The MPG and release-support producer changes remain local. Existing working +trees were preserved. + +DNS migration, staging verification, full historical R2 verification and the +6.4.0 documentation canary are complete. Jekyll still serves the active website. +The remaining traffic changes are the full documentation routes, a one-day +soak, and the separate Astro cutover. Future coordinated releases have a +working local preparation slice; their publication coordinator is unfinished. The hosting arrangement is Cloudflare DNS/proxy, Workers and private R2, with **GitHub Pages as the Astro origin**. Moving Astro itself to Cloudflare Pages @@ -26,14 +24,14 @@ or Workers is a separate project and is unnecessary for this cutover. | DNS | Cloudflare nameservers `milan` and `tegan` are authoritative; the zone is active. MX and SPF use Cloudflare Email Routing. | | TLS | Cloudflare uses Full (strict) and Always Use HTTPS. Public HTTP redirects to HTTPS and the apex redirects to `www`. GitHub reports `https_enforced: false`; reconcile the origin setting separately. | | Production website | `/download.html` returns 200; `/download/` returns 404. Jekyll remains live. | -| Production documentation | Responses lack the documentation Worker's version header; `/doc/sitemap.xml` returns 404. There are no production or canary zone routes. | +| Production documentation | The 6.4.0 canary passes live HTTP and browser checks. Both narrow routes are fail-closed. Other versions and aliases still use the old origin pending production rollout. | | Staging documentation | The reviewed Worker was deployed through GitHub. Its tests and live HTTP smoke checks pass for immutable 6.4.0, both aliases, canonical links, redirects, static assets and PDF range behavior. | -| R2 | Public `r2.dev` access is disabled. The 14-day lifecycle applies only to staging. Nine local historical manifests exist; the complete bucket was not rehashed during this review. | +| R2 | Public `r2.dev` access is disabled. The 14-day lifecycle applies only to staging. All nine archives were fully verified: 52,385 objects / 1,138,898,740 bytes, with no missing objects or hash failures. The 326 historical image-map MIME declarations are documented below. | | Email | Email Routing destinations and catch-all Worker configuration are present. Real delivery and delivery alerts were not exercised. | -| GitHub | The latest production Pages run remains the [15 July deployment](https://github.com/Gecode/gecode.github.io/actions/runs/29430643821). The [5 September staging Worker run](https://github.com/Gecode/gecode.github.io/actions/runs/33964652752) passed. The migration is not on the remote default branch. | +| GitHub | The latest production Pages run remains the [15 July deployment](https://github.com/Gecode/gecode.github.io/actions/runs/29430643821). The [5 September staging Worker run](https://github.com/Gecode/gecode.github.io/actions/runs/33964652752) passed. The migration is merged on `main`; clean-checkout [CI passed](https://github.com/Gecode/gecode.github.io/actions/runs/33967188164), with Pages deployment skipped. | | Deployment credentials | All three Cloudflare environments now contain the account ID and dedicated `gecode-github-workers` token. Production accepts only `main` and `release/*-website` branches; the existing `zayenz` reviewer and approval settings were preserved. | -## Fixes completed locally +## Fixes completed | Area | Result | | --- | --- | @@ -73,8 +71,8 @@ exceed the published-site size limit. The new mail archive, dereferences symlinks and checks required pages and size. A real build succeeded: **91,168,795 bytes** uncompressed. Its archive is -[dist/classic-rollback.tar.gz](../dist/classic-rollback.tar.gz). Retain it outside -this checkout and ephemeral CI storage before cutover. It requires the +[dist/classic-rollback.tar.gz](../dist/classic-rollback.tar.gz). A verified copy is retained outside +this checkout at `/Users/zayenz/gecode/website-rollback/classic-site-2026-07-15.tar.gz`. It requires the production documentation Worker and R2; remove active-site redirects when restoring classic pages. The build was verified locally, not deployed. @@ -85,10 +83,10 @@ Do not delete immutable R2 objects during rollback. ## Remaining cutover sequence -1. **Land the reviewed code.** Review and commit the current website, Worker, - archive and producer changes, including untracked dependencies. Run CI from - clean checkouts. Keep the first Pages deployment manual and retain the - classic rollback artifact. +1. **Website code landed.** The migration, archive and Worker changes are on + `main`, and clean-checkout CI passed. Pages deployment remains manual and + the classic rollback archive is retained outside the checkout. MPG and + release-support changes still need separate commits and producer rehearsal. 2. **Deployment access verified.** The dedicated Worker token is installed in `cloudflare-staging`, `cloudflare-canary` and `cloudflare-production`. Production branch restrictions and the existing reviewer were verified. @@ -98,10 +96,9 @@ Do not delete immutable R2 objects during rollback. 3. **Check operational records.** Confirm historical upload verification, fail-closed documentation routes, fail-open redirect routes, real mail delivery and useful error/delivery alerts. Keep the current DNS delegation. -4. **Canary documentation.** Deploy documentation only on the configured 6.4.0 - canary routes. Check HTML, source view, changelog fragment, CSS/JS/image, PDF - ranges, missing paths, canonical links and sitemap. Ordinary Jekyll pages - must continue to work. +4. **Canary documentation verified.** The 6.4.0 deployment passed HTML, source + folding, changelog fragment, CSS/JS/image, PDF ranges, missing paths, + canonical links and sitemap checks. Ordinary Jekyll pages remain unchanged. 5. **Move documentation.** Deploy production documentation routes, verify historical versions and both aliases, then remove the narrower canary. Allow at least one day with the Jekyll origin still available as fallback. @@ -135,7 +132,40 @@ used commit `5ec53f3b111a46d92c42f8b996d79b86dc39cd80` on live checks in `scripts/docs/smoke-worker.mjs` passed. A bounded retry allows for propagation immediately after deployment; the first attempt observed the old redirect behavior before the new Worker reached that request. Production -routes remain absent and production still serves the classic origin. +routes were absent at that stage. The subsequent canary is recorded below. + + +## Historical archive and canary verification + +Every historical R2 object was downloaded and compared with its manifest using +SHA-256, byte count, response length and MIME type. Complete key sets and sizes +were checked before and after downloading. All 52,385 files passed, across +1.3.1, 2.2.0, 3.7.3, 4.4.0, 5.1.0, 6.0.1, 6.1.1, 6.2.0 and 6.4.0. +Verification records are retained beside the rollback artifact in +`/Users/zayenz/gecode/website-rollback/verification-2026-09-05/`. + +The old manifests wrongly declared 326 Graphviz `.map` files as JSON. R2 serves +them as `application/octet-stream`. Their bytes are intact. Verification accepts +only the known image-map formats (` Date: Sat, 5 Sep 2026 15:34:41 +0200 Subject: [PATCH 3/3] Index only canonical latest documentation while preserving archived URLs --- docs/deployment-runbook.md | 15 +- docs/gecode-release-pipeline.md | 19 ++- docs/migration-readiness-review-2026-09-05.md | 13 ++ docs/static-documentation-hosting.md | 32 ++-- documentation.html | 6 +- robots.txt | 2 - scripts/docs/smoke-worker.mjs | 52 +++++- workers/docs/README.md | 16 +- workers/docs/src/index.test.ts | 156 +++++++++++++++++- workers/docs/src/index.ts | 83 +++++++++- workers/docs/wrangler.jsonc | 4 + 11 files changed, 351 insertions(+), 47 deletions(-) diff --git a/docs/deployment-runbook.md b/docs/deployment-runbook.md index 8ac548bc3..8e2a06d03 100644 --- a/docs/deployment-runbook.md +++ b/docs/deployment-runbook.md @@ -235,14 +235,21 @@ Worker route remains sufficient rollback. documentation Worker. 5. Deploy the production documentation routes and rerun the smoke tests. Confirm `/doc/sitemap.xml` serves the selected version's index and that its - shards contain immutable versioned URLs. + shards contain only canonical `/doc/latest/...` URLs. Confirm versioned + content and `/doc-latest/...` return `X-Robots-Tag: noindex`, including PDFs. + `/robots.txt` must allow documentation crawling so search engines can read + those headers; it continues to exclude the users archive. 6. Use the workflow's `remove-canary` operation with the `canary` environment. The narrow canary route is more specific than `/doc/*` and would otherwise keep intercepting that version. -The production documentation routes are `/doc`, `/doc/*`, `/doc-latest`, and -`/doc-latest/*`. The Worker selects aliases through `LATEST_DOC_VERSION`; it -does not copy alias objects. +The production documentation routes are `/doc`, `/doc/*`, `/doc-latest`, +`/doc-latest/*` and `/robots.txt`. The Worker serves the same checked-in robots +file as Astro so the indexing policy takes effect before the website cutover. +It selects aliases through `LATEST_DOC_VERSION`; it does not copy alias objects. +Only `https://www.gecode.dev/doc/latest/...` is indexable. Versioned URLs remain +available for citations and downloads. Stored versioned sitemaps stay immutable; +the Worker rewrites the selected sitemap responses to latest URLs. Set the documentation routes to fail closed: after documentation leaves the Pages artifact, fail-open traffic would reach a missing origin path. Set the diff --git a/docs/gecode-release-pipeline.md b/docs/gecode-release-pipeline.md index a1bd48cec..a9e540e2a 100644 --- a/docs/gecode-release-pipeline.md +++ b/docs/gecode-release-pipeline.md @@ -85,6 +85,11 @@ Record the website-tool commit and final manifest digest in release state. The final manifest's SHA-256 values identify the approved local release tree; do not describe them as hashes recomputed by R2. +Stored sitemap artifacts may contain immutable version URLs. The Worker +rewrites the selected release's published sitemap index and shards to +`/doc/latest/...`; submit only `/doc/sitemap.xml`. Do not rewrite completed R2 +objects to change indexing policy. + The ordinary release must not invoke this repository's historical Doxygen patcher, staging publisher, staging verifier, or promotion scripts. Those scripts may remain useful for the one-time migration of historical content, @@ -160,8 +165,12 @@ content release, the candidate updates: - the production `LATEST_DOC_VERSION` selection. Keep generated documentation and R2 credentials out of the candidate. Use -`latest` aliases for human entry points, but use immutable version URLs in -release news and citations. +`/doc/latest/...` for human entry points and immutable version URLs in release +news and citations. Only the production latest URLs are indexable and +canonical. Immutable version URLs, including PDFs, and the HTTP 200 +`/doc-latest/...` compatibility alias carry `X-Robots-Tag: noindex`; staging +documentation is also `noindex`. Producer HTML must not contain conflicting +versioned canonical links; the Worker owns the served canonical selection. Run the configured website quality command and validate the release, download, and documentation pages. Before changing either alias, smoke-test @@ -197,6 +206,12 @@ caches may serve the previous version for up to five minutes. Roll back an alias failure by redeploying the previous `LATEST_DOC_VERSION`; immutable versions remain unchanged. +Check indexing headers on HTML and PDFs: only production `/doc/latest/...` +may be indexed. Verify latest HTML canonicals and that the published sitemap +index and every shard contain only latest URLs, including after alias +promotion. Keep immutable and compatibility paths crawlable so crawlers can +observe their `noindex` headers. + Staging and canary Worker deployments belong to the initial migration or to a Worker-code change. They are not required for an ordinary content-only release. diff --git a/docs/migration-readiness-review-2026-09-05.md b/docs/migration-readiness-review-2026-09-05.md index f53ceb9d5..c2d22cfe8 100644 --- a/docs/migration-readiness-review-2026-09-05.md +++ b/docs/migration-readiness-review-2026-09-05.md @@ -167,6 +167,19 @@ Production deployment smoke checks allow for the five-minute alias cache TTL. Email Routing is ready, but delivery-event access requires Zone Analytics Read; real delivery remains unverified. +## Documentation indexing policy + +The requested policy is to index only `https://www.gecode.dev/doc/latest/...`. +Versioned URLs, PDFs included, remain accessible with `X-Robots-Tag: noindex`. +The `/doc-latest/...` compatibility alias and staging hosts also return noindex. +Latest HTML and PDF responses identify their own latest URL as canonical. + +The Worker rewrites the selected sitemap responses to latest URLs while leaving +immutable R2 artifacts intact. The shared robots file permits documentation +crawling, because crawlers must fetch a URL to see its noindex header. The +production Worker also serves `/robots.txt` so this policy can take effect before +Astro is deployed. Search results will change as search engines recrawl the URLs. + ## Gecode and MPG release plan Use the existing [release-support task list](/Users/zayenz/gecode/release-support/.zdev/cf/TASKS.md). diff --git a/docs/static-documentation-hosting.md b/docs/static-documentation-hosting.md index 267c7d6a2..a4cc6af08 100644 --- a/docs/static-documentation-hosting.md +++ b/docs/static-documentation-hosting.md @@ -104,11 +104,17 @@ version. A request for `/doc/latest/reference/index.html` reads promotion a single configuration change. Alias caches converge within five minutes; promotion is not instantaneous. -Treat immutable `/doc//...` URLs as canonical. HTML responses carry an -HTTP `Link` canonical for their versioned URL, including responses reached -through either alias. Allow crawlers to index versioned documentation, exclude -the aliases in `robots.txt`, and publish only versioned URLs in documentation -sitemaps. +Only production `/doc/latest/...` documentation is indexable. Its HTML uses +the corresponding latest URL as its canonical. Every immutable +`/doc//...` response, including PDFs, carries `X-Robots-Tag: noindex`. +The `/doc-latest/...` compatibility alias continues to serve content with HTTP +200 and `noindex`; all staging documentation also carries `noindex`. +Keep these paths crawlable so search engines can read the indexing headers. + +Submit `/doc/sitemap.xml`, which exposes the selected release using only +`/doc/latest/...` sitemap and page URLs. The Worker rewrites stored sitemap +URLs in its responses; immutable R2 sitemap artifacts remain unchanged and +are never submitted as versioned sitemaps. ## Worker behavior @@ -125,7 +131,8 @@ The Worker should implement only the behavior object storage lacks: 8. Return a small branded 404 page without trying extension fallbacks. 9. Add `X-Content-Type-Options: nosniff` and a conservative referrer policy. 10. Emit structured logs for misses, range failures, and unexpected methods. -11. Add an HTTP canonical link to versioned HTML responses. +11. Apply latest-only canonical and indexing headers to HTML, PDFs, and other + documentation responses; rewrite published sitemap URLs to `/doc/latest/`. Versioned objects can use a one-year shared cache because their keys never change. Alias responses should use a five-minute cache and expose the resolved @@ -175,7 +182,8 @@ The local implementation now includes the tested Worker in `workers/docs/`, manifest, inventory, and sitemap generators in `scripts/docs/`, an immutable staged `rclone` publisher with SHA-256 verification, and CI validation. Generated sitemap files are part of each immutable release, and the Worker -serves the selected sitemap at `/doc/sitemap.xml`. Provisioning buckets and +serves the selected sitemap at `/doc/sitemap.xml`, rewriting its URLs to +`/doc/latest/...` without modifying stored objects. Provisioning buckets and credentials, uploading objects, and changing DNS remain operator actions because they affect external infrastructure. @@ -232,8 +240,9 @@ while ordinary website URLs still come from GitHub Pages. commit. Keep a tagged pre-migration commit for provenance. - Consider a separate, carefully announced history rewrite only if clone size remains a problem; it is not required for serving the site. -- Update website links to versioned documentation URLs. Keep `latest` for human - entry points, not for citations or release notes. +- Use `/doc/latest/...` for website documentation entry points. Keep immutable + version URLs for citations and release notes; those URLs remain available + but are not indexable. Exit criterion: the GitHub Pages artifact contains only the Astro site and small first-party downloads. @@ -262,8 +271,11 @@ Automate these checks before changing DNS: - directory index behavior; - `GET`, `HEAD`, conditional requests, and byte ranges; - cache headers for versioned and alias paths; +- latest-only indexing and canonical URLs, with `noindex` on immutable URLs, + PDFs under those URLs, the compatibility alias, and staging; - branded 404s with no accidental bucket listing; -- sitemap size and URL-count limits; +- sitemap size and URL-count limits, with only latest URLs in the submitted + sitemap index and shards; - ordinary `www.gecode.dev` pages bypassing the documentation Worker. Set documentation routes to fail closed after the Pages archive is removed; diff --git a/documentation.html b/documentation.html index 788fb8b17..e51824dc8 100644 --- a/documentation.html +++ b/documentation.html @@ -10,7 +10,7 @@

Modeling and Programming with Gecode

- Modeling and Programming with Gecode + Modeling and Programming with Gecode provides comprehensive documentation of how to model and program with Gecode.

@@ -36,7 +36,7 @@

Modeling and Programming with Gecode

  • Download: - Modeling and Programming with Gecode. + Modeling and Programming with Gecode.
  • License: @@ -57,7 +57,7 @@

    Reference Documentation

    • - + Online HTML ({{ GECODEDOCSTAMP }})
      • diff --git a/robots.txt b/robots.txt index 4380c6862..5ce801733 100644 --- a/robots.txt +++ b/robots.txt @@ -1,6 +1,4 @@ User-agent: * -Disallow: /doc/latest -Disallow: /doc-latest Disallow: /users-archive/ Sitemap: https://www.gecode.dev/sitemap.xml diff --git a/scripts/docs/smoke-worker.mjs b/scripts/docs/smoke-worker.mjs index a3a4f564a..a7ef83e6b 100644 --- a/scripts/docs/smoke-worker.mjs +++ b/scripts/docs/smoke-worker.mjs @@ -6,6 +6,13 @@ assert(base && /^\d+\.\d+\.\d+(?:-[A-Za-z0-9.-]+)?$/.test(version ?? ""), assert(mode === undefined || mode === "--immutable-only", "Unknown smoke-check mode"); const immutableOnly = mode === "--immutable-only"; const origin = new URL(base).origin; +const production = origin === "https://www.gecode.dev"; +const latestBase = "https://www.gecode.dev/doc/latest/"; + +function assertNoindex(response, expected = true) { + const noindex = /\bnoindex\b/i.test(response.headers.get("x-robots-tag") ?? ""); + assert.equal(noindex, expected, "Unexpected documentation indexing policy"); +} async function check(path, status, options = {}, verify = () => {}) { const response = await fetch(`${origin}${path}`, { @@ -13,10 +20,13 @@ async function check(path, status, options = {}, verify = () => {}) { }); try { assert.equal(response.status, status, `${path}: HTTP status`); + if (!production || path.startsWith(`/doc/${version}/`) || path.startsWith("/doc-latest/")) { + assertNoindex(response); + } await verify(response); console.log(`${options.method ?? "GET"} ${path}: ${status}`); } finally { - await response.body?.cancel(); + if (!response.bodyUsed) await response.body?.cancel(); } } @@ -24,8 +34,10 @@ for (const prefix of immutableOnly ? [`/doc/${version}`] : [`/doc/${version}`, " await check(`${prefix}/reference/index.html`, 200, {}, (response) => { assert.equal(response.headers.get("x-gecode-documentation-version"), version); assert.match(response.headers.get("content-type"), /text\/html/); - assert.equal(response.headers.get("link"), - `; rel="canonical"`); + const indexable = production && prefix === "/doc/latest"; + assertNoindex(response, !indexable); + assert.equal(response.headers.get("link"), indexable + ? `<${latestBase}reference/index.html>; rel="canonical"` : null); }); await check(`${prefix}/reference?smoke=1`, 308, {}, (response) => { assert.equal(response.headers.get("location"), `${origin}${prefix}/reference/?smoke=1`); @@ -40,10 +52,34 @@ if (!immutableOnly) { await check(`/doc/${version}/reference/doxygen.css`, 200, {}, (response) => { assert.match(response.headers.get("content-type"), /text\/css/); }); -await check(immutableOnly ? `/doc/${version}/sitemap.xml` : "/doc/sitemap.xml", 200, {}, (response) => { - assert.match(response.headers.get("content-type"), /xml/); - assert.equal(response.headers.get("x-gecode-documentation-version"), version); -}); +async function checkSitemap(indexPath, latestUrls) { + let shardPath; + const locations = (xml) => [...xml.matchAll(/([^<]+)<\/loc>/g)].map((match) => match[1]); + await check(indexPath, 200, {}, async (response) => { + assert.match(response.headers.get("content-type"), /xml/); + assert.equal(response.headers.get("x-gecode-documentation-version"), version); + const xml = await response.text(); + assert.match(xml, / 0, "Documentation sitemap index is empty"); + if (latestUrls) assert(urls.every((url) => url.startsWith(latestBase)), "Sitemap index must advertise only latest URLs"); + const first = new URL(urls[0]); + assert.equal(first.origin, "https://www.gecode.dev"); + shardPath = first.pathname; + assert(shardPath.startsWith(latestUrls ? "/doc/latest/" : `/doc/${version}/`)); + }); + await check(shardPath, 200, {}, async (response) => { + assert.match(response.headers.get("content-type"), /xml/); + assert.equal(response.headers.get("x-gecode-documentation-version"), version); + const xml = await response.text(); + assert.match(xml, / 0, "Documentation sitemap shard is empty"); + if (latestUrls) assert(urls.every((url) => url.startsWith(latestBase)), "Sitemap shard must advertise only latest URLs"); + }); +} +await checkSitemap(`/doc/${version}/sitemap.xml`, false); +if (!immutableOnly) await checkSitemap("/doc/sitemap.xml", true); await check(`/doc/${version}/readiness-missing-page.html`, 404); const pdf = `/doc/${version}/MPG.pdf`; @@ -52,12 +88,14 @@ await check(pdf, 200, { method: "HEAD", headers: { Range: "bytes=0-15" } }, (res assert.match(response.headers.get("content-type"), /application\/pdf/); assert(Number(response.headers.get("content-length")) > 16); assert.equal(response.headers.get("content-range"), null); + assert.equal(response.headers.get("link"), null); etag = response.headers.get("etag"); assert(etag); }); await check(pdf, 206, { headers: { Range: "bytes=0-15", "If-Range": etag } }, (response) => { assert.match(response.headers.get("content-range"), /^bytes 0-15\/\d+$/); assert.equal(response.headers.get("content-length"), "16"); + assert.equal(response.headers.get("link"), null); }); await check(pdf, 200, { headers: { Range: "bytes=0-15", "If-Range": '"stale-readiness-validator"' } }, (response) => { assert.equal(response.headers.get("content-range"), null); diff --git a/workers/docs/README.md b/workers/docs/README.md index af24d371a..1e82d95d8 100644 --- a/workers/docs/README.md +++ b/workers/docs/README.md @@ -14,9 +14,12 @@ The bucket stores releases at its root: The Worker maps `/doc//...` directly to those keys. It maps both `/doc/latest/...` and `/doc-latest/...` to `LATEST_DOC_VERSION`; aliases are not -copied into R2. HTML responses include an HTTP `Link` canonical that points to -the immutable versioned URL. Search engines may index versioned documentation; -`robots.txt` excludes both aliases to avoid duplicate indexing. +copied into R2. Only production `/doc/latest/...` documentation is indexable, +and its HTML canonical points to the corresponding latest URL. All immutable +version URLs, including PDFs, carry `X-Robots-Tag: noindex`. The +`/doc-latest/...` compatibility alias still serves content with HTTP 200 and +`noindex`; staging documentation is also `noindex`. These paths remain +crawlable so search engines can read the indexing headers. ## Local validation @@ -160,6 +163,7 @@ described in the release contract. Existing alias responses can remain cached fo therefore bounded rather than instantaneous. Do not remove an older prefix. The Worker exposes the selected release's `sitemap.xml` at the stable -`/doc/sitemap.xml` URL advertised by `robots.txt`. Sitemap shards and all page -URLs remain immutable, versioned `/doc//...` URLs; aliases are not -listed. +`/doc/sitemap.xml` URL advertised by `robots.txt`. It rewrites sitemap index +and shard responses to use only `/doc/latest/...` URLs. Stored immutable R2 +sitemaps may retain versioned URLs; those artifacts are never submitted +directly and do not need to be republished when the indexing policy changes. diff --git a/workers/docs/src/index.test.ts b/workers/docs/src/index.test.ts index 47337512a..56bffd66d 100644 --- a/workers/docs/src/index.test.ts +++ b/workers/docs/src/index.test.ts @@ -1,5 +1,5 @@ import { env, createExecutionContext, waitOnExecutionContext } from "cloudflare:test"; -import { beforeEach, describe, expect, it } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import worker from "./index"; declare module "cloudflare:test" { @@ -11,9 +11,9 @@ declare module "cloudflare:test" { const base = "https://www.gecode.dev"; -async function request(path: string, init?: RequestInit): Promise { +async function request(path: string, init?: RequestInit, latestVersion = env.LATEST_DOC_VERSION): Promise { const context = createExecutionContext(); - const response = await worker.fetch(new Request(`${base}${path}`, init), env, context); + const response = await worker.fetch(new Request(new URL(path, base), init), { ...env, LATEST_DOC_VERSION: latestVersion }, context); await waitOnExecutionContext(context); return response; } @@ -39,9 +39,8 @@ describe("documentation worker", () => { expect(await page.text()).toBe("0123456789"); expect(page.headers.get("cache-control")).toContain("immutable"); expect(page.headers.get("content-type")).toBe("text/html; charset=utf-8"); - expect(page.headers.get("link")).toBe( - '; rel="canonical"', - ); + expect(page.headers.get("link")).toBeNull(); + expect(page.headers.get("x-robots-tag")).toBe("noindex"); const index = await request("/doc/6.4.0/"); expect(await index.text()).toBe("release home"); @@ -103,9 +102,11 @@ describe("documentation worker", () => { expect(await response.text()).toBe("0123456789"); expect(response.headers.get("cache-control")).toContain("max-age=300"); expect(response.headers.get("x-gecode-documentation-version")).toBe("6.4.0"); - expect(response.headers.get("link")).toBe( - '; rel="canonical"', - ); + const canonical = path.startsWith("/doc/latest/"); + expect(response.headers.get("x-robots-tag")).toBe(canonical ? null : "noindex"); + expect(response.headers.get("link")).toBe(canonical + ? '; rel="canonical"' + : null); }, ); @@ -116,6 +117,125 @@ describe("documentation worker", () => { expect(response.headers.get("content-type")).toBe("application/xml; charset=utf-8"); expect(response.headers.get("cache-control")).toContain("max-age=300"); expect(response.headers.get("link")).toBeNull(); + expect(response.headers.get("x-robots-tag")).toBe("noindex"); + }); + + it("keeps historical versions and staging out of the index", async () => { + await env.DOCS.put("6.2.0/reference/PageChange.html", "historical content", { + httpMetadata: { contentType: "text/html; charset=utf-8" }, + }); + for (const path of [ + "/doc/6.2.0/reference/PageChange.html", + "/doc/6.4.0/reference/PageChange.html", + "https://docs-staging.gecode.dev/doc/latest/reference/PageChange.html", + "https://preview.workers.dev/doc/latest/reference/PageChange.html", + ]) { + const response = await request(path); + expect(response.status).toBe(200); + expect(response.headers.get("x-robots-tag")).toBe("noindex"); + expect(response.headers.get("link")).toBeNull(); + } + }); + + it("applies the current indexing policy even to cached headers", async () => { + const match = vi.spyOn(caches.default, "match").mockImplementation(async () => new Response("cached content", { + headers: { + "Content-Type": "text/html; charset=utf-8", + Link: '; rel="canonical"', + "X-Robots-Tag": "index", + }, + })); + try { + const immutable = await request("/doc/6.4.0/reference/PageChange.html"); + expect(await immutable.text()).toBe("cached content"); + expect(immutable.headers.get("x-robots-tag")).toBe("noindex"); + expect(immutable.headers.get("link")).toBeNull(); + const latest = await request("/doc/latest/reference/PageChange.html"); + expect(await latest.text()).toBe("cached content"); + expect(latest.headers.get("x-robots-tag")).toBeNull(); + expect(latest.headers.get("link")).toBe( + '; rel="canonical"', + ); + } finally { + match.mockRestore(); + } + }); + + it("selects new latest content without reusing the preceding release's cache", async () => { + await request("/doc/latest/reference/PageChange.html"); + await env.DOCS.put("7.0.0/reference/PageChange.html", "new release", { + httpMetadata: { contentType: "text/html; charset=utf-8" }, + }); + const response = await request("/doc/latest/reference/PageChange.html", undefined, "7.0.0"); + expect(await response.text()).toBe("new release"); + expect(response.headers.get("x-gecode-documentation-version")).toBe("7.0.0"); + expect(response.headers.get("link")).toBe( + '; rel="canonical"', + ); + }); + + it("rewrites selected sitemap indexes and shards without changing versioned objects", async () => { + const version = "6.5.0"; + const indexXml = 'https://www.gecode.dev/doc/6.5.0/sitemap-1.xml'; + const shardXml = 'https://www.gecode.dev/doc/6.5.0/reference/PageChange.html'; + await env.DOCS.put(`${version}/sitemap.xml`, indexXml, { httpMetadata: { contentType: "application/xml" } }); + await env.DOCS.put(`${version}/sitemap-1.xml`, shardXml, { httpMetadata: { contentType: "application/xml" } }); + // Entries cached before the indexing-policy change must not leak old URLs. + await caches.default.put(new Request(`${base}/doc/sitemap.xml`), new Response(indexXml, { + headers: { "Cache-Control": "public, max-age=300", "Content-Type": "application/xml" }, + })); + for (const path of ["/doc/sitemap.xml", "/doc/latest/sitemap.xml", "/doc-latest/sitemap.xml"]) { + const response = await request(path, undefined, version); + expect(response.status).toBe(200); + expect(await response.text()).toBe(indexXml.replaceAll(`/doc/${version}/`, "/doc/latest/")); + } + const shard = await request("/doc/latest/sitemap-1.xml", undefined, version); + expect(await shard.text()).toBe(shardXml.replaceAll(`/doc/${version}/`, "/doc/latest/")); + expect(shard.headers.get("x-robots-tag")).toBeNull(); + const historical = await request(`/doc/${version}/sitemap-1.xml`, undefined, version); + expect(await historical.text()).toBe(shardXml); + expect(historical.headers.get("x-robots-tag")).toBe("noindex"); + expect(await (await env.DOCS.get(`${version}/sitemap-1.xml`))!.text()).toBe(shardXml); + }); + + it("uses rewritten sitemap lengths and validators for GET, HEAD and conditional requests", async () => { + const xml = 'https://www.gecode.dev/doc/6.4.0/reference/例.html'; + const stored = await env.DOCS.put("6.4.0/sitemap-1.xml", xml, { httpMetadata: { contentType: "application/xml" } }); + const expected = xml.replaceAll("/doc/6.4.0/", "/doc/latest/"); + const path = "/doc/latest/sitemap-1.xml"; + const response = await request(path); + const etag = response.headers.get("etag")!; + expect(etag).not.toBe(stored.httpEtag); + expect(response.headers.get("content-length")).toBe(String(new TextEncoder().encode(expected).byteLength)); + expect(await response.text()).toBe(expected); + const head = await request(path, { method: "HEAD" }); + expect(head.status).toBe(200); + expect(await head.text()).toBe(""); + expect(head.headers.get("etag")).toBe(etag); + expect(head.headers.get("content-length")).toBe(response.headers.get("content-length")); + const conditional = await request(path, { headers: { "If-None-Match": `W/${etag}` } }); + expect(conditional.status).toBe(304); + expect(await conditional.text()).toBe(""); + expect(conditional.headers.get("etag")).toBe(etag); + const oldValidator = await request(path, { headers: { "If-None-Match": stored.httpEtag } }); + expect(oldValidator.status).toBe(200); + const range = await request(path, { headers: { Range: "bytes=0-10" } }); + expect(range.status).toBe(200); + expect(range.headers.get("content-range")).toBeNull(); + expect(await range.text()).toBe(expected); + }); + + it("serves shared robots rules that allow latest documentation crawling", async () => { + const response = await request("/robots.txt"); + const body = await response.text(); + expect(response.status).toBe(200); + expect(response.headers.get("content-type")).toBe("text/plain; charset=utf-8"); + expect(body).not.toMatch(/^Disallow:\s*\/doc(?:\/latest|-latest)/m); + expect(body).toContain("Sitemap: https://www.gecode.dev/doc/sitemap.xml"); + expect(response.headers.get("cache-control")).toContain("max-age=300"); + const head = await request("/robots.txt", { method: "HEAD" }); + expect(await head.text()).toBe(""); + expect(head.headers.get("content-length")).toBe(String(new TextEncoder().encode(body).byteLength)); }); it("supports HEAD and conditional requests", async () => { @@ -128,6 +248,24 @@ describe("documentation worker", () => { headers: { "If-None-Match": head.headers.get("etag")! }, }); expect(cached.status).toBe(304); + expect(cached.headers.get("x-robots-tag")).toBe("noindex"); + expect(cached.headers.get("link")).toBeNull(); + }); + + it("keeps PDF indexing headers consistent on GET, HEAD, ranges and 304 responses", async () => { + const object = await env.DOCS.put("6.4.0/MPG.pdf", "0123456789", { + httpMetadata: { contentType: "application/pdf" }, + }); + for (const path of ["/doc/6.4.0/MPG.pdf", "/doc/latest/MPG.pdf", "/doc-latest/MPG.pdf", "https://docs-staging.gecode.dev/doc/latest/MPG.pdf"]) { + for (const init of [undefined, { method: "HEAD" }, { headers: { Range: "bytes=0-3" } }, { headers: { "If-None-Match": object.httpEtag } }]) { + const response = await request(path, init); + const canonical = path === "/doc/latest/MPG.pdf"; + expect(response.headers.get("x-robots-tag")).toBe(canonical ? null : "noindex"); + expect(response.headers.get("link")).toBe(canonical + ? '; rel="canonical"' + : null); + } + } }); it("supports byte and suffix ranges", async () => { diff --git a/workers/docs/src/index.ts b/workers/docs/src/index.ts index 577cc5720..c72cfcd3f 100644 --- a/workers/docs/src/index.ts +++ b/workers/docs/src/index.ts @@ -1,3 +1,5 @@ +import robots from "../../../robots.txt"; + export interface Env { DOCS: R2Bucket; LATEST_DOC_VERSION: string; @@ -108,19 +110,76 @@ function applyObjectHeaders(headers: Headers, object: R2Object, resolved: Resolv ? "public, max-age=300, s-maxage=300" : "public, max-age=3600, s-maxage=31536000, immutable", ); - if (object.httpMetadata?.contentType?.toLowerCase().startsWith("text/html")) { - const canonicalPath = resolved.key.split("/").map(encodeURIComponent).join("/"); - headers.set("Link", `; rel="canonical"`); - } for (const [name, value] of Object.entries(securityHeaders)) headers.set(name, value); } +function applyIndexingPolicy(request: Request, response: Response, env: Env): Response { + const url = new URL(request.url); + const decoded = safeDecodePath(url.pathname); + const resolved = resolvePath(url.pathname, env.LATEST_DOC_VERSION); + const indexable = url.origin === "https://www.gecode.dev" + && decoded?.startsWith("/doc/latest/") + && resolved !== null + && [200, 206, 304].includes(response.status); + const headers = new Headers(response.headers); + // Apply this after cache reads too: a previous deployment may have cached + // version-specific canonical links or different indexing instructions. + headers.delete("Link"); + if (indexable) { + headers.delete("X-Robots-Tag"); + if (/^(?:text\/html|application\/pdf)(?:;|$)/i.test(headers.get("Content-Type") ?? "")) { + const relative = resolved.key.slice(resolved.version.length + 1); + const canonicalPath = relative.split("/").map(encodeURIComponent).join("/"); + headers.set("Link", `; rel="canonical"`); + } + } else { + headers.set("X-Robots-Tag", "noindex"); + } + return new Response(request.method === "HEAD" ? null : response.body, { + status: response.status, + statusText: response.statusText, + headers, + }); +} + +async function sitemapResponse(request: Request, object: R2ObjectBody, resolved: ResolvedPath): Promise { + const source = await object.text(); + const text = source.replaceAll( + `https://www.gecode.dev/doc/${resolved.version}/`, + "https://www.gecode.dev/doc/latest/", + ); + const bytes = new TextEncoder().encode(text); + const digest = await crypto.subtle.digest("SHA-256", bytes); + const hash = Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join(""); + const headers = new Headers(); + applyObjectHeaders(headers, object, resolved); + headers.set("Content-Type", "application/xml; charset=utf-8"); + headers.set("Content-Length", String(bytes.byteLength)); + headers.set("ETag", `"${hash}"`); + headers.delete("Accept-Ranges"); + const validators = request.headers.get("If-None-Match")?.split(",").map((value) => value.trim().replace(/^W\//, "")); + if (validators?.some((value) => value === "*" || value === headers.get("ETag"))) { + return new Response(null, { status: 304, headers }); + } + // XML ranges refer to the stored bytes, so serve the complete rewritten XML. + return new Response(request.method === "HEAD" ? null : bytes, { status: 200, headers }); +} + async function serve(request: Request, env: Env, context: ExecutionContext): Promise { if (request.method !== "GET" && request.method !== "HEAD") { return errorResponse(405, "Method not allowed", { Allow: "GET, HEAD" }); } const url = new URL(request.url); + if (url.pathname === "/robots.txt") { + return new Response(request.method === "HEAD" ? null : robots, { + headers: { + "Content-Type": "text/plain; charset=utf-8", + "Content-Length": String(new TextEncoder().encode(robots).byteLength), + "Cache-Control": "public, max-age=300, s-maxage=300", + }, + }); + } const redirect = (pathname: string) => { const destination = new URL(url); destination.pathname = pathname; @@ -146,6 +205,9 @@ async function serve(request: Request, env: Env, context: ExecutionContext): Pro && !request.headers.has("If-None-Match"); const cacheUrl = new URL(request.url); cacheUrl.search = ""; + // Do not reuse old sitemap bodies, or an alias entry from another release. + cacheUrl.searchParams.set("__gecode_docs_policy", "latest-index-v1"); + cacheUrl.searchParams.set("__gecode_docs_version", resolved.version); const cacheKey = new Request(cacheUrl, { method: "GET" }); if (cacheableRequest) { try { @@ -156,6 +218,18 @@ async function serve(request: Request, env: Env, context: ExecutionContext): Pro } } + if (resolved.isAlias && /^sitemap(?:-\d+)?\.xml$/.test(resolved.key.slice(resolved.version.length + 1))) { + const object = await env.DOCS.get(resolved.key); + if (!object) return missingObject(); + const response = await sitemapResponse(request, object, resolved); + if (cacheableRequest) { + context.waitUntil(caches.default.put(cacheKey, response.clone()).catch((error) => { + console.error(JSON.stringify({ event: "cache_write_failed", message: String(error) })); + })); + } + return response; + } + // Range applies only to GET. HEAD describes the complete representation. const rangeHeader = request.method === "GET" ? request.headers.get("Range") : null; if (request.method === "HEAD" || rangeHeader) { @@ -222,6 +296,7 @@ export default { })); response = errorResponse(503, "Documentation is temporarily unavailable", { "Retry-After": "60" }); } + response = applyIndexingPolicy(request, response, env); console.log(JSON.stringify({ method: request.method, path: new URL(request.url).pathname, diff --git a/workers/docs/wrangler.jsonc b/workers/docs/wrangler.jsonc index 6ba56b536..48fa789df 100644 --- a/workers/docs/wrangler.jsonc +++ b/workers/docs/wrangler.jsonc @@ -50,6 +50,10 @@ "workers_dev": false, "observability": { "enabled": true }, "routes": [ + { + "pattern": "www.gecode.dev/robots.txt", + "zone_name": "gecode.dev" + }, { "pattern": "www.gecode.dev/doc", "zone_name": "gecode.dev"