From f39ead8433ba25d96d1713abcb82927244ae6c26 Mon Sep 17 00:00:00 2001 From: Mikael Zayenz Lagerkvist Date: Sat, 5 Sep 2026 18:40:59 +0200 Subject: [PATCH] Support independent documentation revisions in R2 --- .github/workflows/workers.yml | 65 ++++++++-- docs/gecode-release-pipeline.md | 134 +++++++++++++------- documentation.html | 6 +- scripts/docs/README.md | 70 +++++++++++ scripts/docs/conditional-upload.test.mjs | 113 +++++++++++++++++ scripts/docs/lib.mjs | 12 +- scripts/docs/lib.test.mjs | 13 ++ scripts/docs/promote-version.mjs | 15 ++- scripts/docs/promotion.test.mjs | 80 +++++++++++- scripts/docs/publish-release.mjs | 46 +++++++ scripts/docs/publish-version.mjs | 16 ++- scripts/docs/remote-lib.mjs | 142 +++++++++++++++------ scripts/docs/smoke-worker.mjs | 112 +++++++++++++---- scripts/docs/verify-version.mjs | 13 +- workers/docs/README.md | 149 ++++++++++++----------- workers/docs/src/index.test.ts | 93 +++++++++++++- workers/docs/src/index.ts | 61 ++++++++-- workers/docs/wrangler.jsonc | 21 +++- 18 files changed, 926 insertions(+), 235 deletions(-) create mode 100644 scripts/docs/README.md create mode 100644 scripts/docs/conditional-upload.test.mjs create mode 100644 scripts/docs/publish-release.mjs diff --git a/.github/workflows/workers.yml b/.github/workflows/workers.yml index 7461a84ae..596952ecb 100644 --- a/.github/workflows/workers.yml +++ b/.github/workflows/workers.yml @@ -104,11 +104,27 @@ jobs: - name: Verify staging documentation if: inputs.operation == 'deploy' && inputs.environment == 'staging' && (inputs.worker == 'both' || inputs.worker == 'documentation') run: | - version=$(node -p "JSON.parse(require('fs').readFileSync('workers/docs/wrangler.jsonc', 'utf8')).vars.LATEST_DOC_VERSION") for attempt in 1 2 3 4 5 6; do - if node scripts/docs/smoke-worker.mjs https://docs-staging.gecode.dev "$version"; then exit 0; fi + if node --input-type=module <<'NODE' + import { readFileSync } from 'node:fs'; + import { spawnSync } from 'node:child_process'; + const config = JSON.parse(readFileSync('workers/docs/wrangler.jsonc', 'utf8')); + const vars = config.vars; + const revisions = JSON.parse(vars.DOC_REVISIONS ?? '{}'); + const selected = new Map([[vars.LATEST_DOC_VERSION, revisions[vars.LATEST_DOC_VERSION]]]); + for (const [version, revision] of Object.entries(revisions)) selected.set(version, revision); + let passed = true; + for (const [version, revision] of selected) { + const args = ['scripts/docs/smoke-worker.mjs', 'https://docs-staging.gecode.dev', version]; + if (revision) args.push('--revision', revision); + if (version !== vars.LATEST_DOC_VERSION) args.push('--immutable-only'); + if (spawnSync(process.execPath, args, { stdio: 'inherit' }).status !== 0) passed = false; + } + process.exit(passed ? 0 : 1); + NODE + then exit 0; fi if [ "$attempt" -eq 6 ]; then exit 1; fi - echo "Waiting for the staging deployment to propagate (attempt $attempt)." + echo "Waiting for the documentation deployment and selected revisions to propagate (attempt $attempt)." sleep 10 done @@ -123,22 +139,53 @@ jobs: - name: Verify production documentation if: inputs.operation == 'deploy' && inputs.environment == 'production' && (inputs.worker == 'both' || inputs.worker == 'documentation') run: | - version=$(node -p "JSON.parse(require('fs').readFileSync('workers/docs/wrangler.jsonc', 'utf8')).env.production.vars.LATEST_DOC_VERSION") for attempt in 1 2 3 4 5 6 7 8 9 10 11 12; do - if node scripts/docs/smoke-worker.mjs https://www.gecode.dev "$version"; then exit 0; fi + if node --input-type=module <<'NODE' + import { readFileSync } from 'node:fs'; + import { spawnSync } from 'node:child_process'; + const config = JSON.parse(readFileSync('workers/docs/wrangler.jsonc', 'utf8')); + const vars = config.env.production.vars; + const revisions = JSON.parse(vars.DOC_REVISIONS ?? '{}'); + const selected = new Map([[vars.LATEST_DOC_VERSION, revisions[vars.LATEST_DOC_VERSION]]]); + for (const [version, revision] of Object.entries(revisions)) selected.set(version, revision); + let passed = true; + for (const [version, revision] of selected) { + const args = ['scripts/docs/smoke-worker.mjs', 'https://www.gecode.dev', version]; + if (revision) args.push('--revision', revision); + if (version !== vars.LATEST_DOC_VERSION) args.push('--immutable-only'); + if (spawnSync(process.execPath, args, { stdio: 'inherit' }).status !== 0) passed = false; + } + process.exit(passed ? 0 : 1); + NODE + then exit 0; fi if [ "$attempt" -eq 12 ]; then exit 1; fi - echo "Waiting for the production deployment to propagate (attempt $attempt)." + echo "Waiting for the documentation deployment and selected revisions to propagate (attempt $attempt)." sleep 30 done - name: Verify documentation canary if: inputs.operation == 'deploy' && inputs.environment == 'canary' && inputs.worker == 'documentation' run: | - version=$(node -p "JSON.parse(require('fs').readFileSync('workers/docs/wrangler.jsonc', 'utf8')).env.canary.vars.LATEST_DOC_VERSION") for attempt in 1 2 3 4 5 6; do - if node scripts/docs/smoke-worker.mjs https://www.gecode.dev "$version" --immutable-only; then exit 0; fi + if node --input-type=module <<'NODE' + import { readFileSync } from 'node:fs'; + import { spawnSync } from 'node:child_process'; + const config = JSON.parse(readFileSync('workers/docs/wrangler.jsonc', 'utf8')); + const vars = config.env.canary.vars; + const revisions = JSON.parse(vars.DOC_REVISIONS ?? '{}'); + const selected = new Map([[vars.LATEST_DOC_VERSION, revisions[vars.LATEST_DOC_VERSION]]]); + let passed = true; + for (const [version, revision] of selected) { + const args = ['scripts/docs/smoke-worker.mjs', 'https://www.gecode.dev', version]; + if (revision) args.push('--revision', revision); + args.push('--immutable-only'); + if (spawnSync(process.execPath, args, { stdio: 'inherit' }).status !== 0) passed = false; + } + process.exit(passed ? 0 : 1); + NODE + then exit 0; fi if [ "$attempt" -eq 6 ]; then exit 1; fi - echo "Waiting for the canary deployment to propagate (attempt $attempt)." + echo "Waiting for the documentation deployment and selected revisions to propagate (attempt $attempt)." sleep 10 done diff --git a/docs/gecode-release-pipeline.md b/docs/gecode-release-pipeline.md index a9e540e2a..e4ea93ff1 100644 --- a/docs/gecode-release-pipeline.md +++ b/docs/gecode-release-pipeline.md @@ -1,8 +1,8 @@ # Gecode release documentation pipeline -This document defines the intended steady-state release path after the Astro and -Cloudflare migration. Implementation is tracked in release-support’s `cf-001` -through `cf-005`; the current coordinator is not yet ready for this path. The one-time migration and DNS cutover are covered in +This document defines the coordinated release path and documentation-only +updates for the Astro and Cloudflare website. The one-time migration and DNS +cutover are covered in the [deployment runbook](deployment-runbook.md). Release-support is the publication authority. It coordinates the Gecode @@ -34,6 +34,7 @@ The release job assembles one clean directory: ```text release-tree/ + index.html documentation entry page reference/ Doxygen output modeling/ MPG website MPG.pdf MPG PDF @@ -82,10 +83,11 @@ node website-tools/scripts/docs/create-manifest.mjs \ ``` Record the website-tool commit and final manifest digest in release state. -The final manifest's SHA-256 values identify the approved local release tree; -do not describe them as hashes recomputed by R2. +The final manifest contains SHA-256 and MD5 values. SHA-256 identifies the +approved local bytes; R2 verification normally compares MD5 from object +metadata. Do not describe the local SHA-256 values as hashes recomputed by R2. -Stored sitemap artifacts may contain immutable version URLs. The Worker +Stored sitemap artifacts may contain version URLs. The Worker rewrites the selected release's published sitemap index and shards to `/doc/latest/...`; submit only `/doc/sitemap.xml`. Do not rewrite completed R2 objects to change indexing policy. @@ -122,36 +124,49 @@ does not provide a persistent bucket token that can upload without also being able to list and delete objects, so protect and rotate this credential. Prefer short-lived, prefix-scoped credentials if the workflow can mint them. -Copy the combined tree directly to the immutable `/` prefix with -`rclone copy` and immutable semantics. Never use `sync`: publication must not -delete remote objects. Refuse to overwrite an existing object with different -content. A retry may resume a partial upload only when every existing object -matches the approved local tree. +Choose a new revision ID for this exact combined tree. Copy it directly to +`_revisions///` with the pinned publication tool: -After upload, run `rclone check` against the final prefix. Require all of the -following without downloading the complete tree: - -- the complete relative-path set matches; -- every object size matches; and -- every object MD5 matches between the local and R2 backends. - -These documentation files use ordinary single-part uploads, so a missing -common hash or a size-only result is a publication failure. Do not make a -staging-to-final copy and do not re-download the full tree during an ordinary -release. - -Check any existing completion record before writing release objects. A completed -identical version is verification-only; a conflicting completed version fails. -Run one publisher per version. For R2 completion records, rclone 1.75.0 or newer -supports `--header-upload "If-None-Match: *"`; combine that with -`--ignore-existing` and exact readback. Do not rely on `copyto --immutable` to -protect the manifest. +```sh +node website-tools/scripts/docs/publish-release.mjs \ + --root release-tree --version "$VERSION" --revision "$REVISION" \ + --manifest manifest.json --remote r2:gecode-documentation --confirm-upload +``` -Only after the final prefix passes the check, write -`_manifests/.json` as its completion record and read that exact object -back. An existing conflicting object or manifest stops publication without -deleting or replacing anything. Record the immutable version, object count, -total bytes, final manifest digest, and website-tool commit. +For a tree with many small files, optional environment settings +`RCLONE_TRANSFERS=32` and `RCLONE_CHECKERS=32` increase upload and check +concurrency. They do not change publication semantics or persistent defaults. +An interrupted upload resumes with the same revision and reviewed manifest. + +The tool validates the local manifest, checks any existing completion record, +and refuses conflicting existing objects. Matching partial uploads can resume; +matching completed revisions are verification-only. It does not make a +staging-to-final copy and never uses `sync` or deletes remote objects. + +Every upload uses `If-None-Match: *`, including the completion record. This +requires rclone 1.75 or newer and protects against a concurrent writer after the +initial check. Server-side CopyObject is disabled because its destination +conditions differ from PutObject. Uploads use a single PUT; individual objects +must be smaller than 5 GiB. + +For new MD5 manifests, remote verification uses a fast listing to compare the +complete relative-path set and every object's size and hash. It requests MIME +metadata for one representative of each expected content type, rather than +issuing a serial HEAD request for every file. Live deployment smoke checks also +exercise HTML, CSS, JavaScript, JSON, XML, and PDF responses. This is representative +MIME validation, not a claim to inspect every object's Content-Type header. + +Normal R2 verification does not download the release tree. Historical manifests +without MD5 retain per-object metadata checks. Objects without comparable hashes +and historical image-map format checks use a targeted download fallback rather +than accepting a size-only result. + +Only after verification succeeds, the tool conditionally writes +`_manifests//.json` and reads that exact object back. Record +the version, revision, object count, total bytes, final manifest digest, and +website-tool commit. Omitting `--revision` retains the historical `/` +and `_manifests/.json` layout; documentation updates must use new +revision IDs rather than alter those completed prefixes. ## Prepare the website candidate @@ -162,19 +177,21 @@ content release, the candidate updates: - `src/data/site.ts`; - the transitional `_data/versions.yaml` with equivalent values; - the release news item, using immutable documentation URLs; and -- the production `LATEST_DOC_VERSION` selection. +- the production `DOC_REVISIONS[version]` selection and `LATEST_DOC_VERSION`. Keep generated documentation and R2 credentials out of the candidate. Use -`/doc/latest/...` for human entry points and immutable version URLs in release -news and citations. Only the production latest URLs are indexable and -canonical. Immutable version URLs, including PDFs, and the HTTP 200 +`/doc/latest/...` for human entry points, `/doc//...` for a version's +selected documentation, and `/doc//revisions//...` when a +citation must identify immutable documentation bytes. Only the production latest URLs are indexable and +canonical. Version and explicit revision URLs, including PDFs, and the HTTP 200 `/doc-latest/...` compatibility alias carry `X-Robots-Tag: noindex`; staging documentation is also `noindex`. Producer HTML must not contain conflicting versioned canonical links; the Worker owns the served canonical selection. Run the configured website quality command and validate the release, download, and documentation pages. Before changing either alias, smoke-test -the new immutable routes through the existing production Worker, including +the new explicit `/doc//revisions//...` routes through the +existing production Worker, including reference HTML, modeling assets, `MPG.pdf` range requests, anchors, and 404s. Publish the candidate with exact-base checks: @@ -192,19 +209,19 @@ Publish the candidate with exact-base checks: 6. Verify the Worker's run SHA, workflow identity, dispatch event, protected environment, and inputs. -Then verify both aliases and the resolved-version header: +Then verify both aliases and both selection headers: ```text /doc/latest/reference/index.html /doc-latest/reference/index.html X-Gecode-Documentation-Version: +X-Gecode-Documentation-Revision: ``` Also verify immutable and alias modeling assets, PDF range requests, a -documentation 404, ordinary Astro pages, and the release-news anchor. Alias -caches may serve the previous version for up to five minutes. Roll back an -alias failure by redeploying the previous `LATEST_DOC_VERSION`; immutable -versions remain unchanged. +documentation 404, ordinary Astro pages, and the release-news anchor. Selected version routes and alias caches may serve previous selections for up +to five minutes. Roll back by restoring the previous `DOC_REVISIONS` and, if +changed, `LATEST_DOC_VERSION`. Stored revisions remain unchanged. Check indexing headers on HTML and PDFs: only production `/doc/latest/...` may be indexed. Verify latest HTML canonicals and that the published sitemap @@ -216,6 +233,35 @@ Staging and canary Worker deployments belong to the initial migration or to a Worker-code change. They are not required for an ordinary content-only release. +## Update documentation without releasing Gecode + +Use release-support's `scripts/documentation.py` commands to prepare the +combined tree from an existing reference build and a validated MPG package, +publish a new revision, verify its explicit preview route, and edit the +website's production selection. `select` edits configuration only; it does +not deploy, publish Gecode, publish MPG, change release news, or change +`LATEST_DOC_VERSION` unless `--latest` is explicitly requested. + +Review and land the `DOC_REVISIONS` change on website `main`, then dispatch +`workers.yml` from `main` with `operation=deploy`, `environment=production`, +and `worker=documentation`. The documentation-only CLI does not create an +approved `docs/...-website` deployment branch. Production keeps its existing +`main` and normal `release/-website` ref rules. + +Verify the newly selected version and, if it is latest, both aliases: + +```sh +node scripts/docs/smoke-worker.mjs https://www.gecode.dev "$VERSION" \ + --revision "$REVISION" +``` + +Add `--immutable-only` when updating a non-latest version. The deployment +workflow reads `DOC_REVISIONS`, checks both version and revision headers, and +also checks other selected versions so an old revision cannot pass merely +because the Gecode version is unchanged. HTML, modeling search assets, +reference CSS, sitemaps, exact PDF ranges, indexing policy, and 404s are part +of these checks. + ## Keep website work independent Normal website development and deployment continue while release-support diff --git a/documentation.html b/documentation.html index e51824dc8..5503c0e6d 100644 --- a/documentation.html +++ b/documentation.html @@ -10,7 +10,7 @@

Modeling and Programming with Gecode

- Modeling and Programming with Gecode + Modeling and Programming with Gecode provides comprehensive documentation of how to model and program with Gecode.

@@ -34,6 +34,10 @@

Modeling and Programming with Gecode

    +
  • + Read online: + HTML manual with search and source downloads. +
  • Download: Modeling and Programming with Gecode. diff --git a/scripts/docs/README.md b/scripts/docs/README.md new file mode 100644 index 000000000..f29748fa3 --- /dev/null +++ b/scripts/docs/README.md @@ -0,0 +1,70 @@ +# Documentation publication tools + +Publish a reviewed local tree directly into immutable R2 objects: + +```sh +node scripts/docs/create-manifest.mjs \ + --root /path/to/docs --version 6.4.0 --output /path/to/manifest.json +node scripts/docs/publish-release.mjs \ + --root /path/to/docs --version 6.4.0 --revision 20260905-rst2 \ + --manifest /path/to/manifest.json --remote r2:gecode-documentation +``` + +For archives with many small files, optionally increase rclone concurrency +before running the publication command: + +```sh +export RCLONE_TRANSFERS=32 +export RCLONE_CHECKERS=32 +``` + +These environment settings apply to rclone processes in that shell; the tools +do not change persistent defaults. An interrupted upload can resume with the same revision +and reviewed manifest: existing objects are verified before missing objects +are uploaded. + +The second command validates the local tree and prints the destination. Add +`--confirm-upload` to upload, verify the remote objects, and write the completion +manifest. Publication does not change the worker's selected revision. + +A revision uses `_revisions///` for objects and +`_manifests//.json` for completion. Revision IDs must be a +single safe build identifier: letters, digits, periods, underscores, and +hyphens, beginning with a letter or digit, up to 128 characters. Omitting +`--revision` preserves the existing `/` and `_manifests/.json` +paths. Manifest keys and sitemap URLs continue to describe public +`/doc//...` URLs. + +Verify a completed revision without uploading: + +```sh +node scripts/docs/verify-version.mjs --version 6.4.0 --revision 20260905-rst2 \ + --manifest /path/to/manifest.json --remote r2:gecode-documentation --final +``` + +Manifests contain SHA-256 and MD5 hashes. For new MD5 manifests, remote +verification lists every object's path, size, and hash without requesting +per-object MIME or modification metadata. It then checks actual MIME metadata +for one representative of each expected content type. This avoids thousands of +serial HEAD requests while retaining complete path, size, and hash coverage. +Live deployment smoke checks also exercise the main served content types. + +Historical SHA-256-only manifests retain per-object metadata checks. Objects +without comparable remote hashes use a targeted download fallback; historical +Graphviz image maps also require their existing content-format check. Normal +new publications do not download the tree. + +Publication requires rclone 1.75 or newer. Every upload uses `If-None-Match: *`, +including the completion manifest, so a concurrent writer cannot overwrite an +object between the initial check and upload. Existing objects are verified and +skipped. Server-side copy is disabled, and uploads use a single PutObject; +individual objects must be smaller than 5 GiB. A different manifest cannot +replace a completed revision; choose a new revision ID. + +The historical `publish-version.mjs`, `verify-version.mjs`, and +`promote-version.mjs` commands still support staging. Each accepts optional +`--revision`; its staging prefix becomes +`staging////`. Without that option the original +staging paths remain unchanged. Promotion uses the same conditional upload +protection, streaming through the client instead of using S3 CopyObject. New +release automation should use `publish-release.mjs` directly. diff --git a/scripts/docs/conditional-upload.test.mjs b/scripts/docs/conditional-upload.test.mjs new file mode 100644 index 000000000..f0013fb38 --- /dev/null +++ b/scripts/docs/conditional-upload.test.mjs @@ -0,0 +1,113 @@ +import assert from "node:assert/strict"; +import { spawn, execFileSync } from "node:child_process"; +import { createServer } from "node:http"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { conditionalUploadArgs, publicationPaths, readRemoteFile } from "./remote-lib.mjs"; + +let hasRclone = false; +try { execFileSync("rclone", ["version"], { stdio: "ignore" }); hasRclone = true; } catch {} + +test("revision paths retain legacy prefixes and reject path traversal", () => { + assert.deepEqual(publicationPaths("r2:docs", "6.4.0", { buildId: "job" }), { + production: "r2:docs/6.4.0", manifest: "r2:docs/_manifests/6.4.0.json", staging: "r2:docs/staging/job/6.4.0", + }); + assert.deepEqual(publicationPaths("r2:docs", "6.4.0", { revision: "20260905-rst", buildId: "job" }), { + production: "r2:docs/_revisions/6.4.0/20260905-rst", + manifest: "r2:docs/_manifests/6.4.0/20260905-rst.json", + staging: "r2:docs/staging/job/6.4.0/20260905-rst", + }); + for (const revision of ["../x", "x/y", "", "..", "/x"]) { + assert.throws(() => publicationPaths("r2:docs", "6.4.0", { revision }), /Invalid build ID/); + } + assert.throws(() => conditionalUploadArgs({ files: [{ bytes: 5 * 1024 ** 3 }] }), /smaller than 5 GiB/); +}); + +test("rclone sends an atomic destination condition when an object appears after its initial check", { skip: !hasRclone && "rclone is not installed", timeout: 30_000 }, async () => { + const temporary = await mkdtemp(path.join(os.tmpdir(), "gecode-conditional-put-")); + const requests = []; + let stored = "concurrent writer"; + const server = createServer((request, response) => { + requests.push({ method: request.method, url: request.url, headers: request.headers }); + request.resume(); + if (request.method === "HEAD") { + response.writeHead(request.url === "/bucket" ? 200 : 404); + } else if (request.method === "PUT" && request.url.split("?", 1)[0] === "/bucket/index.html") { + // The HEAD observed no object; another publisher completed before PUT. + if (request.headers["if-none-match"] === "*") { + response.writeHead(412, { "content-type": "application/xml" }); + response.end("PreconditionFailedAlready exists"); + return; + } + stored = "overwritten"; + response.writeHead(200); + } else { + response.writeHead(200, { "content-type": "application/xml" }); + } + response.end(); + }); + await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", resolve); }); + try { + const filename = path.join(temporary, "index.html"); + await writeFile(filename, "new writer"); + const child = spawn("rclone", ["copyto", filename, "race:bucket/index.html", + "--retries", "1", "--low-level-retries", "1", "--s3-no-check-bucket", + ...conditionalUploadArgs({ files: [{ bytes: 10 }] }), + ], { env: { ...process.env, + RCLONE_CONFIG_RACE_TYPE: "s3", RCLONE_CONFIG_RACE_PROVIDER: "Other", + RCLONE_CONFIG_RACE_ACCESS_KEY_ID: "test", RCLONE_CONFIG_RACE_SECRET_ACCESS_KEY: "test", + RCLONE_CONFIG_RACE_ENDPOINT: `http://127.0.0.1:${server.address().port}`, + }, stdio: ["ignore", "ignore", "pipe"] }); + let stderr = ""; + child.stderr.on("data", (chunk) => { stderr += chunk; }); + const status = await new Promise((resolve, reject) => { child.on("close", resolve); child.on("error", reject); }); + assert.notEqual(status, 0, stderr); + const uploads = requests.filter((request) => request.method === "PUT" && request.url.split("?", 1)[0] === "/bucket/index.html"); + assert.ok(uploads.length > 0, JSON.stringify(requests) + stderr); + assert.ok(uploads.every((request) => request.headers["if-none-match"] === "*")); + assert.ok(uploads.every((request) => !request.headers["x-amz-copy-source"])); + assert.equal(stored, "concurrent writer"); + } finally { + await new Promise((resolve) => server.close(resolve)); + await rm(temporary, { recursive: true, force: true }); + } +}); + +test("S3 completion lookup distinguishes a missing virtual prefix from an empty object", { skip: !hasRclone && "rclone is not installed", timeout: 30_000 }, async () => { + const server = createServer((request, response) => { + request.resume(); + const url = new URL(request.url, "http://localhost"); + if (url.pathname === "/bucket/empty.json") { + response.writeHead(200, { "content-length": "0", etag: '"d41d8cd98f00b204e9800998ecf8427e"', "last-modified": "Sat, 05 Sep 2026 12:00:00 GMT" }); + response.end(); + } else if (request.method === "HEAD") { + response.writeHead(404); + response.end(); + } else { + response.writeHead(200, { "content-type": "application/xml" }); + response.end('bucketmissing.json/01000false'); + } + }); + await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", resolve); }); + const configuration = { + RCLONE_CONFIG_LOOKUP_TYPE: "s3", RCLONE_CONFIG_LOOKUP_PROVIDER: "Other", + RCLONE_CONFIG_LOOKUP_ACCESS_KEY_ID: "test", RCLONE_CONFIG_LOOKUP_SECRET_ACCESS_KEY: "test", + RCLONE_CONFIG_LOOKUP_ENDPOINT: `http://127.0.0.1:${server.address().port}`, + }; + const previous = Object.fromEntries(Object.keys(configuration).map((name) => [name, process.env[name]])); + Object.assign(process.env, configuration); + try { + assert.equal(await readRemoteFile("lookup:bucket/missing.json", { allowMissing: true }), null); + await assert.rejects(readRemoteFile("lookup:bucket/missing.json"), /does not exist/); + assert.equal(await readRemoteFile("lookup:bucket/empty.json", { allowMissing: true }), ""); + assert.equal(await readRemoteFile("lookup:bucket/empty.json"), ""); + } finally { + for (const [name, value] of Object.entries(previous)) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + await new Promise((resolve) => server.close(resolve)); + } +}); diff --git a/scripts/docs/lib.mjs b/scripts/docs/lib.mjs index 7fb82130f..3dbcd7241 100644 --- a/scripts/docs/lib.mjs +++ b/scripts/docs/lib.mjs @@ -33,9 +33,13 @@ export function validateVersion(version) { } async function hashFile(filename) { - const hash = createHash("sha256"); - for await (const chunk of createReadStream(filename)) hash.update(chunk); - return hash.digest("hex"); + const sha256 = createHash("sha256"); + const md5 = createHash("md5"); + for await (const chunk of createReadStream(filename)) { + sha256.update(chunk); + md5.update(chunk); + } + return { sha256: sha256.digest("hex"), md5: md5.digest("hex") }; } async function findFiles(root, directory = root) { @@ -67,7 +71,7 @@ export async function createManifest(root, version) { path: relativePath, key: `${version}/${relativePath}`, bytes: stat.size, - sha256: await hashFile(filename), + ...await hashFile(filename), contentType: contentType(filename), }); } diff --git a/scripts/docs/lib.test.mjs b/scripts/docs/lib.test.mjs index 380c83c8c..62bf259b5 100644 --- a/scripts/docs/lib.test.mjs +++ b/scripts/docs/lib.test.mjs @@ -93,3 +93,16 @@ test("patches modern Doxygen HTML idempotently", async () => { assert.match(patched, /href="https:\/\/www\.gecode\.dev\/"/); assert.deepEqual(await patchDoxygenHtml(root), { changed: 0, visited: 1 }); }); + +test("continues accepting historical SHA-256-only manifests", async () => { + const temporary = await mkdtemp(path.join(os.tmpdir(), "gecode-old-manifest-")); + const root = path.join(temporary, "source"); + await mkdir(root); + await writeFile(path.join(root, "index.html"), "legacy"); + const manifest = await createManifest(root, "6.4.0"); + assert.match(manifest.files[0].md5, /^[a-f0-9]{32}$/); + delete manifest.files[0].md5; + const manifestPath = path.join(temporary, "manifest.json"); + await writeFile(manifestPath, JSON.stringify(manifest)); + await loadAndVerifyManifest(manifestPath, root, "6.4.0"); +}); diff --git a/scripts/docs/promote-version.mjs b/scripts/docs/promote-version.mjs index 3f643eb06..73e5abf9d 100644 --- a/scripts/docs/promote-version.mjs +++ b/scripts/docs/promote-version.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node import { readFile } from "node:fs/promises"; import { validateVersion } from "./lib.mjs"; -import { readRemoteFile, requireConditionalUploadSupport, runRclone, validateBuildId, validateRemote, verifyRemoteManifest } from "./remote-lib.mjs"; +import { conditionalUploadArgs, publicationPaths, readRemoteFile, requireConditionalUploadSupport, runRclone, verifyRemoteManifest } from "./remote-lib.mjs"; function option(name) { const index = process.argv.indexOf(name); @@ -12,8 +12,9 @@ const version = option("--version"); const remote = option("--remote"); const manifestPath = option("--manifest"); const buildId = option("--build-id"); +const revision = option("--revision"); if (!version || !remote || !manifestPath || !buildId || !process.argv.includes("--confirm-promotion")) { - console.error("Usage: promote-version.mjs --version --manifest --build-id --remote --confirm-promotion"); + console.error("Usage: promote-version.mjs --version --manifest --build-id --remote [--revision ] --confirm-promotion"); process.exit(2); } validateVersion(version); @@ -21,10 +22,8 @@ validateVersion(version); const manifestText = await readFile(manifestPath, "utf8"); const manifest = JSON.parse(manifestText); if (manifest.documentationVersion !== version) throw new Error("Manifest and requested versions do not match"); -const base = validateRemote(remote); -const staging = `${base}/staging/${validateBuildId(buildId)}/${version}`; -const production = `${base}/${version}`; -const remoteManifest = `${base}/_manifests/${version}.json`; +const { staging, production, manifest: remoteManifest } = publicationPaths(remote, version, { revision, buildId }); +const uploadArgs = conditionalUploadArgs(manifest); await requireConditionalUploadSupport(); const completed = await readRemoteFile(remoteManifest, { allowMissing: true }); @@ -39,9 +38,9 @@ console.log(`Verifying staged release ${staging}`); await verifyRemoteManifest(staging, manifest); await verifyRemoteManifest(production, manifest, { allowPartial: true }); console.log(`Promoting ${staging} to immutable prefix ${production}`); -await runRclone(["copy", staging, production, "--checksum", "--fast-list", "--immutable", "--metadata", "--progress"]); +await runRclone(["copy", staging, production, "--checksum", "--fast-list", "--metadata", "--progress", ...uploadArgs]); console.log(`Verifying promoted release ${production}`); await verifyRemoteManifest(production, manifest); console.log(`Persisting reviewed manifest at ${remoteManifest}`); -await runRclone(["copyto", manifestPath, remoteManifest, "--ignore-existing", "--header-upload", "If-None-Match: *"]); +await runRclone(["copyto", manifestPath, remoteManifest, ...uploadArgs]); if (await readRemoteFile(remoteManifest) !== manifestText) throw new Error("Persisted remote manifest differs from the reviewed manifest"); diff --git a/scripts/docs/promotion.test.mjs b/scripts/docs/promotion.test.mjs index 86236b0ef..d6f591d22 100644 --- a/scripts/docs/promotion.test.mjs +++ b/scripts/docs/promotion.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; -import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { chmod, cp, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import test from "node:test"; @@ -82,3 +82,81 @@ test("promotion preserves completed versions and resumes only matching partial u await rm(root, { recursive: true, force: true }); } }); + +test("direct publication keeps revisions immutable and leaves legacy versions untouched", { skip: !hasRclone && "rclone is not installed" }, async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "gecode-direct-publication-")); + try { + const source = path.join(root, "source"); + const manifestPath = path.join(root, "manifest.json"); + const bucket = path.join(root, "bucket"); + const version = "6.4.0"; + await mkdir(source); + await mkdir(path.join(bucket, version), { recursive: true }); + await writeFile(path.join(bucket, version, "index.html"), "legacy"); + const env = { ...process.env, RCLONE_CONFIG_REVIEW_TYPE: "alias", RCLONE_CONFIG_REVIEW_REMOTE: root }; + const publish = (revision, confirmed = true) => execFileSync(process.execPath, [ + "scripts/docs/publish-release.mjs", "--root", source, "--version", version, + "--revision", revision, "--manifest", manifestPath, "--remote", "review:bucket", + ...(confirmed ? ["--confirm-upload"] : []), + ], { env, stdio: "pipe", encoding: "utf8" }); + await writeFile(path.join(source, "index.html"), "first revision"); + await writeFile(manifestPath, JSON.stringify(await createManifest(source, version))); + publish("first", false); + await assert.rejects(readFile(path.join(bucket, "_revisions", version, "first", "index.html")), { code: "ENOENT" }); + publish("first"); + assert.match(publish("first"), /no upload needed/); + await writeFile(path.join(source, "index.html"), "second revision"); + await writeFile(manifestPath, JSON.stringify(await createManifest(source, version))); + assert.throws(() => publish("first"), /different manifest/); + publish("second"); + assert.equal(await readFile(path.join(bucket, version, "index.html"), "utf8"), "legacy"); + assert.equal(await readFile(path.join(bucket, "_revisions", version, "first", "index.html"), "utf8"), "first revision"); + assert.equal(await readFile(path.join(bucket, "_revisions", version, "second", "index.html"), "utf8"), "second revision"); + assert.equal(await readFile(path.join(bucket, "_manifests", version, "second.json"), "utf8"), await readFile(manifestPath, "utf8")); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("verifies every MD5 with a fast listing and samples MIME once per content type", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "gecode-md5-listing-")); + const originalPath = process.env.PATH; + try { + const source = path.join(root, "source"); + await mkdir(source); + await writeFile(path.join(source, "index.html"), "release"); + await writeFile(path.join(source, "second.html"), "another"); + await writeFile(path.join(source, "style.css"), "body{}"); + const manifest = await createManifest(source, "6.4.0"); + const listing = manifest.files.map((file) => ({ Path: file.path, Size: file.bytes, Hashes: { MD5: file.md5 } })); + const stats = manifest.files.map((file) => ({ Path: file.path, Size: file.bytes, MimeType: file.contentType, IsDir: false })); + const log = path.join(root, "commands.jsonl"); + const fake = path.join(root, "rclone"); + await writeFile(fake, `#!${process.execPath} +const args = process.argv.slice(2); +require("node:fs").appendFileSync(${JSON.stringify(log)}, JSON.stringify(args) + "\\n"); +if (args[0] !== "lsjson") { console.error("Unexpected object download"); process.exit(77); } +if (args.includes("--stat")) { + console.log(JSON.stringify(${JSON.stringify(stats)}.find((file) => args[1] === "test:bucket/" + file.Path))); +} else { + if (!["--recursive", "--files-only", "--hash", "--no-mimetype", "--no-modtime"].every((flag) => args.includes(flag)) || args.includes("--metadata")) process.exit(78); + console.log(${JSON.stringify(JSON.stringify(listing))}); +} +`); + await chmod(fake, 0o755); + process.env.PATH = `${root}:${originalPath}`; + await verifyRemoteManifest("test:bucket", manifest); + const calls = (await readFile(log, "utf8")).trim().split("\n").map((line) => JSON.parse(line)); + assert.equal(calls.length, 3); + assert.equal(calls.filter((args) => args.includes("--stat")).length, 2); + const corrupt = structuredClone(manifest); + corrupt.files.find((file) => file.path === "second.html").md5 = "0".repeat(32); + await assert.rejects(verifyRemoteManifest("test:bucket", corrupt), /MD5 manifest/); + const wrongMime = structuredClone(manifest); + wrongMime.files.find((file) => file.path === "style.css").contentType = "text/plain"; + await assert.rejects(verifyRemoteManifest("test:bucket", wrongMime), /Remote content type differs/); + } finally { + process.env.PATH = originalPath; + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/scripts/docs/publish-release.mjs b/scripts/docs/publish-release.mjs new file mode 100644 index 000000000..0e9fa9d54 --- /dev/null +++ b/scripts/docs/publish-release.mjs @@ -0,0 +1,46 @@ +#!/usr/bin/env node +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { conditionalUploadArgs, loadAndVerifyManifest, publicationPaths, readRemoteFile, + requireConditionalUploadSupport, runRclone, verifyRemoteManifest } from "./remote-lib.mjs"; + +function option(name) { + const index = process.argv.indexOf(name); + return index === -1 ? undefined : process.argv[index + 1]; +} + +const root = option("--root"); +const version = option("--version"); +const revision = option("--revision"); +const remote = option("--remote"); +const manifestPath = option("--manifest"); +const confirmed = process.argv.includes("--confirm-upload"); +if (!root || !version || !remote || !manifestPath) { + console.error("Usage: publish-release.mjs --root --version [--revision ] --manifest --remote [--confirm-upload]"); + process.exit(2); +} +const source = path.resolve(root); +const manifest = await loadAndVerifyManifest(manifestPath, source, version); +const manifestText = await readFile(manifestPath, "utf8"); +const destination = publicationPaths(remote, version, { revision }); +const uploadArgs = conditionalUploadArgs(manifest); +if (!confirmed) { + console.log(`Dry run: verified ${manifest.fileCount} local objects; would publish directly to ${destination.production} and complete at ${destination.manifest}`); + process.exit(0); +} + +await requireConditionalUploadSupport(); +const completed = await readRemoteFile(destination.manifest, { allowMissing: true }); +if (completed !== null) { + if (completed !== manifestText) throw new Error(`Completed publication ${version}${revision ? `/${revision}` : ""} has a different manifest`); + await verifyRemoteManifest(destination.production, manifest); + console.log("Completed publication verified; no upload needed."); + process.exit(0); +} +await verifyRemoteManifest(destination.production, manifest, { allowPartial: true }); +console.log(`Publishing local documentation directly to ${destination.production}`); +await runRclone(["copy", source, destination.production, "--checksum", "--fast-list", "--metadata", "--progress", ...uploadArgs]); +await verifyRemoteManifest(destination.production, manifest); +await runRclone(["copyto", manifestPath, destination.manifest, ...uploadArgs]); +if (await readRemoteFile(destination.manifest) !== manifestText) throw new Error("Persisted remote manifest differs from the reviewed manifest"); +console.log(`Publication completed: ${destination.manifest}`); diff --git a/scripts/docs/publish-version.mjs b/scripts/docs/publish-version.mjs index 01e383688..7fefafd6b 100644 --- a/scripts/docs/publish-version.mjs +++ b/scripts/docs/publish-version.mjs @@ -2,7 +2,7 @@ import { lstat } from "node:fs/promises"; import path from "node:path"; import { validateVersion } from "./lib.mjs"; -import { loadAndVerifyManifest, runRclone, validateBuildId, validateRemote } from "./remote-lib.mjs"; +import { conditionalUploadArgs, loadAndVerifyManifest, publicationPaths, requireConditionalUploadSupport, runRclone, verifyRemoteManifest } from "./remote-lib.mjs"; function option(name) { const index = process.argv.indexOf(name); @@ -14,9 +14,10 @@ const version = option("--version"); const remote = option("--remote"); const manifest = option("--manifest"); const buildId = option("--build-id"); +const revision = option("--revision"); const confirmed = process.argv.includes("--confirm-upload"); if (!root || !version || !remote || !manifest || !buildId) { - console.error("Usage: publish-version.mjs --root --version --manifest --build-id --remote [--confirm-upload]"); + console.error("Usage: publish-version.mjs --root --version --manifest --build-id --remote [--revision ] [--confirm-upload]"); process.exit(2); } validateVersion(version); @@ -24,16 +25,20 @@ validateVersion(version); const source = path.resolve(root); const stat = await lstat(source); if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error(`Archive root is not a directory: ${source}`); -await loadAndVerifyManifest(manifest, source, version); +const reviewedManifest = await loadAndVerifyManifest(manifest, source, version); -const destination = `${validateRemote(remote)}/staging/${validateBuildId(buildId)}/${version}`; +const destination = publicationPaths(remote, version, { revision, buildId }).staging; +if (confirmed) { + await requireConditionalUploadSupport(); + await verifyRemoteManifest(destination, reviewedManifest, { allowPartial: true }); +} const args = [ "copy", source, destination, "--checksum", "--fast-list", - "--immutable", + ...conditionalUploadArgs(reviewedManifest), "--metadata", "--progress", ]; @@ -41,3 +46,4 @@ if (!confirmed) args.push("--dry-run"); console.log(`${confirmed ? "Uploading" : "Dry run for"} immutable documentation ${version} to ${destination}`); await runRclone(args); +if (confirmed) await verifyRemoteManifest(destination, reviewedManifest); diff --git a/scripts/docs/remote-lib.mjs b/scripts/docs/remote-lib.mjs index 3f1f33f66..2ddbd4cfb 100644 --- a/scripts/docs/remote-lib.mjs +++ b/scripts/docs/remote-lib.mjs @@ -1,8 +1,8 @@ import { spawn } from "node:child_process"; -import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { contentType, createManifest } from "./lib.mjs"; +import { contentType, createManifest, validateVersion } from "./lib.mjs"; export function validateRemote(remote) { if (!/^[A-Za-z0-9_-]+:[^/].*$/.test(remote)) { @@ -40,15 +40,44 @@ async function captureRclone(args, { allowMissing = false } = {}) { return output; } -export async function readRemoteFile(remotePath, options) { - return captureRclone(["cat", remotePath], options); +export async function readRemoteFile(remotePath, { allowMissing = false } = {}) { + // S3 cat may succeed with empty stdout for a missing virtual prefix. Stat + // distinguishes that from an existing zero-byte completion object. + const listing = await captureRclone(["lsjson", remotePath, "--stat"], { allowMissing }); + if (listing === null || JSON.parse(listing).IsDir !== false) { + if (allowMissing) return null; + throw new Error(`Remote file does not exist: ${remotePath}`); + } + return captureRclone(["cat", remotePath]); } export async function requireConditionalUploadSupport() { const version = (await captureRclone(["version"])).match(/^rclone v(\d+)\.(\d+)\./m); if (!version || Number(version[1]) < 1 || (Number(version[1]) === 1 && Number(version[2]) < 75)) { - throw new Error("rclone 1.75.0 or newer is required for conditional R2 manifest writes"); + throw new Error("rclone 1.75.0 or newer is required for conditional R2 object writes"); + } +} + +// Keep the public documentation keys stable; revisions select private R2 prefixes. +export function publicationPaths(remote, version, { revision, buildId } = {}) { + const base = validateRemote(remote); + validateVersion(version); + if (revision !== undefined) validateBuildId(revision); + return { + production: revision ? `${base}/_revisions/${version}/${revision}` : `${base}/${version}`, + manifest: revision ? `${base}/_manifests/${version}/${revision}.json` : `${base}/_manifests/${version}.json`, + staging: buildId === undefined ? undefined : `${base}/staging/${validateBuildId(buildId)}/${version}${revision ? `/${revision}` : ""}`, + }; +} + +export function conditionalUploadArgs(manifest) { + // R2's conditional PutObject protects against concurrent writers. Force a + // single PUT and disable CopyObject, whose destination conditions differ. + if (manifest.files.some((file) => file.bytes >= 5 * 1024 ** 3)) { + throw new Error("Conditional documentation publication requires each object to be smaller than 5 GiB"); } + return ["--immutable", "--ignore-existing", "--no-update-modtime", + "--disable", "Copy", "--s3-upload-cutoff", "5G", "--header-upload", "If-None-Match: *"]; } export async function loadAndVerifyManifest(manifestPath, root, version) { @@ -57,6 +86,12 @@ export async function loadAndVerifyManifest(manifestPath, root, version) { throw new Error(`Manifest version ${expected.documentationVersion} does not match ${version}`); } const actual = await createManifest(path.resolve(root), version); + // SHA-256-only manifests remain valid for previously published archives. + actual.files = actual.files.map((file, index) => { + if (expected.files[index]?.md5 !== undefined) return file; + const { md5, ...legacy } = file; + return legacy; + }); if (JSON.stringify(actual) !== JSON.stringify(expected)) { throw new Error("The source tree does not match the supplied manifest; regenerate and review it"); } @@ -64,15 +99,15 @@ export async function loadAndVerifyManifest(manifestPath, root, version) { } export async function verifyRemoteManifest(remotePath, expectedManifest, { allowPartial = false } = {}) { + const fastHashes = expectedManifest.files.every((file) => /^[a-f0-9]{32}$/.test(file.md5 ?? "")); + // R2 exposes single-part MD5 in its listing. Asking for MIME or modification + // time would turn this into a serial HEAD request for every object. const listing = JSON.parse(await captureRclone([ - "lsjson", - remotePath, - "--recursive", - "--files-only", - "--metadata", - "--hash", + "lsjson", remotePath, "--recursive", "--files-only", "--hash", + ...(fastHashes ? ["--no-mimetype", "--no-modtime"] : ["--metadata"]), ], { allowMissing: allowPartial }) ?? "[]"); const expectedByPath = new Map(expectedManifest.files.map((file) => [file.path, file])); + const download = new Set(); const legacyMaps = new Set(); const mimeBase = (value) => value?.split(";", 1)[0].trim().toLowerCase(); const compatibleContentType = (actual, expected) => { @@ -83,50 +118,79 @@ export async function verifyRemoteManifest(remotePath, expectedManifest, { allow return new Set([actualBase, expectedBase]).size === 2 && [actualBase, expectedBase].every((value) => value === "text/javascript" || value === "application/javascript"); }; - if (!allowPartial && listing.length !== expectedByPath.size) throw new Error(`Remote object count differs at ${remotePath}`); - for (const remoteObject of listing) { - const expected = expectedByPath.get(remoteObject.Path); - if (!expected) throw new Error(`Unexpected remote object: ${remoteObject.Path}`); - if (remoteObject.Size !== expected.bytes) throw new Error(`Remote size differs: ${remoteObject.Path}`); + const verifyContentType = (remoteObject, expected) => { if (!compatibleContentType(remoteObject.MimeType, expected.contentType)) { if (/\.map$/i.test(expected.path) && contentType(expected.path) === "application/octet-stream" && mimeBase(expected.contentType) === "application/json" && mimeBase(remoteObject.MimeType) === "application/octet-stream") { legacyMaps.add(expected.path); + download.add(expected.path); } else { - throw new Error(`Remote content type differs for ${remoteObject.Path}: ${remoteObject.MimeType ?? "missing"}`); + throw new Error(`Remote content type differs for ${expected.path}: ${remoteObject.MimeType ?? "missing"}`); } } + }; + const representatives = new Map(); + if (!allowPartial && listing.length !== expectedByPath.size) throw new Error(`Remote object count differs at ${remotePath}`); + const seen = new Set(); + for (const remoteObject of listing) { + const expected = expectedByPath.get(remoteObject.Path); + if (!expected || seen.has(remoteObject.Path)) throw new Error(`Unexpected remote object: ${remoteObject.Path}`); + seen.add(remoteObject.Path); + if (remoteObject.Size !== expected.bytes) throw new Error(`Remote size differs: ${remoteObject.Path}`); + if (fastHashes) { + if (!representatives.has(expected.contentType)) representatives.set(expected.contentType, expected); + } else { + verifyContentType(remoteObject, expected); + } + const hashes = Object.fromEntries(Object.entries(remoteObject.Hashes ?? {}) + .map(([name, value]) => [name.toLowerCase().replaceAll("-", ""), value.toLowerCase()])); + if (hashes.sha256) { + if (hashes.sha256 !== expected.sha256) throw new Error(`Remote object differs from SHA-256 manifest: ${expected.path}`); + } else if (expected.md5 && hashes.md5) { + if (hashes.md5 !== expected.md5) throw new Error(`Remote object differs from MD5 manifest: ${expected.path}`); + } else { + download.add(expected.path); + } } - - if (allowPartial) { - const present = new Set(listing.map((object) => object.Path)); - const files = expectedManifest.files.filter((file) => present.has(file.path)); - expectedManifest = { ...expectedManifest, files, fileCount: files.length, totalBytes: files.reduce((sum, file) => sum + file.bytes, 0) }; - if (files.length === 0) return; + // Check actual HTTP metadata once per expected content type, while hashes, + // sizes, and paths above are still checked for every object in the tree. + for (const expected of representatives.values()) { + const remoteObject = JSON.parse(await captureRclone(["lsjson", `${remotePath}/${expected.path}`, "--stat"])); + if (remoteObject.IsDir !== false || remoteObject.Size !== expected.bytes) { + throw new Error(`Remote representative object differs: ${expected.path}`); + } + verifyContentType(remoteObject, expected); } + if (!download.size) return; + // Normal R2 publications expose MD5 via ETag (or rclone's multipart metadata). + // Only historical SHA-256-only or hashless objects need a download fallback. const temporaryDirectory = await mkdtemp(path.join(os.tmpdir(), "gecode-doc-verify-")); try { - await runRclone(["copy", remotePath, temporaryDirectory, "--checksum", "--fast-list", "--metadata", "--progress"]); - const actual = await createManifest(temporaryDirectory, expectedManifest.documentationVersion); - // Historical manifests mislabeled Graphviz image maps as JSON. Accept only - // recognizable map content whose downloaded bytes still match the manifest. + for (const filename of download) { + if (/^[\/]|[\\\r\n]/.test(filename) || filename.split("/").some((part) => part === ".." || part === ".")) { + throw new Error(`Unsafe manifest path: ${filename}`); + } + } + const listPath = path.join(temporaryDirectory, "files.txt"); + const destination = path.join(temporaryDirectory, "objects"); + await writeFile(listPath, [...download].join("\n") + "\n"); + console.log(`Downloading ${download.size} objects without comparable hashes or with historical map metadata`); + await runRclone(["copy", remotePath, destination, "--files-from-raw", listPath, "--checksum", "--metadata"]); + const actual = await createManifest(destination, expectedManifest.documentationVersion); + if (actual.fileCount !== download.size) throw new Error(`Downloaded object count differs at ${remotePath}`); for (const file of actual.files) { - if (!legacyMaps.has(file.path)) continue; - const body = await readFile(path.join(temporaryDirectory, file.path), "utf8"); const expected = expectedByPath.get(file.path); - if (!/^(?:<(?:map|area)(?:\s|>)|base referer\r?\nrect\s)/.test(body) || file.sha256 !== expected.sha256) { - throw new Error(`Remote tree ${remotePath} does not match the SHA-256 manifest or historical image-map format: ${file.path}`); + if (!expected || file.sha256 !== expected.sha256 || (expected.md5 && file.md5 !== expected.md5)) { + throw new Error(`Remote tree ${remotePath} does not match the SHA-256 manifest: ${file.path}`); + } + if (legacyMaps.has(file.path)) { + const body = await readFile(path.join(destination, file.path), "utf8"); + if (!/^(?:<(?:map|area)(?:\s|>)|base referer\r?\nrect\s)/.test(body)) { + throw new Error(`Remote object does not match historical image-map format: ${file.path}`); + } } - } - expectedManifest = { - ...expectedManifest, - files: expectedManifest.files.map((file) => legacyMaps.has(file.path) - ? { ...file, contentType: "application/octet-stream" } : file), - }; - if (JSON.stringify(actual) !== JSON.stringify(expectedManifest)) { - throw new Error(`Remote tree ${remotePath} does not match the SHA-256 manifest`); } } finally { await rm(temporaryDirectory, { recursive: true, force: true }); diff --git a/scripts/docs/smoke-worker.mjs b/scripts/docs/smoke-worker.mjs index e42e5c6d6..328ecb1c6 100644 --- a/scripts/docs/smoke-worker.mjs +++ b/scripts/docs/smoke-worker.mjs @@ -1,10 +1,17 @@ import assert from "node:assert/strict"; -const [base, version, mode] = process.argv.slice(2); +const [base, version, ...options] = process.argv.slice(2); assert(base && /^\d+\.\d+\.\d+(?:-[A-Za-z0-9.-]+)?$/.test(version ?? ""), - "Usage: node scripts/docs/smoke-worker.mjs [--immutable-only]"); -assert(mode === undefined || mode === "--immutable-only", "Unknown smoke-check mode"); -const immutableOnly = mode === "--immutable-only"; + "Usage: node scripts/docs/smoke-worker.mjs [--revision ] [--immutable-only]"); +let revision; +let immutableOnly = false; +for (let index = 0; index < options.length; index++) { + if (options[index] === "--immutable-only") immutableOnly = true; + else if (options[index] === "--revision") { + revision = options[++index]; + assert(/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(revision ?? ""), "Invalid documentation revision"); + } else assert.fail(`Unknown smoke-check option: ${options[index]}`); +} const origin = new URL(base).origin; const production = origin === "https://www.gecode.dev"; const latestBase = "https://www.gecode.dev/doc/latest/"; @@ -23,6 +30,10 @@ async function check(path, status, options = {}, verify = () => {}) { if (!production || path.startsWith(`/doc/${version}/`) || path.startsWith("/doc-latest/")) { assertNoindex(response); } + if ([200, 206, 304].includes(status) && /^\/(?:doc\/|doc-latest\/)/.test(path)) { + assert.equal(response.headers.get("x-gecode-documentation-version"), version, `${path}: selected version`); + assert.equal(response.headers.get("x-gecode-documentation-revision"), revision ?? "legacy", `${path}: selected revision`); + } await verify(response); console.log(`${options.method ?? "GET"} ${path}: ${status}`); } finally { @@ -30,7 +41,18 @@ async function check(path, status, options = {}, verify = () => {}) { } } -for (const prefix of immutableOnly ? [`/doc/${version}`] : [`/doc/${version}`, "/doc/latest", "/doc-latest"]) { +const prefixes = [`/doc/${version}`]; +if (revision) prefixes.push(`/doc/${version}/revisions/${revision}`); +if (!immutableOnly) prefixes.push("/doc/latest", "/doc-latest"); + +function assertCanonical(response, prefix, relative) { + const indexable = production && prefix === "/doc/latest"; + assertNoindex(response, !indexable); + assert.equal(response.headers.get("link"), indexable + ? `<${latestBase}${relative}>; rel="canonical"` : null); +} + +for (const prefix of prefixes) { await check(`${prefix}/reference/index.html`, 200, {}, (response) => { assert.equal(response.headers.get("x-gecode-documentation-version"), version); assert.match(response.headers.get("content-type"), /text\/html/); @@ -42,6 +64,41 @@ for (const prefix of immutableOnly ? [`/doc/${version}`] : [`/doc/${version}`, " await check(`${prefix}/reference?smoke=1`, 308, {}, (response) => { assert.equal(response.headers.get("location"), `${origin}${prefix}/reference/?smoke=1`); }); + if (revision) { + for (const relative of ["index.html", "modeling/index.html", "modeling/search/index.html"]) { + await check(`${prefix}/${relative}`, 200, {}, async (response) => { + assert.match(response.headers.get("content-type"), /text\/html/); + assertCanonical(response, prefix, relative); + const html = await response.text(); + assert.match(html, / { + assert.match(response.headers.get("content-type"), mime, `${relative}: asset content type`); + }); + } + let language; + await check(`${prefix}/modeling/pagefind/pagefind-entry.json`, 200, {}, async (response) => { + assert.match(response.headers.get("content-type"), /application\/json/); + const index = await response.json(); + language = index.languages?.en; + assert(language?.page_count > 0, "English search index is empty"); + assert(/^[A-Za-z0-9_-]+$/.test(language.hash), "Invalid search metadata filename"); + assert(/^[A-Za-z0-9_-]+$/.test(language.wasm), "Invalid search runtime filename"); + }); + for (const asset of [`pagefind.${language.hash}.pf_meta`, `wasm.${language.wasm}.pagefind`]) { + await check(`${prefix}/modeling/pagefind/${asset}`, 200); + } + await check(`${prefix}/readiness-missing-page.html`, 404); + } } if (!immutableOnly) { @@ -94,23 +151,28 @@ await checkSitemap(`/doc/${version}/sitemap.xml`, false); if (!immutableOnly) await checkSitemap("/doc/sitemap.xml", true); await check(`/doc/${version}/readiness-missing-page.html`, 404); -const pdf = `/doc/${version}/MPG.pdf`; -let etag; -await check(pdf, 200, { method: "HEAD", headers: { Range: "bytes=0-15" } }, (response) => { - assert.match(response.headers.get("content-type"), /application\/pdf/); - assert(Number(response.headers.get("content-length")) > 16); - assert.equal(response.headers.get("content-range"), null); - assert.equal(response.headers.get("link"), null); - etag = response.headers.get("etag"); - assert(etag); -}); -await check(pdf, 206, { headers: { Range: "bytes=0-15", "If-Range": etag } }, (response) => { - assert.match(response.headers.get("content-range"), /^bytes 0-15\/\d+$/); - assert.equal(response.headers.get("content-length"), "16"); - assert.equal(response.headers.get("link"), null); -}); -await check(pdf, 200, { headers: { Range: "bytes=0-15", "If-Range": '"stale-readiness-validator"' } }, (response) => { - assert.equal(response.headers.get("content-range"), null); - assert(Number(response.headers.get("content-length")) > 16); -}); -console.log(`Documentation smoke checks passed for ${origin}, version ${version}.`); +for (const prefix of prefixes) { + const pdf = `${prefix}/MPG.pdf`; + let etag; + await check(pdf, 200, { method: "HEAD", headers: { Range: "bytes=0-15" } }, (response) => { + assert.match(response.headers.get("content-type"), /application\/pdf/); + assert(Number(response.headers.get("content-length")) > 16); + assert.equal(response.headers.get("content-range"), null); + assertCanonical(response, prefix, "MPG.pdf"); + etag = response.headers.get("etag"); + assert(etag); + }); + await check(pdf, 206, { headers: { Range: "bytes=0-15", "If-Range": etag } }, async (response) => { + assert.match(response.headers.get("content-range"), /^bytes 0-15\/\d+$/); + assert.equal(response.headers.get("content-length"), "16"); + assertCanonical(response, prefix, "MPG.pdf"); + const bytes = new Uint8Array(await response.arrayBuffer()); + assert.equal(bytes.length, 16, `${pdf}: range response length`); + assert.equal(new TextDecoder().decode(bytes.slice(0, 5)), "%PDF-", `${pdf}: PDF signature`); + }); + await check(pdf, 200, { headers: { Range: "bytes=0-15", "If-Range": '"stale-readiness-validator"' } }, (response) => { + assert.equal(response.headers.get("content-range"), null); + assert(Number(response.headers.get("content-length")) > 16); + }); +} +console.log(`Documentation smoke checks passed for ${origin}, version ${version}, revision ${revision ?? "legacy"}.`); diff --git a/scripts/docs/verify-version.mjs b/scripts/docs/verify-version.mjs index 9ebf8933d..b5c31a9dd 100644 --- a/scripts/docs/verify-version.mjs +++ b/scripts/docs/verify-version.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node import { readFile } from "node:fs/promises"; import { validateVersion } from "./lib.mjs"; -import { validateBuildId, validateRemote, verifyRemoteManifest } from "./remote-lib.mjs"; +import { publicationPaths, verifyRemoteManifest } from "./remote-lib.mjs"; function option(name) { const index = process.argv.indexOf(name); @@ -12,17 +12,18 @@ const version = option("--version"); const remote = option("--remote"); const manifestPath = option("--manifest"); const buildId = option("--build-id"); +const revision = option("--revision"); const final = process.argv.includes("--final"); if (!version || !remote || !manifestPath || (!buildId && !final) || (buildId && final)) { - console.error("Usage: verify-version.mjs --version --manifest --remote (--build-id | --final)"); + console.error("Usage: verify-version.mjs --version --manifest --remote [--revision ] (--build-id | --final)"); process.exit(2); } validateVersion(version); const manifest = JSON.parse(await readFile(manifestPath, "utf8")); if (manifest.documentationVersion !== version) throw new Error("Manifest and requested versions do not match"); -const base = validateRemote(remote); -const source = final ? `${base}/${version}` : `${base}/staging/${validateBuildId(buildId)}/${version}`; -console.log(`Downloading and verifying every object in ${source}`); +const paths = publicationPaths(remote, version, { revision, buildId }); +const source = final ? paths.production : paths.staging; +console.log(`Verifying object metadata and available hashes in ${source}`); await verifyRemoteManifest(source, manifest); -console.log(`Verified ${manifest.fileCount} objects against their SHA-256 manifest`); +console.log(`Verified ${manifest.fileCount} objects against the manifest`); diff --git a/workers/docs/README.md b/workers/docs/README.md index dc7ff4e82..0d6fe6a73 100644 --- a/workers/docs/README.md +++ b/workers/docs/README.md @@ -4,22 +4,35 @@ This Worker serves Gecode's generated documentation from a private Cloudflare R2 bucket while preserving the existing `www.gecode.dev` paths. It does not serve the Astro website. -The bucket stores releases at its root: +The bucket retains historical versions and adds immutable documentation revisions: ```text -6.4.0/reference/index.html -6.4.0/MPG.pdf -6.5.0/modeling/index.html +6.4.0/reference/index.html # historical publication +_revisions/6.4.0/20260905-rst2/reference/index.html +_revisions/6.4.0/20260905-rst2/modeling/index.html +_revisions/6.4.0/20260905-rst2/MPG.pdf +_manifests/6.4.0/20260905-rst2.json # completion record ``` -The Worker maps `/doc//...` directly to those keys. It maps both -`/doc/latest/...` and `/doc-latest/...` to `LATEST_DOC_VERSION`; aliases are not -copied into R2. Only production `/doc/latest/...` documentation is indexable, -and its HTML canonical points to the corresponding latest URL. All immutable -version URLs, including PDFs, carry `X-Robots-Tag: noindex`. The -`/doc-latest/...` compatibility alias still serves content with HTTP 200 and -`noindex`; staging documentation is also `noindex`. These paths remain -crawlable so search engines can read the indexing headers. +`DOC_REVISIONS` is a JSON string mapping Gecode versions to published revision +IDs, for example `{"6.4.0":"20260905-rst2"}`. The Worker resolves +`/doc/6.4.0/...` through that selection. An absent entry uses the historical +`6.4.0/...` prefix. Both `/doc/latest/...` and `/doc-latest/...` first resolve +`LATEST_DOC_VERSION`, then its selected revision. No aliases are copied into R2. + +The explicit `/doc/6.4.0/revisions/20260905-rst2/...` route always addresses that +revision, independently of `DOC_REVISIONS`. Verify it before selecting a newly +published revision. Only this explicit revision route has a one-year immutable +cache policy; selected version routes and aliases have a five-minute policy. +Responses identify both choices with `X-Gecode-Documentation-Version` and +`X-Gecode-Documentation-Revision` (the latter is `legacy` for an unselected +historical prefix). + +Only production `/doc/latest/...` documentation is indexable. HTML and PDF +responses there have a canonical Link header for the corresponding latest URL. +Version routes, explicit revision routes, the HTTP 200 `/doc-latest/...` +compatibility alias, and staging documentation carry `X-Robots-Tag: noindex`. +These paths remain crawlable so search engines can read the indexing headers. Production routes use `doc*` and `robots.txt*` because Cloudflare matches query strings against route patterns. Requests outside the exact documentation @@ -68,68 +81,52 @@ clean release directory, never `doc/latest` or `doc-latest`. The second manifest includes `sitemap.xml` and its shards, so publication and later verification cannot omit them. -## Historical migration publisher +## Publish a documentation revision -For ordinary releases, follow [the release contract](../../docs/gecode-release-pipeline.md). -The commands below are historical migration tools, not the coordinated release -path. Run only one publisher for any version. Promotion requires rclone 1.75.0 -or newer for conditional R2 manifest writes. - -Bulk uploads use `rclone`, which is better suited to tens of thousands of -objects than one-object-at-a-time Wrangler uploads. The publishing command is -a dry run unless `--confirm-upload` is present: +Follow [the release contract](../../docs/gecode-release-pipeline.md) for +coordinated releases and documentation-only updates. The pinned website tool +uploads a reviewed local tree directly to its final immutable prefix: ```sh -node scripts/docs/publish-version.mjs \ - --root release-tree \ - --version 6.4.0 \ +node scripts/docs/publish-release.mjs \ + --root release-tree --version 6.4.0 --revision 20260905-rst2 \ --manifest .documentation-manifests/6.4.0.json \ - --build-id local-6.4.0 \ - --remote r2:gecode-documentation + --remote r2:gecode-documentation --confirm-upload ``` -The command first proves that the source still matches the manifest, then uses -`copy --immutable` to upload into `staging//`. It neither -overwrites an existing object nor deletes remote objects. Configure the named -`r2` remote with an R2 token limited to the documentation bucket. - -After uploading, compare every local file with the remote release: +Omit `--confirm-upload` to validate the local tree and print the destination +without uploading. Every write, including the completion manifest, uses a +conditional PutObject. The tool requires rclone 1.75 or newer, disables +server-side copy, and rejects individual objects of 5 GiB or larger. Existing +objects must match; completed revisions cannot be extended or replaced. -```sh -node scripts/docs/verify-version.mjs \ - --version 6.4.0 \ - --manifest .documentation-manifests/6.4.0.json \ - --build-id local-6.4.0 \ - --remote r2:gecode-documentation -``` +New manifests include SHA-256 and MD5. Remote verification checks every object's +path, size, and comparable hash through a fast listing, then checks MIME metadata +for one representative of each expected content type. It does not download the +normal release tree or issue a HEAD request for every file. Deployment smoke +checks exercise the main served types as well. Historical SHA-256-only manifests +retain per-object metadata checks and use targeted downloads when hashes are +unavailable. The completion manifest is written only after verification. -This downloads the staged tree and recomputes its SHA-256 manifest. The check -fails on a missing, extra, or changed object. Promote only after it succeeds: +Verify the stored revision with: ```sh -node scripts/docs/promote-version.mjs \ - --version 6.4.0 \ +node scripts/docs/verify-version.mjs --version 6.4.0 --revision 20260905-rst2 \ --manifest .documentation-manifests/6.4.0.json \ - --build-id local-6.4.0 \ - --remote r2:gecode-documentation \ - --confirm-promotion + --remote r2:gecode-documentation --final ``` -Promotion first rejects a conflicting completion manifest. An identical -completed release is verified without requiring staging or writing objects. -For an unfinished release it verifies staged content and any matching partial -final tree before copying, then verifies the final tree and conditionally -creates `_manifests/.json`. The readback must match exactly. It never -extends a completed version or replaces an existing completion manifest. A final release can also be checked later by -downloading that manifest and replacing `--build-id ...` with `--final` in the -verification command. +Publication does not update `DOC_REVISIONS` or deploy a Worker. The optional +staging commands remain available for historical migrations; see the +[publication tools](../../scripts/docs/README.md) for their compatible revision +and legacy-prefix modes. ## Provision and deploy The bucket, lifecycle rule, DNS proxy, Email Routing, secrets, and billing alerts are intentionally not created by this repository. The staging, canary, and production Workers read the same private production bucket; only their -routes and selected `latest` versions differ. Configure the bucket to expire +routes, selected revisions, and `latest` versions differ. Configure the bucket to expire `staging/` keys after 14 days. After an operator creates the bucket named in `wrangler.jsonc` and the `docs-staging.gecode.dev` Worker custom domain: @@ -140,9 +137,9 @@ npx wrangler deploy --env canary --config workers/docs/wrangler.jsonc npx wrangler deploy --env production --config workers/docs/wrangler.jsonc ``` -The checked-in canary environment owns only the current immutable version -route, such as `/doc/6.4.0/*`. Update both the route and -`LATEST_DOC_VERSION` before each canary deployment. Do not create a temporary +The checked-in canary environment owns only the current selected version +route, such as `/doc/6.4.0/*`. Update the route, +`LATEST_DOC_VERSION`, and that environment's `DOC_REVISIONS` before each canary deployment. Do not create a temporary dashboard route: a later Wrangler deployment would replace it. Remove the canary after the production smoke test because its more-specific route takes precedence over the production `/doc/*` route: @@ -155,19 +152,29 @@ The `Deploy edge workers` workflow exposes the same action as `remove-canary`. Select the `canary` environment and `documentation` Worker when deploying or removing the canary. -After promotion, test the explicit immutable version through the proxied -`docs-staging.gecode.dev/doc//...` route before selecting it as -`latest`. Cloudflare's Cache API does not cache requests on `workers.dev`, so -that hostname cannot validate production cache behavior. -During the first production canary, keep the documentation in the GitHub Pages -artifact. Removing the documentation Worker routes then restores the current -origin without rebuilding the site. - -For a Worker-code change, validate staging before production. For an ordinary -content release, verify the immutable version through the existing production -Worker, then deploy its approved production `LATEST_DOC_VERSION` selection as -described in the release contract. Existing alias responses can remain cached for at most five minutes; the promotion is -therefore bounded rather than instantaneous. Do not remove an older prefix. +After publication, verify the explicit revision route before changing its +selection. For a Worker-code change, validate staging before production. For +an ordinary content update, change the approved production `DOC_REVISIONS` +entry, and change `LATEST_DOC_VERSION` only when releasing a new Gecode +version. Deploy through the protected `Deploy edge workers` workflow from +`main` or the approved normal-release website branch; it does not accept a +separate documentation-only branch pattern. + +The workflow reads the environment's configured revisions and checks the +latest version plus every other selected version. Its smoke command can also +be run directly after deployment: + +```sh +node scripts/docs/smoke-worker.mjs https://www.gecode.dev 6.4.0 \ + --revision 20260905-rst2 +``` + +For a selected version that is not latest, add `--immutable-only` to skip +latest aliases. Revision checks cover the modeling entry page, Pagefind index +and runtime assets, reference HTML, sitemap headers, exact PDF ranges, and 404s. +Previously cached selected routes may remain visible for up to five minutes. +Rollback restores the previous `DOC_REVISIONS` entry (or removes it to select +historical objects), without changing stored documentation. The Worker exposes the selected release's `sitemap.xml` at the stable `/doc/sitemap.xml` URL advertised by `robots.txt`. It rewrites sitemap index diff --git a/workers/docs/src/index.test.ts b/workers/docs/src/index.test.ts index 93eff3b79..018c55241 100644 --- a/workers/docs/src/index.test.ts +++ b/workers/docs/src/index.test.ts @@ -6,14 +6,15 @@ declare module "cloudflare:test" { interface ProvidedEnv { DOCS: R2Bucket; LATEST_DOC_VERSION: string; + DOC_REVISIONS?: string; } } const base = "https://www.gecode.dev"; -async function request(path: string, init?: RequestInit, latestVersion = env.LATEST_DOC_VERSION): Promise { +async function request(path: string, init?: RequestInit, latestVersion = env.LATEST_DOC_VERSION, revisions = env.DOC_REVISIONS): Promise { const context = createExecutionContext(); - const response = await worker.fetch(new Request(new URL(path, base), init), { ...env, LATEST_DOC_VERSION: latestVersion }, context); + const response = await worker.fetch(new Request(new URL(path, base), init), { ...env, LATEST_DOC_VERSION: latestVersion, DOC_REVISIONS: revisions }, context); await waitOnExecutionContext(context); return response; } @@ -37,7 +38,8 @@ describe("documentation worker", () => { const page = await request("/doc/6.4.0/reference/PageChange.html"); expect(page.status).toBe(200); expect(await page.text()).toBe("0123456789"); - expect(page.headers.get("cache-control")).toContain("immutable"); + expect(page.headers.get("cache-control")).toBe("public, max-age=300, s-maxage=300"); + expect(page.headers.get("x-gecode-documentation-revision")).toBe("legacy"); expect(page.headers.get("content-type")).toBe("text/html; charset=utf-8"); expect(page.headers.get("link")).toBeNull(); expect(page.headers.get("x-robots-tag")).toBe("noindex"); @@ -375,4 +377,89 @@ describe("documentation worker", () => { expect((await request("/doc/6.4.0/%252e%252e/secret")).status).toBe(400); expect((await request("/doc/6.4.0/reference%2fPageChange.html")).status).toBe(400); }); + it("promotes revisions without reusing the previous selection's cache", async () => { + const relative = "modeling/revision-test/index.html"; + for (const [revision, body] of [["r1", "first"], ["r2", "second"]]) { + await env.DOCS.put(`_revisions/6.4.0/${revision}/${relative}`, body, { + httpMetadata: { contentType: "text/html" }, + }); + } + for (const prefix of ["/doc/6.4.0/", "/doc/latest/"]) { + const first = await request(prefix + relative, undefined, "6.4.0", '{"6.4.0":"r1"}'); + expect(await first.text()).toBe("first"); + const promoted = await request(prefix + relative, undefined, "6.4.0", '{"6.4.0":"r2"}'); + expect(await promoted.text()).toBe("second"); + expect(promoted.headers.get("x-gecode-documentation-version")).toBe("6.4.0"); + expect(promoted.headers.get("x-gecode-documentation-revision")).toBe("r2"); + expect(promoted.headers.get("cache-control")).toBe("public, max-age=300, s-maxage=300"); + const rollback = await request(prefix + relative, undefined, "6.4.0", '{"6.4.0":"r1"}'); + expect(await rollback.text()).toBe("first"); + } + // A selected incomplete bundle must not silently mix in the old release. + expect((await request("/doc/6.4.0/reference/PageChange.html", undefined, "6.4.0", '{"6.4.0":"r2"}')).status).toBe(404); + }); + + it("serves immutable revision previews without changing public canonical paths", async () => { + const revision = "manual-2026.09_2"; + const prefix = `/doc/6.4.0/revisions/${revision}`; + await env.DOCS.put(`_revisions/6.4.0/${revision}/index.html`, "preview", { + httpMetadata: { contentType: "text/html" }, + }); + const preview = await request(prefix + "/", undefined, "6.4.0", '{"6.4.0":"other"}'); + expect(await preview.text()).toBe("preview"); + expect(preview.headers.get("x-gecode-documentation-revision")).toBe(revision); + expect(preview.headers.get("cache-control")).toContain("immutable"); + expect(preview.headers.get("x-robots-tag")).toBe("noindex"); + expect(preview.headers.get("link")).toBeNull(); + expect((await request(prefix)).headers.get("location")).toBe(`${base}${prefix}/`); + expect((await request("/doc/latest/revisions/r1/")).status).toBe(400); + }); + + it("keeps PDF ranges and validators on the selected revision", async () => { + const object = await env.DOCS.put("_revisions/6.4.0/pdf-r2/MPG.pdf", "new PDF bytes", { + httpMetadata: { contentType: "application/pdf" }, + }); + const revisions = '{"6.4.0":"pdf-r2"}'; + const path = "/doc/latest/MPG.pdf"; + for (const init of [undefined, { method: "HEAD" }, { headers: { Range: "bytes=4-6" } }, { headers: { "If-None-Match": object.httpEtag } }]) { + const response = await request(path, init, "6.4.0", revisions); + expect(response.headers.get("x-gecode-documentation-revision")).toBe("pdf-r2"); + expect(response.headers.get("link")).toBe('; rel="canonical"'); + if (init?.headers && "Range" in init.headers) { + expect(response.status).toBe(206); + expect(new TextDecoder().decode(await response.arrayBuffer())).toBe("PDF"); + } + } + const stale = await request(path, { headers: { Range: "bytes=4-6", "If-Range": '"old-revision"' } }, "6.4.0", revisions); + expect(stale.status).toBe(200); + expect(new TextDecoder().decode(await stale.arrayBuffer())).toBe("new PDF bytes"); + }); + + it("rewrites selected revision sitemaps using only public version URLs", async () => { + const xml = 'https://www.gecode.dev/doc/6.4.0/modeling/chapter/'; + await env.DOCS.put("_revisions/6.4.0/sitemap-r2/sitemap.xml", xml, { + httpMetadata: { contentType: "application/xml" }, + }); + const revisions = '{"6.4.0":"sitemap-r2"}'; + const latest = await request("/doc/sitemap.xml", undefined, "6.4.0", revisions); + expect(await latest.text()).toBe(xml.replaceAll("/doc/6.4.0/", "/doc/latest/")); + expect(latest.headers.get("x-gecode-documentation-revision")).toBe("sitemap-r2"); + for (const path of ["/doc/6.4.0/sitemap.xml", "/doc/6.4.0/revisions/sitemap-r2/sitemap.xml"]) { + const response = await request(path, undefined, "6.4.0", revisions); + expect(await response.text()).toBe(xml); + expect(response.headers.get("x-robots-tag")).toBe("noindex"); + } + }); + + it.each([ + "{", "null", "[]", '{"6.4.0":null}', '{"6.4.0":42}', + '{"6.4.0":"../escape"}', '{"6.4.0":""}', '{"bad-version":"r1"}', + JSON.stringify({ "6.4.0": "r".repeat(129) }), + ])("fails closed for malformed revision configuration: %s", async (revisions) => { + await request("/doc/6.4.0/reference/PageChange.html"); + const response = await request("/doc/6.4.0/reference/PageChange.html", undefined, "6.4.0", revisions); + expect(response.status).toBe(503); + expect(response.headers.get("x-robots-tag")).toBe("noindex"); + }); + }); diff --git a/workers/docs/src/index.ts b/workers/docs/src/index.ts index 66ab90ff9..0c035c356 100644 --- a/workers/docs/src/index.ts +++ b/workers/docs/src/index.ts @@ -3,14 +3,35 @@ import robots from "../../../robots.txt"; export interface Env { DOCS: R2Bucket; LATEST_DOC_VERSION: string; + DOC_REVISIONS?: string; } type ResolvedPath = { key: string; version: string; + relative: string; + revision?: string; isAlias: boolean; + isRevision: boolean; }; +const versionPattern = /^\d+\.\d+\.\d+(?:-[A-Za-z0-9.-]+)?$/; +const revisionPattern = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; + +function configuredRevisions(value: string | undefined): Record { + if (value === undefined) return {}; + const revisions: unknown = JSON.parse(value); + if (!revisions || typeof revisions !== "object" || Array.isArray(revisions)) { + throw new Error("DOC_REVISIONS must be a JSON object mapping versions to revisions"); + } + for (const [version, revision] of Object.entries(revisions)) { + if (!versionPattern.test(version) || typeof revision !== "string" || !revisionPattern.test(revision)) { + throw new Error(`Invalid DOC_REVISIONS entry for ${version}`); + } + } + return revisions as Record; +} + const securityHeaders = { "Referrer-Policy": "strict-origin-when-cross-origin", "X-Content-Type-Options": "nosniff", @@ -70,12 +91,19 @@ function resolvePath(pathname: string, latestVersion: string): ResolvedPath | nu if (!match) return null; version = match[1]; relative = match[2] ?? ""; - if (!/^\d+\.\d+\.\d+(?:-[A-Za-z0-9.-]+)?$/.test(version)) return null; + if (!versionPattern.test(version)) return null; } - if (!/^\d+\.\d+\.\d+(?:-[A-Za-z0-9.-]+)?$/.test(version)) return null; + if (!versionPattern.test(version)) return null; + let revision: string | undefined; + if (relative === "revisions" || relative.startsWith("revisions/")) { + const match = relative.match(/^revisions\/([^/]+)(?:\/(.*))?$/); + if (isAlias || !match || !revisionPattern.test(match[1])) return null; + revision = match[1]; + relative = match[2] ?? ""; + } if (relative === "" || relative.endsWith("/")) relative += "index.html"; - return { key: `${version}/${relative}`, version, isAlias }; + return { key: `${version}/${relative}`, version, relative, revision, isAlias, isRevision: revision !== undefined }; } function parseRange(value: string, size: number): { offset: number; length: number } | "invalid" { @@ -104,11 +132,12 @@ function applyObjectHeaders(headers: Headers, object: R2Object, resolved: Resolv headers.set("Last-Modified", object.uploaded.toUTCString()); headers.set("Accept-Ranges", "bytes"); headers.set("X-Gecode-Documentation-Version", resolved.version); + headers.set("X-Gecode-Documentation-Revision", resolved.revision ?? "legacy"); headers.set( "Cache-Control", - resolved.isAlias - ? "public, max-age=300, s-maxage=300" - : "public, max-age=3600, s-maxage=31536000, immutable", + resolved.isRevision + ? "public, max-age=31536000, immutable" + : "public, max-age=300, s-maxage=300", ); for (const [name, value] of Object.entries(securityHeaders)) headers.set(name, value); } @@ -120,6 +149,7 @@ function applyIndexingPolicy(request: Request, response: Response, env: Env): Re const indexable = url.origin === "https://www.gecode.dev" && decoded?.startsWith("/doc/latest/") && resolved !== null + && !resolved.isRevision && [200, 206, 304].includes(response.status); const headers = new Headers(response.headers); // Apply this after cache reads too: a previous deployment may have cached @@ -128,8 +158,7 @@ function applyIndexingPolicy(request: Request, response: Response, env: Env): Re if (indexable) { headers.delete("X-Robots-Tag"); if (/^(?:text\/html|application\/pdf)(?:;|$)/i.test(headers.get("Content-Type") ?? "")) { - const relative = resolved.key.slice(resolved.version.length + 1); - const canonicalPath = relative.split("/").map(encodeURIComponent).join("/"); + const canonicalPath = resolved.relative.split("/").map(encodeURIComponent).join("/"); headers.set("Link", `; rel="canonical"`); } } else { @@ -188,8 +217,13 @@ async function serve(request: Request, env: Env, context: ExecutionContext): Pro if (url.pathname === "/doc" || url.pathname === "/doc/") return redirect("/documentation.html"); const resolved = resolvePath(url.pathname, env.LATEST_DOC_VERSION); if (!resolved) return errorResponse(400, "Invalid documentation path"); + const revisions = configuredRevisions(env.DOC_REVISIONS); + resolved.revision ??= revisions[resolved.version]; + if (resolved.revision) { + resolved.key = `_revisions/${resolved.version}/${resolved.revision}/${resolved.relative}`; + } - if (resolved.key === `${resolved.version}/index.html` + if (resolved.relative === "index.html" && !url.pathname.endsWith("/") && !safeDecodePath(url.pathname)!.endsWith("/index.html")) { return redirect(`${url.pathname}/`); } @@ -205,9 +239,10 @@ async function serve(request: Request, env: Env, context: ExecutionContext): Pro && !request.headers.has("If-None-Match"); const cacheUrl = new URL(request.url); cacheUrl.search = ""; - // Do not reuse old sitemap bodies, or an alias entry from another release. - cacheUrl.searchParams.set("__gecode_docs_policy", "latest-index-v1"); - cacheUrl.searchParams.set("__gecode_docs_version", resolved.version); + // Promotion changes the physical key, including when the public version stays + // the same. The policy also avoids old immutable version-URL cache entries. + cacheUrl.searchParams.set("__gecode_docs_policy", "revisions-v1"); + cacheUrl.searchParams.set("__gecode_docs_object", resolved.key); const cacheKey = new Request(cacheUrl, { method: "GET" }); if (cacheableRequest) { try { @@ -218,7 +253,7 @@ async function serve(request: Request, env: Env, context: ExecutionContext): Pro } } - if (resolved.isAlias && /^sitemap(?:-\d+)?\.xml$/.test(resolved.key.slice(resolved.version.length + 1))) { + if (resolved.isAlias && /^sitemap(?:-\d+)?\.xml$/.test(resolved.relative)) { const object = await env.DOCS.get(resolved.key); if (!object) return missingObject(); const response = await sitemapResponse(request, object, resolved); diff --git a/workers/docs/wrangler.jsonc b/workers/docs/wrangler.jsonc index 663e893fc..387e82471 100644 --- a/workers/docs/wrangler.jsonc +++ b/workers/docs/wrangler.jsonc @@ -4,7 +4,9 @@ "main": "src/index.ts", "compatibility_date": "2026-08-08", "workers_dev": true, - "observability": { "enabled": true }, + "observability": { + "enabled": true + }, "routes": [ { "pattern": "docs-staging.gecode.dev", @@ -18,13 +20,16 @@ } ], "vars": { - "LATEST_DOC_VERSION": "6.4.0" + "LATEST_DOC_VERSION": "6.4.0", + "DOC_REVISIONS": "{}" }, "env": { "canary": { "name": "gecode-documentation-canary", "workers_dev": false, - "observability": { "enabled": true }, + "observability": { + "enabled": true + }, "routes": [ { "pattern": "www.gecode.dev/doc/6.4.0", @@ -42,13 +47,16 @@ } ], "vars": { - "LATEST_DOC_VERSION": "6.4.0" + "LATEST_DOC_VERSION": "6.4.0", + "DOC_REVISIONS": "{}" } }, "production": { "name": "gecode-documentation", "workers_dev": false, - "observability": { "enabled": true }, + "observability": { + "enabled": true + }, "routes": [ { "pattern": "www.gecode.dev/robots.txt*", @@ -66,7 +74,8 @@ } ], "vars": { - "LATEST_DOC_VERSION": "6.4.0" + "LATEST_DOC_VERSION": "6.4.0", + "DOC_REVISIONS": "{\"6.4.0\":\"20260905-rst2\"}" } } }