diff --git a/.github/workflows/availability.yml b/.github/workflows/availability.yml new file mode 100644 index 000000000..d7fb622e3 --- /dev/null +++ b/.github/workflows/availability.yml @@ -0,0 +1,31 @@ +name: Check production availability + +on: + schedule: + - cron: '7,22,37,52 * * * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: production-availability + cancel-in-progress: false + +jobs: + check: + runs-on: ubuntu-latest + timeout-minutes: 8 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v6 + with: + node-version: 24 + - name: Check public website and documentation + run: | + for attempt in 1 2 3; do + if node scripts/check-availability.mjs; then exit 0; fi + if [ "$attempt" -eq 3 ]; then exit 1; fi + echo "Retrying public availability checks in 20 seconds (attempt $attempt)." + sleep 20 + done diff --git a/docs/deployment-runbook.md b/docs/deployment-runbook.md index b6b4fc5ea..701676312 100644 --- a/docs/deployment-runbook.md +++ b/docs/deployment-runbook.md @@ -323,6 +323,11 @@ caches may retain the preceding version for up to five minutes. ## 8. Monitor cost and availability +The [operations guide](operations.md) describes the scheduled public checks, +verified failure-notification setting and email Worker diagnostics. The +availability workflow runs every fifteen minutes and does not require a +Cloudflare token. Budget notifications remain separate. + The migrated archive contains 52,385 documentation files and 1.139 GB of object data. R2 Standard includes 10 GB-month, one million Class A operations, and ten million Class B operations each month. The initial direct uploads and checksum diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 000000000..daebd05ff --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,77 @@ +# Production availability checks + +The `Check production availability` GitHub workflow checks the public site at +minutes 7, 22, 37 and 52 of every hour. It uses Node 24 without installing +dependencies, deploying anything, or accessing Cloudflare credentials. + +The checks cover the Astro home and download pages, a classic URL redirect +with its query string, the users archive, robots rules, the documentation +sitemap, latest and versioned reference URLs, and PDF headers plus a 16-byte +range. When the selected production `DOC_REVISIONS` entry is present, they also +check the modeling manual and its search JavaScript. Version and revision come +from `workers/docs/wrangler.jsonc`; update that configuration when promoting +documentation. Each run makes ten requests, or twelve with the modeling manual, +and caps each response read at 1 MB. It never downloads the complete PDF. + +Transient failures get two retries, twenty seconds apart. Persistent failures +fail the workflow and leave the failing path in its log. Check the failed path, +the latest Pages and Worker deployments, and Cloudflare Workers logs. A version +or revision mismatch can mean the checked-in promotion has not reached +production yet. Use the deployment runbook for rollback; this workflow never +changes traffic automatically. + +## Run manually + +From the website checkout: + +```sh +node scripts/check-availability.mjs +gh workflow run availability.yml --repo Gecode/gecode.github.io --ref main +gh run list --repo Gecode/gecode.github.io --workflow availability.yml --limit 5 +``` + +Use `scripts/docs/smoke-worker.mjs` for the broader documentation checks after a +deployment. The scheduled checks deliberately use a smaller request set. + +## Make failures reach a maintainer + +The responsible maintainer must enable GitHub Actions email or web notifications +and select failed workflows only. Scheduled-run notifications go to the user +who created or last changed the schedule, or who re-enabled it. Repository +workflow success alone does not establish that anyone receives failure alerts. +Confirm that maintainer's settings before relying on this workflow. +See [GitHub workflow notifications](https://docs.github.com/en/actions/concepts/workflows-and-actions/notifications-for-workflow-runs). + +On 5 September 2026, the authenticated `zayenz` account's settings were checked: +Actions email notifications were enabled for failed workflows only. Keep that +account as the schedule owner, or verify the replacement owner's settings. + +GitHub schedules can be delayed or dropped under load. Public-repository +schedules are automatically disabled after sixty days without repository +activity. Check that this workflow remains enabled during routine maintenance; +re-enable it when necessary. This is a periodic availability check, with no +guaranteed detection time, and cannot detect every regional or intermittent +failure. See [GitHub schedule limitations](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#schedule). + +Cloudflare's budget notification remains separate from availability monitoring. +A passive origin alert can complement these checks, but it does not cover +every failure generated by the documentation Worker or R2. A Workers +Observability notification also needs an actual alert rule; the notification +destination alone does not monitor errors. + +## Email forwarding + +The email Worker waits for every configured forward to finish and reports any +forwarding failure. Native Worker logging is enabled; the handler does not log +message bodies. Deploy it separately from the website and documentation: + +```sh +gh workflow run workers.yml --repo Gecode/gecode.github.io --ref main \ + -f operation=deploy -f environment=production -f worker=email +``` + +The routing tests cover address selection and partial forwarding failure. +They do not establish inbox delivery. Use Cloudflare Email Routing delivery +analytics for that check; its zone-level Analytics Read permission is separate +from the website deployment token. A failed availability run does not diagnose +mail delivery, because the public HTTP checks exercise only the website. diff --git a/scripts/check-availability.mjs b/scripts/check-availability.mjs new file mode 100644 index 000000000..bac56b78b --- /dev/null +++ b/scripts/check-availability.mjs @@ -0,0 +1,112 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; + +const origin = "https://www.gecode.dev"; +const config = JSON.parse(await readFile("workers/docs/wrangler.jsonc", "utf8")).env.production.vars; +const version = config.LATEST_DOC_VERSION; +const revision = JSON.parse(config.DOC_REVISIONS ?? "{}")[version]; +assert(/^\d+\.\d+\.\d+(?:-[A-Za-z0-9.-]+)?$/.test(version), "Invalid production documentation version"); +assert(revision === undefined || /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(revision), + "Invalid production documentation revision"); + +async function bytes(response) { + const chunks = []; + let length = 0; + for await (const chunk of response.body) { + length += chunk.length; + assert(length <= 1_000_000, "Availability response exceeded 1 MB"); + chunks.push(chunk); + } + return Buffer.concat(chunks); +} + +async function check(path, status, verify, options = {}) { + const response = await fetch(`${origin}${path}`, { + redirect: "manual", signal: AbortSignal.timeout(10_000), ...options, + }); + try { + assert.equal(response.status, status, `${path}: HTTP status`); + await verify(response); + console.log(`${options.method ?? "GET"} ${path}: ${status}`); + } catch (error) { + throw new Error(`${path}: ${error.message}`, { cause: error }); + } finally { + if (!response.bodyUsed) await response.body?.cancel(); + } +} + +function docsHeaders(response, relative, indexable) { + assert.equal(response.headers.get("x-gecode-documentation-version"), version, "Selected version"); + assert.equal(response.headers.get("x-gecode-documentation-revision"), revision ?? "legacy", "Selected revision"); + assert.equal(/\bnoindex\b/i.test(response.headers.get("x-robots-tag") ?? ""), !indexable, "Indexing policy"); + assert.equal(response.headers.get("link"), indexable + ? `<${origin}/doc/latest/${relative}>; rel="canonical"` : null, "Documentation canonical"); +} + +for (const path of ["/", "/download/"]) { + await check(path, 200, async (response) => { + assert.match(response.headers.get("content-type"), /text\/html/); + const html = (await bytes(response)).toString(); + assert.match(html, /`), "Website canonical"); + assert(!/\bnoindex\b/i.test(response.headers.get("x-robots-tag") ?? ""), "Website must remain indexable"); + }); +} +await check("/download.html?availability=1", 308, (response) => { + assert.equal(response.headers.get("location"), `${origin}/download/?availability=1`); +}); +await check("/users-archive/index.html", 200, async (response) => { + assert.match(response.headers.get("content-type"), /text\/html/); + assert.match((await bytes(response)).toString(), /Gecode users archive/); +}); +await check("/robots.txt?availability=1", 200, async (response) => { + assert.match(response.headers.get("content-type"), /text\/plain/); + const text = (await bytes(response)).toString(); + assert(text.includes(`Sitemap: ${origin}/doc/sitemap.xml`)); + assert.doesNotMatch(text, /^Disallow:\s*\/(?:\s*$|doc(?:\/latest)?\/?\s*$|doc-latest)/m, + "Robots rules must allow latest documentation crawling"); +}); +await check("/doc/latest/reference/index.html", 200, async (response) => { + docsHeaders(response, "reference/index.html", true); + assert.match(response.headers.get("content-type"), /text\/html/); + assert.match((await bytes(response)).toString(), / { + docsHeaders(response, "reference/index.html", false); + assert.match(response.headers.get("content-type"), /text\/html/); +}, { method: "HEAD" }); +await check("/doc/sitemap.xml", 200, async (response) => { + assert.match(response.headers.get("content-type"), /xml/); + const xml = (await bytes(response)).toString(); + assert.match(xml, /([^<]+)<\/loc>/g)].map((match) => match[1]); + assert(locations.length > 0 && locations.every((url) => url.startsWith(`${origin}/doc/latest/`)), + "Documentation sitemap must advertise latest URLs"); +}); +const pdf = `/doc/${version}/MPG.pdf`; +await check(pdf, 200, (response) => { + docsHeaders(response, "MPG.pdf", false); + assert.match(response.headers.get("content-type"), /application\/pdf/); + assert(Number(response.headers.get("content-length")) > 16, "PDF size"); +}, { method: "HEAD" }); +await check(pdf, 206, async (response) => { + docsHeaders(response, "MPG.pdf", false); + assert.equal(response.headers.get("content-length"), "16"); + assert.match(response.headers.get("content-range"), /^bytes 0-15\/\d+$/); + const body = await bytes(response); + assert.equal(body.length, 16); + assert.equal(body.subarray(0, 5).toString(), "%PDF-"); +}, { headers: { Range: "bytes=0-15" } }); + +if (revision) { + await check("/doc/latest/modeling/index.html", 200, async (response) => { + docsHeaders(response, "modeling/index.html", true); + assert.match(response.headers.get("content-type"), /text\/html/); + assert.match((await bytes(response)).toString(), /pagefind/i, "Modeling search UI"); + }); + await check("/doc/latest/modeling/pagefind/pagefind.js", 200, async (response) => { + assert.match(response.headers.get("content-type"), /(?:text|application)\/javascript/); + assert((await bytes(response)).length > 0, "Modeling search runtime"); + }); +} +console.log(`Production availability passed: documentation ${version}, revision ${revision ?? "legacy"}.`); diff --git a/workers/email/wrangler.jsonc b/workers/email/wrangler.jsonc index 2ec27d954..206320a5b 100644 --- a/workers/email/wrangler.jsonc +++ b/workers/email/wrangler.jsonc @@ -3,5 +3,6 @@ "name": "gecode-email-routing", "main": "src/index.ts", "compatibility_date": "2026-08-08", - "workers_dev": false + "workers_dev": false, + "observability": { "enabled": true } }