diff --git a/apps/web/Dockerfile b/apps/web/Dockerfile index 4da1635..bfd3686 100644 --- a/apps/web/Dockerfile +++ b/apps/web/Dockerfile @@ -32,8 +32,17 @@ ENV HOSTNAME=0.0.0.0 # The web service gets 24 GB on Railway; letting the heap use half of it turns # a surge that would have been fatal into one that is merely slow. This is a # ceiling, not a reservation — it costs nothing until it is needed. +# +# This is the ceiling for a single server, which is what the primary process and +# a WEB_WORKERS=1 deployment are. When the primary forks workers it computes +# each one's share of the container's memory and passes it on their command +# line, where it wins over this — see src/lib/workers.js. Sizing it per worker +# here instead would mean this number had to be re-derived by hand every time +# the container changed shape. ENV NODE_OPTIONS=--max-old-space-size=12288 -# server.mjs is `next start` plus a ceiling on concurrent requests. Reasoning, -# and the outage behind it, in src/lib/loadShed.js. +# server.mjs is `next start`, plus a ceiling on concurrent requests, plus one +# copy of the server per CPU the container is allowed. Reasoning, and the two +# different outages behind the two parts, in src/lib/loadShed.js (memory) and +# src/lib/workers.js (CPU). CMD ["node", "server.mjs"] diff --git a/apps/web/server.mjs b/apps/web/server.mjs index 8154859..bc9480d 100644 --- a/apps/web/server.mjs +++ b/apps/web/server.mjs @@ -3,54 +3,98 @@ import { createServer } from 'node:http'; import next from 'next'; import { admit, inflight } from './src/lib/loadShed.js'; +import { forkWorkers, workerCount } from './src/lib/workers.js'; /** - * The HTTP server, with a ceiling on concurrent work. + * The HTTP server, on every core, with a ceiling on concurrent work. * - * This replaces `next start`, and does one thing `next start` cannot: it - * refuses a request when too many are already in flight. Reasoning, and the - * outage that motivated it, in src/lib/loadShed.js. Everything else is what - * `next start` does — the platform's PORT and HOSTNAME, Next's own request - * handler, no options of our own. + * This replaces `next start`, and does two things `next start` cannot. It + * refuses a request when too many are already in flight — reasoning, and the + * outage that motivated it, in src/lib/loadShed.js. And it runs one copy of the + * server per CPU the container is allowed, because JavaScript renders a page on + * one thread and a single copy leaves the rest of the machine idle while the + * site is down — reasoning, and *that* outage, in src/lib/workers.js. + * + * Everything else is what `next start` does — the platform's PORT and HOSTNAME, + * Next's own request handler, no options of our own. * * The port and host come from the environment and nothing else. Railway * injects PORT, and a hardcoded value here would leave the edge proxy * forwarding to a port nothing listens on — see the note in the Dockerfile. + * + * Every worker listens on the same port; `node:cluster` gives the primary the + * socket and hands connections round-robin. Nothing below needs to know whether + * it is the only server or one of sixteen, with one exception worth naming: all + * of the module state behind these requests — the throttle's counters, the + * traffic tally, the verified-key cache — is now per worker rather than per + * container. For the counters that bound *memory* that is the correct place for + * them, and `loadShed` divides its allowance so the container-wide total is + * unchanged. For the counters that meter *a caller* it is a loosening: a client + * holding a keep-alive connection stays on one worker, so its own limit is + * intact, but a caller opening fresh connections is metered by each worker + * separately. That is deliberate. The traffic this was written for arrives one + * request per address and defeats a per-caller limit outright, and tightening + * those limits by a factor of sixteen during an outage would refuse readers to + * no purpose. See src/lib/workers.js on why capacity is not a defence. */ const port = Number(process.env.PORT) || 3000; const hostname = process.env.HOSTNAME || '0.0.0.0'; -const app = next({ dev: false, hostname, port }); -const handle = app.getRequestHandler(); +// The primary forks and then has nothing to do. It must not go on to stand up +// Next and bind the port itself: that would put a seventeenth server on the +// socket with none of the workers' heap settings. +if ( + forkWorkers({ + onExit: ({ pid, code, signal }) => { + console.warn(`[web] worker ${pid} exited (code ${code}, signal ${signal}), replacing it`); + }, + }) +) { + console.log(`[web] primary ${process.pid} running ${workerCount()} workers`); +} else { + await serve(); +} -await app.prepare(); +/** + * Stand up Next and answer requests until the process ends. + * + * @returns {Promise} + */ +async function serve() { + const app = next({ dev: false, hostname, port }); + const handle = app.getRequestHandler(); -/** Say so when refusing starts, and then once a minute while it goes on. */ -let lastNoted = 0; + await app.prepare(); -const server = createServer((req, res) => { - const release = admit(pathOf(req.url)); + const server = createServer((req, res) => { + const release = admit(pathOf(req.url)); - if (release === null) { - refuse(res); - return; - } + if (release === null) { + refuse(res); + return; + } - // `close` fires whether the response finished or the socket died under it, - // which is the one event that means the request is no longer costing us. - res.once('close', release); + // `close` fires whether the response finished or the socket died under it, + // which is the one event that means the request is no longer costing us. + res.once('close', release); - handle(req, res).catch((err) => { - console.error('[web] request failed', err); - if (!res.headersSent) res.statusCode = 500; - res.end(); + handle(req, res).catch((err) => { + console.error('[web] request failed', err); + if (!res.headersSent) res.statusCode = 500; + res.end(); + }); }); -}); -server.listen(port, hostname, () => { - console.log(`[web] listening on http://${hostname}:${port}, in-flight cap ${inflight().limit}`); -}); + server.listen(port, hostname, () => { + console.log( + `[web] ${process.pid} listening on http://${hostname}:${port}, in-flight cap ${inflight().limit}`, + ); + }); +} + +/** Say so when refusing starts, and then once a minute while it goes on. */ +let lastNoted = 0; /** * The refusal: 503, tiny, uncacheable, with a Retry-After a client can obey. @@ -67,7 +111,9 @@ function refuse(res) { if (now - lastNoted > 60_000) { lastNoted = now; const { active, limit, refused } = inflight(); - console.warn(`[web] shedding load: ${active}/${limit} in flight, ${refused} refused so far`); + console.warn( + `[web] ${process.pid} shedding load: ${active}/${limit} in flight, ${refused} refused so far`, + ); } res.writeHead(503, { diff --git a/apps/web/src/lib/loadShed.js b/apps/web/src/lib/loadShed.js index 7bf2b07..51a3159 100644 --- a/apps/web/src/lib/loadShed.js +++ b/apps/web/src/lib/loadShed.js @@ -49,8 +49,10 @@ * standing Next up. */ +import { share } from './workers.js'; + /** - * How many requests may be in flight. + * How many requests may be in flight, across the whole container. * * Sized from the incident: the process survived an hour at roughly 20 a * second with sub-second responses, which is fewer than twenty in flight, and @@ -60,6 +62,12 @@ * process's footprint is bounded at the cap times one request's worth of * work — tens of megabytes at the top end — rather than at whatever the * arrival rate happens to be. + * + * Since 2026-09-07 the container runs one server per CPU rather than one in + * total (`workers.js`), and this number is divided between them. It stayed as + * a container-wide figure on purpose: it was sized against a container's heap, + * and giving each of sixteen workers the whole of it would raise the real + * ceiling to 2,048 and hand back the outage it was written to prevent. */ const DEFAULT_LIMIT = 128; @@ -75,7 +83,12 @@ const DEFAULT_LIMIT = 128; const ALWAYS = /^\/(?:_next\/static\/|icons\/|favicon\.ico$|manifest\.webmanifest$|sw\.js$|robots\.txt$)/; /** - * The limit, from the environment when it is set to something sensible. + * This process's limit, from the environment when it is set to something + * sensible. + * + * `WEB_MAX_INFLIGHT` is read as a container-wide number, like the default it + * replaces, and divided the same way — so the dial keeps meaning what it meant + * before there were workers, and raising it does not have to be done per CPU. * * Read through a non-literal property access for the reason `lib/db.js` gives, * and junk falls back to the default rather than to unlimited, for the reason @@ -86,7 +99,8 @@ const ALWAYS = /^\/(?:_next\/static\/|icons\/|favicon\.ico$|manifest\.webmanifes export function limit() { const env = process.env; const raw = Number(env['WEB_MAX_INFLIGHT']); - return Number.isInteger(raw) && raw > 0 ? raw : DEFAULT_LIMIT; + const total = Number.isInteger(raw) && raw > 0 ? raw : DEFAULT_LIMIT; + return share(total); } /** Requests currently being worked on. */ diff --git a/apps/web/src/lib/workers.js b/apps/web/src/lib/workers.js new file mode 100644 index 0000000..e50f050 --- /dev/null +++ b/apps/web/src/lib/workers.js @@ -0,0 +1,249 @@ +/** + * How many copies of the server to run, and how much heap to give each. + * + * ## The outage this exists to prevent + * + * On 2026-09-07 the site was dark while twenty-two of its twenty-four CPUs sat + * idle. A residential-proxy fleet was walking `/topics/*`, `/api/topics/*` and + * the reader at roughly 140 requests a second — five hundred requests from five + * hundred distinct addresses, no path asked for twice, so neither the throttle + * (which meters a caller) nor any cache (which needs a repeat) touched a single + * one of them. The container held one `node server.mjs`. Rendering a topic page + * costs around 150 ms of JavaScript, and JavaScript runs on one thread, so the + * whole site could serve about seven requests a second no matter how much + * hardware it was standing on. Everything above that queued behind the + * in-flight ceiling, the event loop never got back to `accept()`, and the edge + * proxy started reporting `connection dial timeout` — the site was down for + * readers too, not only for the fleet. + * + * The 2026-09-03 incident that `loadShed.js` was written for was a memory + * problem and the ceiling was the right answer to it. This is a different + * failure with the same symptom: not enough of the machine was being used. A + * ceiling protects a process that is working too hard; it cannot make a process + * work on more than one thing at a time. + * + * ## What this does not fix + * + * Capacity is not a defence. Running the server on every core raises what the + * site can absorb by more than an order of magnitude, which is enough to absorb + * *this* fleet, and a fleet twice the size would put it back where it started. + * The thing that actually stops a distributed scrape is refusing it before it + * costs a render — see the note in `crawlThrottle.js` about limits keyed on who + * is asking, and `rssamplifier-residential-proxy-fleet` for why the header + * checks in the gateway do not catch this one. This module buys the room to go + * build that; it is not that. + * + * ## Reading the budget from the cgroup, not from `os` + * + * `os.availableParallelism()` reports the *host's* CPUs — 48 on this Railway + * machine — while the container is allowed 24. Forking a worker per host CPU + * would put twice as many runnable threads on the quota as it can run, and the + * scheduler would spend the difference on context switches. The same trap + * applies to memory: `os.totalmem()` is the host's 393 GB and the container's + * limit is 24 GB. Both budgets come from the cgroup, and only fall back to the + * `os` numbers when there is no cgroup to read (a developer's laptop). + */ + +import cluster from 'node:cluster'; +import { readFileSync } from 'node:fs'; +import os from 'node:os'; + +/** + * The most workers to run whatever the machine offers. + * + * Each worker is a whole Next server with its own module state, its own render + * caches and its own heap, so the cost of one is real and the return falls off + * once there are enough of them to keep the CPU quota busy. Sixteen is above + * what this container can run in parallel anyway and keeps the arithmetic on + * heap (below) somewhere sane. + */ +const MAX_WORKERS = 16; + +/** + * The share of the container's memory the workers may size their heaps to. + * + * A `--max-old-space-size` is a ceiling and not a reservation, so the sum of + * the workers' ceilings is allowed to exceed what the container has — every + * worker reaching its ceiling at the same moment is not a state this survives + * either way. What the fraction buys is the rest: the build's own files in page + * cache, the copies of Next that are not heap, and the headroom that makes the + * difference between a slow minute and the platform killing the container. + */ +const HEAP_FRACTION = 0.6; + +/** Never hand a worker less heap than this. */ +const MIN_HEAP_MB = 512; + +/** Never hand a worker more than V8 would have taken on its own. */ +const MAX_HEAP_MB = 12_288; + +/** + * An integer environment variable, or the fallback. + * + * Read through a non-literal property access for the reason `lib/db.js` gives, + * and junk falls back rather than to some other number, for the reason + * `pageGate.js` gives: a typo in a limit must not be the thing that removes it. + * + * @param {string} name + * @returns {number | null} + */ +function envInt(name) { + const env = process.env; + const raw = Number(env[name]); + return Number.isInteger(raw) && raw > 0 ? raw : null; +} + +/** + * The first line of a file, or null when it cannot be read. + * + * Every cgroup file this module wants is absent on a machine that is not a + * container, and that is the ordinary case on a laptop rather than an error. + * + * @param {string} path + * @returns {string | null} + */ +function readOrNull(path) { + try { + return readFileSync(path, 'utf8').trim(); + } catch { + return null; + } +} + +/** + * How many CPUs this process may actually use at once. + * + * cgroup v2 states it as `quota period` in microseconds — `2400000 100000` is + * 24 CPUs — and the literal `max` means no quota, in which case the host's + * count is the truth. v1 splits the same pair across two files and writes `-1` + * for no quota. + * + * @returns {number} + */ +export function cpuBudget() { + const v2 = readOrNull('/sys/fs/cgroup/cpu.max'); + if (v2) { + const [quota, period] = v2.split(/\s+/); + if (quota !== 'max') { + const cpus = Math.floor(Number(quota) / Number(period)); + if (cpus >= 1) return cpus; + } + } + + const quota = Number(readOrNull('/sys/fs/cgroup/cpu/cpu.cfs_quota_us')); + const period = Number(readOrNull('/sys/fs/cgroup/cpu/cpu.cfs_period_us')); + if (quota > 0 && period > 0) { + const cpus = Math.floor(quota / period); + if (cpus >= 1) return cpus; + } + + return os.availableParallelism(); +} + +/** + * How much memory this process may actually use, in bytes. + * + * Both cgroup versions write a sentinel when there is no limit — `max` in v2, a + * number near 2^63 in v1 — and either means the host's total is the ceiling. + * + * @returns {number} + */ +export function memoryBudget() { + const v2 = readOrNull('/sys/fs/cgroup/memory.max'); + if (v2 && v2 !== 'max') { + const bytes = Number(v2); + if (bytes > 0) return bytes; + } + + const v1 = Number(readOrNull('/sys/fs/cgroup/memory/memory.limit_in_bytes')); + if (v1 > 0 && v1 < os.totalmem() * 4) return v1; + + return os.totalmem(); +} + +/** + * How many workers to run. + * + * `WEB_WORKERS` overrides, and `WEB_WORKERS=1` is the way back to the single + * process this replaced — worth having, because a bug that only appears with + * more than one of something is diagnosed by turning the something off. + * + * @returns {number} + */ +export function workerCount() { + const forced = envInt('WEB_WORKERS'); + if (forced) return Math.min(forced, MAX_WORKERS); + + return Math.max(1, Math.min(cpuBudget(), MAX_WORKERS)); +} + +/** + * The heap ceiling for one worker, in megabytes. + * + * The Dockerfile's `NODE_OPTIONS` sets a ceiling sized for one process holding + * the whole container. Inheriting that into every worker would tell each of + * sixteen processes it may take half the container, so the primary computes a + * share instead and passes it on the workers' command line, where it wins over + * the inherited option. With one worker the share is the whole allowance and + * nothing changes from the single-process arrangement. + * + * @param {number} [count] workers to divide the allowance between + * @returns {number} + */ +export function workerHeapMb(count = workerCount()) { + const share = (memoryBudget() * HEAP_FRACTION) / count / (1024 * 1024); + return Math.max(MIN_HEAP_MB, Math.min(MAX_HEAP_MB, Math.floor(share))); +} + +/** + * One worker's share of a limit that is meant to hold for the service. + * + * The in-flight ceiling is the case this exists for. It bounds the memory of + * one process, so it has to be applied per worker — but the number it was sized + * at, 128, was sized against a container, and leaving 128 on each of sixteen + * workers would raise the real ceiling to 2,048 and give back the outage it was + * written to prevent. + * + * Never below one, because a share that rounds to zero refuses everything. + * + * @param {number} total the whole service's allowance + * @param {number} [count] workers to divide it between + * @returns {number} + */ +export function share(total, count = workerCount()) { + return Math.max(1, Math.round(total / count)); +} + +/** + * Run `serve` in each of `workerCount()` worker processes. + * + * Returns false in a worker and in the single-worker case, meaning "you are the + * one doing the work, get on with it". Returns true in a primary that has forked + * — there is nothing else for that process to do, and it must not also bind the + * port. + * + * A worker that dies is replaced. The alternative is a container that keeps + * answering on fewer and fewer processes and never reports that it is degraded, + * which is a worse failure than a restart: the platform's own restart policy + * cannot see inside the container, so nothing else is watching these. + * + * @param {{ onExit?: (info: { pid: number | undefined, code: number, signal: string | null }) => void }} [hooks] + * @returns {boolean} true when this process is a primary that has forked workers + */ +export function forkWorkers(hooks = {}) { + const count = workerCount(); + if (count <= 1 || !cluster.isPrimary) return false; + + cluster.setupPrimary({ + execArgv: [...process.execArgv, `--max-old-space-size=${workerHeapMb(count)}`], + }); + + for (let i = 0; i < count; i += 1) cluster.fork(); + + cluster.on('exit', (worker, code, signal) => { + hooks.onExit?.({ pid: worker.process.pid, code, signal }); + cluster.fork(); + }); + + return true; +} diff --git a/apps/web/test/load-shed.test.js b/apps/web/test/load-shed.test.js index c4ada70..c6730d4 100644 --- a/apps/web/test/load-shed.test.js +++ b/apps/web/test/load-shed.test.js @@ -12,11 +12,35 @@ import { admit, inflight, limit, reset } from '../src/lib/loadShed.js'; * everything with nothing in flight. */ +// The ceiling is a container-wide number divided between the workers, so a +// test that asserts on the number a process actually enforces has to say how +// many workers there are. One, here: these tests are about the accounting, and +// the division has its own tests in workers.test.js. test.beforeEach(() => { delete process.env.WEB_MAX_INFLIGHT; + process.env.WEB_WORKERS = '1'; reset(); }); +test.after(() => { + delete process.env.WEB_WORKERS; +}); + +test('the configured ceiling is divided between the workers', () => { + process.env.WEB_MAX_INFLIGHT = '128'; + process.env.WEB_WORKERS = '16'; + assert.equal(limit(), 8, 'each of sixteen workers gets an eighth of a container-wide 128'); + + process.env.WEB_WORKERS = '1'; + assert.equal(limit(), 128, 'one worker gets all of it'); +}); + +test('a share never rounds down to nothing', () => { + process.env.WEB_MAX_INFLIGHT = '4'; + process.env.WEB_WORKERS = '16'; + assert.equal(limit(), 1, 'a worker that may admit nothing would refuse every request'); +}); + test('admits up to the limit and refuses the next', () => { process.env.WEB_MAX_INFLIGHT = '3'; diff --git a/apps/web/test/workers.test.js b/apps/web/test/workers.test.js new file mode 100644 index 0000000..d2d7b8e --- /dev/null +++ b/apps/web/test/workers.test.js @@ -0,0 +1,79 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import os from 'node:os'; + +import { cpuBudget, memoryBudget, share, workerCount, workerHeapMb } from '../src/lib/workers.js'; + +/** + * The worker pool's arithmetic. + * + * What it must get right is that every number stays inside what the *container* + * has. Reading the host's CPUs would fork twice as many workers as this machine + * can run; reading the host's memory would hand each of them a heap ceiling + * forty times the container's limit, and the platform would kill it. Neither + * mistake announces itself — the site comes up and is simply worse — so the + * budgets are asserted against the cgroup values wherever there is a cgroup to + * read, and the shares are asserted for the cases that round badly. + */ + +test.beforeEach(() => { + delete process.env.WEB_WORKERS; +}); + +test.after(() => { + delete process.env.WEB_WORKERS; +}); + +test('the CPU budget is a whole number of usable CPUs', () => { + const cpus = cpuBudget(); + assert.ok(Number.isInteger(cpus), 'a fractional worker cannot be forked'); + assert.ok(cpus >= 1, 'there is always at least one CPU to run on'); + assert.ok(cpus <= os.availableParallelism(), 'a quota cannot exceed the machine it is on'); +}); + +test('the memory budget never exceeds the machine', () => { + const bytes = memoryBudget(); + assert.ok(bytes > 0); + assert.ok(bytes <= os.totalmem(), 'a container limit above the host total is the no-limit sentinel'); +}); + +test('WEB_WORKERS decides the count, and 1 is the way back to one process', () => { + process.env.WEB_WORKERS = '4'; + assert.equal(workerCount(), 4); + + process.env.WEB_WORKERS = '1'; + assert.equal(workerCount(), 1); +}); + +test('the count is capped however many CPUs the machine offers', () => { + process.env.WEB_WORKERS = '512'; + assert.ok(workerCount() <= 16, 'a worker costs a whole Next server; the return falls off'); +}); + +test('junk in WEB_WORKERS falls back to the CPU budget, never to zero', () => { + for (const junk of ['', 'lots', '0', '-4', '2.5', 'NaN']) { + process.env.WEB_WORKERS = junk; + const count = workerCount(); + assert.ok(count >= 1, `${JSON.stringify(junk)} does not leave the site with no servers`); + assert.equal(count, Math.max(1, Math.min(cpuBudget(), 16))); + } +}); + +test('a worker heap is a share of the container, not of the host', () => { + const one = workerHeapMb(1); + const sixteen = workerHeapMb(16); + + assert.ok(sixteen <= one, 'more workers means less heap each'); + assert.ok(one <= 12_288, 'never more than V8 would have taken on its own'); + assert.ok(sixteen >= 512, 'never so little that a single render cannot finish'); + + const containerMb = memoryBudget() / (1024 * 1024); + assert.ok(one <= containerMb, 'one worker may not be promised more than the container has'); +}); + +test('a share divides a service-wide allowance and never rounds to zero', () => { + assert.equal(share(128, 16), 8); + assert.equal(share(128, 1), 128); + assert.equal(share(4, 16), 1, 'a worker allowed nothing would refuse everything'); + assert.equal(share(1, 16), 1); +});