diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 129e4bd..6bde792 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,6 +14,11 @@ jobs: node: [18, 20, 22, 24] part: [chain, rest] runs-on: ubuntu-latest + # the kit won't start a worker while the machine's memory is over its ceiling: the tests read a + # machine of their own (or none), never the runner's + env: + DEVTEAM_CGROUP: /nonexistent/devteam-cgroup + DEVTEAM_MEMINFO: /nonexistent/devteam-meminfo # a hung test should fail the job, not hold it for hours timeout-minutes: 30 steps: diff --git a/docs/devteam/process/supervisor.md b/docs/devteam/process/supervisor.md index 65e1e85..7645e1d 100644 --- a/docs/devteam/process/supervisor.md +++ b/docs/devteam/process/supervisor.md @@ -116,12 +116,13 @@ Production is watched by **sensors**, and they are yours: you add, change and st | PR head differs from the last phase line's `sha` | stale liveness | message the worker to run `phase` | | owner answer relayed, not in the spec or plan after the next commit | answer not recorded | message the worker | | ticket ready, no worker, under caps | idle capacity | schedule it, or hold it with the reason | +| ticket ready or a goal unplanned, but the machine's memory at `machine_memory_ceiling` | memory full | start nothing new until sessions finish (resumes are fine); lower your `max_active_tickets` if it keeps happening (§5) | | goal fully defined (or its current milestone unplanned), no planner, under `max_active_planners` (no cap by default) | idle planning | start a planner for it now: every goal plans in parallel. The one exception is a goal whose design depends on another goal's design; record it (`goal set --plan-after --reason "…"`, shown to the owner on the goal) and start it once that one is approved | | superseded ticket with a live worker | re-plan not propagated | park it with a ruling | ## 5. Scheduling and merging -- `devteam.mjs ready` lists tickets whose dependencies are merged and that have no worker; `--plan-ahead` adds those that can spec and plan now. To keep a ready ticket unscheduled on purpose (it may duplicate other work, it waits on the owner), `ticket set --json '{"held": ""}'`; `ready` and the idle-capacity finding then leave it out, and `{"held": null}` releases it. Caps: `max_active_tickets` (your review capacity is the real limit) and `max_active_per_component`. Prefer the critical path, then contract tickets, then tickets that close a story. +- `devteam.mjs ready` lists tickets whose dependencies are merged and that have no worker; `--plan-ahead` adds those that can spec and plan now. To keep a ready ticket unscheduled on purpose (it may duplicate other work, it waits on the owner), `ticket set --json '{"held": ""}'`; `ready` and the idle-capacity finding then leave it out, and `{"held": null}` releases it. Caps: `max_active_tickets` (your review capacity is the real limit) and `max_active_per_component`. The machine is shared with other projects' fleets, so a new worker or planner doesn't start while the machine's memory is at `machine_memory_ceiling` (default 0.8 of its cap, from 0.5 to 0.95); a resume of a session that already ran always starts (`--resume` on one that never ran is a new start). When a start is refused for memory, or the `memory-full` finding shows in place of idle capacity, wait for sessions to finish rather than passing `--force`: forcing it is how the box runs out and restarts every fleet on it. If you keep hitting the ceiling, lower your own `max_active_tickets`; it is your setting, not a question for the owner. Prefer the critical path, then contract tickets, then tickets that close a story. - `devteam.mjs worker `; `worker --resume` relaunches a crashed one in the same window and worktree with `resume_command`. - **Merging.** A `merge-request` message names the exact head. Read the diff, the final review record and `verification.md` yourself, then `devteam.mjs merge `. It runs `check-ticket` at that head, the log rule, and every gate in the settings in a clean worktree at that head, merges with `--match-head-commit` (or, with `forge: "git"`, merges that sha and pushes), and logs the act. `--dry-run` does everything but the merge (and works on a branch with no PR yet). A ticket you added since the log last landed fails the forge's checks (`project/tickets// has no ticket`) until its line is on main, because the forge reads only the landed log: land the log first (`log-sync`, then `merge --chore log`) and rerun them. If anything fails it says what; reply to the worker with `--kind ruling --re `. A head that moved after the request is a new request. You never merge any other way, and workers never merge. - **Other repos.** `merge` works per repo; the ticket says which (its component's `link`). `fleet status` lists each repo link with its path, and a sibling whose origin it can't fetch is a blocker. A ticket whose repo has no checkout on this machine waits as "repo unreachable"; `link-unreachable` says what to clone where. diff --git a/docs/devteam/scripts/devteam.mjs b/docs/devteam/scripts/devteam.mjs index 30fadbb..3dbba3d 100644 --- a/docs/devteam/scripts/devteam.mjs +++ b/docs/devteam/scripts/devteam.mjs @@ -496,6 +496,11 @@ const SETTINGS_DEFAULTS = { // planning gets one unless its design waits on another goal's (goal set --plan-after). They don't // count against max_active_tickets. max_active_planners: null, + // The share of the machine's memory (its cgroup cap, else the host's) at or above which a new + // worker or planner doesn't start (--force does anyway); a resume always does. The machine is + // shared with other fleets, so this is the supervisor's, like the caps above: 0.5 to 0.95, and + // an invalid value counts as the default. + machine_memory_ceiling: 0.8, stale_after_minutes: 30, // A goal parked this long (not by a fleet op) is worth asking the owner about again. hold_review_days: 14, @@ -2978,6 +2983,7 @@ function cmdValidate() { const s = loadSettings(); const reg = registryProblems(s); errors.push(...reg.errors, ...sensorProblems(s)); + { const p = memoryCeilingProblem(s.machine_memory_ceiling); if (p) errors.push(p); } warnings.push(...reg.warnings); // With lines, check-pr and merge judge the tree a PR would land, which takes git 2.38; say so // here, in the words merge refuses with, before the first PR meets it. @@ -10805,6 +10811,9 @@ function collectMetrics(since) { for (const [id, n] of Object.entries(reviewPasses()).filter(([id2, n]) => n >= 3 && !mergeShaFor(id2))) add('medium', 'review-churn', id, `${n} final review passes`, 'the ticket may be too big or under-specified; re-plan it'); // An old fleet drain in force starts nothing; goal holds start nothing for their goals. const drained = !!draining(); + // over the machine's memory ceiling, no finding suggests starting a session; memory-full says why + const memFull = memoryFull(settings); + let idlePlanning = 0; const ready = [...ts.values()].filter((t) => isActive(t) && !isHeld(t) && !mergeShaFor(t.id) && !activeWorker(t.id) && (t.depends_on ?? []).every((d) => mergeShaFor(d)) && !notSchedulable(t) && ticketMode(t) === 'run'); holdFindings(add, settings); // What matters most: the strategy, each goal's impact, and the ratings the Inbox is ordered by. @@ -10866,12 +10875,15 @@ function collectMetrics(since) { }).sort((a, b) => (a.priority ?? 99) - (b.priority ?? 99)); const cap = settings.max_active_planners; // while the fleet drains for a move, nothing new starts: no idle planning, no idle capacity - if (!drained && waiting.length && (cap == null || ps.length < cap)) add('medium', 'idle-planning', `${waiting.length} goal${waiting.length > 1 ? 's' : ''}`, `${ps.length}${cap == null ? '' : `/${cap}`} planner${ps.length === 1 ? '' : 's'} active; nobody is planning ${waiting.map((g) => `"${g.title}"`).join(', ')}`, `start a planner for each (goals plan in parallel), or record why one waits on another's design (goal set --plan-after --reason "…"): ${waiting.slice(0, cap == null ? waiting.length : Math.max(1, cap - ps.length)).map((g) => g.id).join(', ')}`); + if (!drained && waiting.length && (cap == null || ps.length < cap)) idlePlanning = waiting.length; + if (idlePlanning && !memFull) add('medium', 'idle-planning', `${waiting.length} goal${waiting.length > 1 ? 's' : ''}`, `${ps.length}${cap == null ? '' : `/${cap}`} planner${ps.length === 1 ? '' : 's'} active; nobody is planning ${waiting.map((g) => `"${g.title}"`).join(', ')}`, `start a planner for each (goals plan in parallel), or record why one waits on another's design (goal set --plan-after --reason "…"): ${waiting.slice(0, cap == null ? waiting.length : Math.max(1, cap - ps.length)).map((g) => g.id).join(', ')}`); } // a ready fix ticket may use an incident slot past the cap const readyFix = ready.filter((t) => t.kind === 'fix').map((t) => ({ ...t, sev: incidentSeverity(incidentOfTicket(t.id) ?? { fingerprints: [] }) })).sort((a, b) => b.sev - a.sev); - if (!drained && readyFix.length && active >= settings.max_active_tickets && active < settings.max_active_tickets + (settings.incident_slots ?? 0)) add('medium', 'idle-capacity', `${readyFix.length} fix ready`, `${active}/${settings.max_active_tickets} workers active, and an incident slot is free`, `make a worker for: ${readyFix[0].id}`); - if (!drained && ready.length && active < settings.max_active_tickets) add('medium', 'idle-capacity', `${ready.length} ready`, `${active}/${settings.max_active_tickets} workers active`, `make workers for: ${ready.slice(0, 5).map((t) => t.id).join(', ')}`); + if (!drained && readyFix.length && active >= settings.max_active_tickets && active < settings.max_active_tickets + (settings.incident_slots ?? 0) && !memFull) add('medium', 'idle-capacity', `${readyFix.length} fix ready`, `${active}/${settings.max_active_tickets} workers active, and an incident slot is free`, `make a worker for: ${readyFix[0].id}`); + if (!drained && ready.length && active < settings.max_active_tickets && !memFull) add('medium', 'idle-capacity', `${ready.length} ready`, `${active}/${settings.max_active_tickets} workers active`, `make workers for: ${ready.slice(0, 5).map((t) => t.id).join(', ')}`); + // the slots and planning above that the machine's memory leaves no room for: say that instead + if (memFull && (idlePlanning || (!drained && ((ready.length && active < settings.max_active_tickets) || (readyFix.length && active < settings.max_active_tickets + (settings.incident_slots ?? 0)))))) add('medium', 'memory-full', 'machine', `${memFull.words}, with ${ready.length} ticket${ready.length === 1 ? '' : 's'} ready, ${active}/${settings.max_active_tickets} workers active${idlePlanning ? `, and ${idlePlanning} waiting for a planner` : ''}`, 'start nothing new until some finish (a resume is fine); if this keeps happening, lower max_active_tickets, your own setting'); // A chat answer's quote that the supervisor's transcript doesn't show the owner typing: a record // for the owner to look at, never a block. for (const q of chatQuotesNotFound()) add('medium', 'chat-quote-not-found', q.id, `recorded from chat as the owner's words, "${q.said.slice(0, 120)}", and the supervisor's transcript has no owner message with them`, 'check it on the dashboard; if the owner never said it, correct the answer'); @@ -12419,6 +12431,42 @@ function versionBefore(a, b) { const portAnswers = async (port) => { try { await fetch(`http://127.0.0.1:${port}/`, { signal: AbortSignal.timeout(3000) }); return 'up'; } catch { return 'DOWN'; } }; const gb = (k) => `${Math.round(k / 1048576)}G`; const readNum = (f) => { const t = readText(f)?.trim(); return t && /^\d+$/.test(t) ? Number(t) : null; }; +// The machine's memory in bytes, { inUse, max, source }, or null when nothing says (macOS: no cgroup +// and no /proc/meminfo) or a file can't be read. The cgroup's cap first, since a platform caps the +// container there (cgroup v2's files: on v1, or a cap set on a parent cgroup, this reads the host's): +// in use is what the kernel can't drop, memory.current less the page cache it can (memory.stat's +// file, less its shmem, which lives in tmpfs and /dev/shm and can't be dropped). With no cap (no +// cgroup, or memory.max "max"), the host's /proc/meminfo: MemTotal less MemAvailable. +// DEVTEAM_CGROUP and DEVTEAM_MEMINFO point both elsewhere (tests). +function machineMemory() { + try { + const cg = process.env.DEVTEAM_CGROUP ?? '/sys/fs/cgroup'; + const max = readNum(path.join(cg, 'memory.max')); + const current = readNum(path.join(cg, 'memory.current')); + if (max > 0 && current !== null) { + const stat = readText(path.join(cg, 'memory.stat')) ?? ''; + const field = (k) => Number(new RegExp(`^${k} (\\d+)$`, 'm').exec(stat)?.[1] ?? 0); + return { inUse: Math.max(0, current - Math.max(0, field('file') - field('shmem'))), max, source: 'cgroup' }; + } + const info = readText(process.env.DEVTEAM_MEMINFO ?? '/proc/meminfo'); + const kb = (k) => Number(new RegExp(`^${k}:\\s+(\\d+)`, 'm').exec(info ?? '')?.[1] ?? NaN); + const total = kb('MemTotal'), avail = kb('MemAvailable'); + if (!(total > 0) || !Number.isFinite(avail)) return null; + return { inUse: Math.max(0, total - avail) * 1024, max: total * 1024, source: 'meminfo' }; + } catch { return null; } +} +const GiB = (b) => (b / 1073741824).toFixed(1); +const MEMORY_CEILING_DEFAULT = 0.8; +const memoryCeilingProblem = (v) => (typeof v === 'number' && v >= 0.5 && v <= 0.95 ? null : `settings.machine_memory_ceiling is ${JSON.stringify(v) ?? String(v)}: it is the share of the machine's memory at which new workers stop starting, a number from 0.5 to 0.95 (default ${MEMORY_CEILING_DEFAULT}); until it is, ${MEMORY_CEILING_DEFAULT} applies`); +// The machine's memory at or over the supervisor's ceiling: { words, … }, or null when under it or +// when nothing says (never a block where the kit can't read memory). An invalid ceiling counts as +// the default, so no value widens what a supervisor may start. +function memoryFull(s = loadSettings()) { + const ceiling = memoryCeilingProblem(s.machine_memory_ceiling) ? MEMORY_CEILING_DEFAULT : s.machine_memory_ceiling; + const m = machineMemory(); + if (!m || m.inUse < ceiling * m.max) return null; + return { ...m, ceiling, words: `the machine's memory is at ${GiB(m.inUse)} of ${GiB(m.max)} GB (${Math.round((m.inUse * 100) / m.max)}%), at or over its ${Math.round(ceiling * 100)}% ceiling` }; +} // The machine's own caps, not the host's: a platform caps CPU and memory in the cgroup, while // /proc/meminfo and the CPU count say what the whole host has (a 4-CPU, 16 GB box reads 32 CPUs, 123 GB). function machineLine(last) { @@ -12430,17 +12478,8 @@ function machineLine(last) { const trend = moved ? ` (${d > 0 ? '+' : ''}${Math.round(d)}G in ${Math.trunc((now - (last.at ?? now)) / 60)}m)` : ''; const low = free < 20971520 || Math.floor((free * 100) / (total + 1)) < 15 ? ' LOW' : ''; const cg = process.env.DEVTEAM_CGROUP ?? '/sys/fs/cgroup'; - const max = readText(path.join(cg, 'memory.max'))?.trim(); - let mem; - if (max && max !== 'max') { - // in use: what the kernel can't drop (the page cache it can), against the cap - const file = Number(/^file (\d+)$/m.exec(readText(path.join(cg, 'memory.stat')) ?? '')?.[1] ?? 0); - mem = `${(Math.max(0, (readNum(path.join(cg, 'memory.current')) ?? 0) - file) / 1073741824).toFixed(1)} GB in use of ${(Number(max) / 1073741824).toFixed(1)} GB`; - } else { - const info = readText('/proc/meminfo'); - const kb = (k) => Number(new RegExp(`^${k}:\\s+(\\d+)`, 'm').exec(info ?? '')?.[1] ?? NaN); - mem = info ? `${(kb('MemAvailable') / 1048576).toFixed(1)} GB free of ${(kb('MemTotal') / 1048576).toFixed(1)} GB` : `${(os.freemem() / 1073741824).toFixed(1)} GB free of ${(os.totalmem() / 1073741824).toFixed(1)} GB`; - } + const m = machineMemory(); + const mem = m?.source === 'cgroup' ? `${GiB(m.inUse)} GB in use of ${GiB(m.max)} GB` : m ? `${GiB(m.max - m.inUse)} GB free of ${GiB(m.max)} GB` : `${GiB(os.freemem())} GB free of ${GiB(os.totalmem())} GB`; // the CPUs it may use, and how much of the last minute every task stood waiting for one const [quota, period] = (readText(path.join(cg, 'cpu.max')) ?? '').trim().split(/\s+/); const cpus = quota && quota !== 'max' && Number(period) > 0 ? String(Number((Number(quota) / Number(period)).toPrecision(6))) : String(os.cpus().length || '?'); @@ -13775,6 +13814,14 @@ function makeSession(role) { const started = role === 'worker' ? !!lastPhaseOf(id) : !!resolveCommit(`refs/heads/plan/${id}`); const refused = ['park', 'let-go'].includes(mode) || (mode === 'drain' && !started); if (refused && !flags.force) die(legacyDrain ? `the fleet is drained by an old drain (${legacyDrain.id}): fleet unpark first (supervisor.md §12); to start this one anyway, add --force (it is logged)` : `its goal is ${holdWords(mode)}: ${mode === 'drain' ? 'a draining goal finishes what is running and starts nothing new' : 'nothing starts for it'} (planning.md §1, holds). To start it anyway, add --force (it is logged)`); + // The machine is shared with other fleets: a new session doesn't start while its memory is at the + // supervisor's ceiling (supervisor.md §5). A resume of one that already ran always does, so a + // dead machine's sessions all come back; --resume on one that never ran is a new start. + // A planner moved from another machine has its plan branch on origin only, and continues from it. + const ran = started || (role === 'planner' && !!resolveCommit(`refs/remotes/origin/plan/${id}`)); + const memNow = memoryFull(); + const memFull = flags.resume && ran ? null : memNow; + if (memFull && !flags.force) die(`${memFull.words}: start this ${role} when some sessions finish, or pass --force (it is logged)`); if (!sh('tmux', ['-V'])) die('tmux not found'); // A warning only: the numbers can be 10 minutes old, and the supervisor may know better. Not for // a relaunch at a limit: the tick starts that once there's room, and this file can lag it. @@ -13883,7 +13930,7 @@ function makeSession(role) { if (role === 'worker' && !resume) appendLog({ type: 'phase', from: null, to: 'starting', note: `started in ${shown}` }, id); // a planner parked by a park is running again if (role === 'planner' && lastPhaseOf(name)?.to === 'parked') appendLog({ type: 'phase', from: 'parked', to: 'starting', note: `started in ${shown}` }, name); - appendLog({ type: 'act', decision: citing(), class: 'worker-slots', what: `${resume ? 'resumed' : 'made'} ${role} ${name} in ${shown}${block ? `, ticket numbers ${padNN(block.from)}-${padNN(block.to)}` : ''}${refused ? ` while its goal is ${holdWords(mode)}, with --force` : ''}`, refs: role === 'worker' ? [id] : goalIds, reversible: true, id_block: block ?? undefined, knowledge: given.length ? given : undefined }, 'supervisor'); + appendLog({ type: 'act', decision: citing(), class: 'worker-slots', what: `${resume ? 'resumed' : 'made'} ${role} ${name} in ${shown}${block ? `, ticket numbers ${padNN(block.from)}-${padNN(block.to)}` : ''}${refused || memFull ? ` while ${[refused && `its goal is ${holdWords(mode)}`, memFull?.words].filter(Boolean).join(' and ')}, with --force` : memNow && !resume ? `, a resume that started fresh while ${memNow.words}` : ''}`, refs: role === 'worker' ? [id] : goalIds, reversible: true, id_block: block ?? undefined, knowledge: given.length ? given : undefined }, 'supervisor'); out(`${resume ? 'resumed' : 'made'} ${role} ${name} in ${shown} at ${wt}${block ? `; ticket numbers ${padNN(block.from)}-${padNN(block.to)} reserved` : ''}`); }); } catch (e) { @@ -17129,7 +17176,7 @@ Setup and roles supervisor-session the id of the supervisor's Claude Code conversation (claude --resume ); nothing if there's none, or another harness recover [--dry-run] [--force] [--here] | recover --machine [--dry-run] [--force] [--in …] [--was-up ] [--session s] [--only ] after a restart (a dead laptop, a redeploy): bring back this repo's supervisor, or every repo's on this machine, if it was up and its fleet isn't held, on its exact conversation where the harness can (Claude Code); --force also starts one the owner stopped or whose fleet is held; --here runs it in this terminal; safe to run twice. --in adds every adopted repo in a folder; --was-up names a list of roots that were up (a box's), used up as they come back; --session puts each in that tmux session as sup- machine status [--questions | --knowledge] [--in …] [--notes ] | machine tell | machine alert |--clear | machine pane-state every project on this machine (recorded in ~/.devteam, and under --in): a read-only digest (with when the operator's daily review is due, from --notes), or the open questions, or the accepted knowledge, not in --notes; a message into a repo's supervisor, marked as not the owner's, only when its pane is at its prompt; a line on the hub and the owner's devices; what a pane is at (ready busy dialog limit typing shell other unknown gone). tell and alert are the operator's or the owner's - worker [--resume [--if-gone]] [--force] | worker --remove [--force] | planner [--goal ] [--resume] [--force] [--id-block ] supervisor: a worktree, a tmux window and an agent for a ticket or a feature plan (a new planner names its goal and reserves n ticket numbers, default 20); refused while its goal is parked or let go, or draining for a session that hasn't run, unless --force (logged); --resume where no session exists starts fresh with the handoff note; worker --remove, once the ticket is merged and its worker logged phase done (--force before that), removes its worktree through the repo it belongs to and prunes that repo's worktree list (the branch stays) + worker [--resume [--if-gone]] [--force] | worker --remove [--force] | planner [--goal ] [--resume] [--force] [--id-block ] supervisor: a worktree, a tmux window and an agent for a ticket or a feature plan (a new planner names its goal and reserves n ticket numbers, default 20); refused while its goal is parked or let go, or draining for a session that hasn't run, or (a resume of one that ran aside) while the machine's memory is at its machine_memory_ceiling, unless --force (logged); --resume where no session exists starts fresh with the handoff note; worker --remove, once the ticket is merged and its worker logged phase done (--force before that), removes its worktree through the repo it belongs to and prunes that repo's worktree list (the branch stays) whoami | settings [--get key] hello [--json] who this instance is (root, name, kit, open questions, fleet): what the hub lists diff --git a/test/lib/cli.mjs b/test/lib/cli.mjs index 1c823ae..c3ee667 100644 --- a/test/lib/cli.mjs +++ b/test/lib/cli.mjs @@ -36,7 +36,14 @@ function start() { slots.push({ dir, child }); } +// The kit reads the machine's memory before it starts a session (and before it suggests one): a test +// reads none of the machine it runs on, where a busy runner would refuse its workers, unless it +// names its own (DEVTEAM_CGROUP, DEVTEAM_MEMINFO). +const NOWHERE = path.join(os.tmpdir(), 'dt-no-such-machine'); +const machineless = (env) => ({ DEVTEAM_CGROUP: NOWHERE, DEVTEAM_MEMINFO: NOWHERE, ...(env ?? process.env) }); + export function runNode(script, args, { cwd, env, input, cold } = {}) { + env = machineless(env); if (cold || input !== undefined || process.env.DEVTEAM_TEST_COLD) { const r = spawnSync(process.execPath, [script, ...args], { cwd, env, encoding: 'utf8', input }); return { status: r.status, stdout: r.stdout, stderr: r.stderr }; diff --git a/test/lib/kit.mjs b/test/lib/kit.mjs index 6a0e519..f8f1d6a 100644 --- a/test/lib/kit.mjs +++ b/test/lib/kit.mjs @@ -23,6 +23,10 @@ export const line = (o) => `${JSON.stringify({ v: 1, ...o })}\n`; process.env.DEVTEAM_CSWAP = 'off'; // A fixture's stand-in supervisor runs under whatever agent runs the tests; it isn't that agent. process.env.DEVTEAM_SUPERVISOR_UP = 'off'; +// The kit won't start a session while the machine's memory is over its ceiling: a test reads a +// machine of its own (DEVTEAM_CGROUP, DEVTEAM_MEMINFO), or none, never the one it runs on. +process.env.DEVTEAM_CGROUP ??= path.join(os.tmpdir(), 'dt-no-such-machine'); +process.env.DEVTEAM_MEMINFO ??= path.join(os.tmpdir(), 'dt-no-such-machine'); // tmux in this test talks to its own server, never the developer's (or a live fleet's) sessions. // tmux falls back to the machine's default server when TMUX_TMPDIR names a directory that is gone, // and a kill-server there takes down every live fleet on the machine. So a shim first on PATH runs diff --git a/test/machine.test.mjs b/test/machine.test.mjs index 07b276e..43f5b18 100644 --- a/test/machine.test.mjs +++ b/test/machine.test.mjs @@ -365,6 +365,10 @@ test('status says the machine\'s own CPU and memory caps, not the host\'s', () = fs.writeFileSync(path.join(cg, 'cpu.max'), 'max 100000\n'); fs.writeFileSync(path.join(cg, 'memory.max'), 'max\n'); assert.match(machine(['machine', 'status'], { DEVTEAM_CGROUP: cg }).stdout, /^machine: disk .*; memory [\d.]+ GB free of [\d.]+ GB; cpus \d+[ ;].*load /m); + // the host's own: MemAvailable of MemTotal (what the kit's memory ceiling reads too) + const meminfo = path.join(cg, 'meminfo'); + fs.writeFileSync(meminfo, 'MemTotal: 8388608 kB\nMemFree: 524288 kB\nMemAvailable: 3145728 kB\n'); + assert.match(machine(['machine', 'status'], { DEVTEAM_CGROUP: cg, DEVTEAM_MEMINFO: meminfo }).stdout, /^machine: disk .*; memory 3\.0 GB free of 8\.0 GB; cpus /m); }); // Every read the operator is allowed, against a repo with something in it: the log, git status and diff --git a/test/memory-ceiling.test.mjs b/test/memory-ceiling.test.mjs new file mode 100644 index 0000000..ccaa190 --- /dev/null +++ b/test/memory-ceiling.test.mjs @@ -0,0 +1,113 @@ +// The machine is shared: several supervisors' fleets on one box with one memory cap. A new worker +// or planner doesn't start while the machine's memory is at the supervisor's machine_memory_ceiling +// (--force does anyway, logged); a resume of a session that ran always does; where the kit can't +// read memory (macOS: no cgroup, no /proc/meminfo) nothing is refused. The idle-capacity and +// idle-planning findings don't suggest a start then; memory-full says the memory is full instead. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fleetFixture, json, hasTmux } from './lib/kit.mjs'; + +const G = 1024 ** 3; +const SUP = { DEVTEAM_ROLE: 'supervisor' }; + +test('a new worker waits while the machine\'s memory is over the ceiling; a resume and --force start', { skip: !hasTmux && 'no tmux' }, () => { + const F = fleetFixture('memory-ceiling'); + const { old, ok, run } = F; + // a 32 GiB container: memory.current less the page cache is what's in use + const cg = fs.mkdtempSync(path.join(F.base, 'cgroup-')); + const none = path.join(F.base, 'no-such-file'); + const use = (inUse, max = 32 * G) => { + fs.writeFileSync(path.join(cg, 'memory.max'), `${max}\n`); + fs.writeFileSync(path.join(cg, 'memory.current'), `${Math.round(inUse + G)}\n`); + fs.writeFileSync(path.join(cg, 'memory.stat'), `anon ${Math.round(inUse)}\nfile ${G}\n`); + }; + const M = { ...SUP, DEVTEAM_CGROUP: cg, DEVTEAM_MEMINFO: none }; + const ticket = (id) => ok(old, ['ticket', 'add', id, '--json', JSON.stringify({ title: id, feature: 'chores', component: 'ch', kind: 'chore', size: 'S', risk: 'normal', serves: [], depends_on: [], touches: ['ch/**'], done_when: ['the tidy check passes in CI'] })], SUP); + const starts = () => json(ok(old, ['log', '--type', 'act', '--json'])).filter((a) => a.class === 'worker-slots'); + const findings = (env) => json(ok(old, ['metrics', '--json'], env)).findings; + const settings = (patch) => { + const sf = path.join(old, 'docs/devteam/project/settings.md'); + const text = fs.readFileSync(sf, 'utf8'); + const block = /```json devteam\s*\n([\s\S]*?)\n```/.exec(text); + fs.writeFileSync(sf, text.replace(block[1], JSON.stringify({ ...JSON.parse(block[1]), ...patch }, null, 2))); + }; + try { + for (const id of ['ch-01-a', 'ch-02-b', 'ch-03-c', 'ch-04-d', 'ch-05-e']) ticket(id); + + // 27.2 of 32 GB, 85%: refused in plain words, nothing started, nothing logged + use(27.2 * G); + const r = run(old, ['worker', 'ch-01-a'], M); + assert.notEqual(r.status, 0); + assert.match(r.err, /the machine's memory is at 27\.2 of 32\.0 GB \(85%\), at or over its 80% ceiling: start this worker when some sessions finish, or pass --force/); + assert.equal(starts().length, 0); + assert.ok(!fs.existsSync(path.join(old, '.worktrees/ch-01-a')), 'no worktree made'); + // --resume on a ticket that never ran is a new start, so it is refused too + assert.match(run(old, ['worker', 'ch-01-a', '--resume'], M).err, /over its 80% ceiling/); + // a planner as well + const goal = ok(old, ['goal', 'add', '--title', 'Tidy', '--outcome', 'tidy', '--measure', 'untidy files', '--target', '0', '--unit', 'files', '--direction', 'down', '--source', 'prod', '--check', 'echo 0', '--non-goals', 'none', '--preconditions', 'a'], SUP).trim(); + assert.match(run(old, ['planner', 'beta', '--goal', goal], M).err, /over its 80% ceiling: start this planner when some sessions finish/); + + // the findings: no "make workers for", memory-full in its place + const full = findings(M); + assert.ok(!full.some((f) => ['idle-capacity', 'idle-planning'].includes(f.kind)), JSON.stringify(full.map((f) => f.kind))); + const mf = full.find((f) => f.kind === 'memory-full'); + assert.ok(mf, 'memory-full is shown'); + assert.match(mf.detail, /^the machine's memory is at 27\.2 of 32\.0 GB \(85%\), at or over its 80% ceiling, with 5 tickets ready, 0\/5 workers active/); + assert.match(mf.suggestion, /lower max_active_tickets/); + assert.doesNotMatch(mf.suggestion, /make a worker|make workers/); + + // --force starts it, and its act says what it overrode + ok(old, ['worker', 'ch-01-a', '--force'], M); + const forced = starts().pop(); + assert.match(forced.what, /^made worker ch-01-a in \S+ while the machine's memory is at 27\.2 of 32\.0 GB \(85%\), at or over its 80% ceiling, with --force$/); + // a resume of a session that ran is never refused for memory, and says nothing of it + use(31 * G); + ok(old, ['worker', 'ch-01-a', '--resume'], M); + assert.match(starts().pop().what, /^resumed worker ch-01-a in \S+$/); + // one that ran but whose worktree is gone (a new machine) starts fresh: allowed, and its act says so + ok(old, ['worker', 'ch-01-a', '--remove', '--force'], SUP); + ok(old, ['worker', 'ch-01-a', '--resume'], M); + assert.match(starts().pop().what, /^made worker ch-01-a in \S+, a resume that started fresh while the machine's memory is at 31\.0 of 32\.0 GB \(97%\)/); + // shmem (tmpfs, /dev/shm) is page cache the kernel can't drop: it counts as in use + fs.writeFileSync(path.join(cg, 'memory.current'), `${28 * G}\n`); + fs.writeFileSync(path.join(cg, 'memory.stat'), `anon ${14 * G}\nfile ${8 * G}\nshmem ${6 * G}\n`); + assert.match(run(old, ['worker', 'ch-02-b'], M).err, /at 26\.0 of 32\.0 GB \(81%\), at or over its 80% ceiling/); + + // under the ceiling: starts, and idle-capacity is back + use(20 * G); + ok(old, ['worker', 'ch-02-b'], M); + assert.match(starts().pop().what, /^made worker ch-02-b in \S+$/); + const under = findings(M); + assert.ok(under.some((f) => f.kind === 'idle-capacity'), JSON.stringify(under.map((f) => f.kind))); + assert.ok(!under.some((f) => f.kind === 'memory-full')); + + // the supervisor's own ceiling counts: 85% under a 0.9 ceiling starts + use(27.2 * G); + settings({ machine_memory_ceiling: 0.9 }); + ok(old, ['worker', 'ch-03-c'], M); + assert.equal(ok(old, ['settings', '--get', 'machine_memory_ceiling']), '0.9'); + // validated: out of range or not a number is an error, and counts as the default, never wider + for (const bad of [0.99, 1, 2, 0.4, '0.9', null]) { + settings({ machine_memory_ceiling: bad }); + const v = run(old, ['validate'], SUP); + assert.notEqual(v.status, 0, `${JSON.stringify(bad)} is refused`); + assert.match(`${v.out}\n${v.err}`, /settings\.machine_memory_ceiling is .*a number from 0\.5 to 0\.95/); + } + settings({ machine_memory_ceiling: 1 }); + assert.match(run(old, ['worker', 'ch-04-d'], M).err, /over its 80% ceiling/, 'an invalid ceiling is the default'); + settings({ machine_memory_ceiling: 0.8 }); + { const v = run(old, ['validate'], SUP); assert.doesNotMatch(`${v.out}\n${v.err}`, /machine_memory_ceiling/, 'a valid ceiling passes'); } + + // memory.max "max": the host's /proc/meminfo, MemTotal less MemAvailable + fs.writeFileSync(path.join(cg, 'memory.max'), 'max\n'); + const meminfo = path.join(F.base, 'meminfo'); + fs.writeFileSync(meminfo, `MemTotal: 16777216 kB\nMemFree: 524288 kB\nMemAvailable: 1677722 kB\n`); + assert.match(run(old, ['worker', 'ch-04-d'], { ...M, DEVTEAM_MEMINFO: meminfo }).err, /at 14\.4 of 16\.0 GB \(90%\), at or over its 80% ceiling/); + // no cgroup and no meminfo (macOS): nothing says, nothing is refused or suppressed + const mac = { ...SUP, DEVTEAM_CGROUP: path.join(F.base, 'no-cgroup'), DEVTEAM_MEMINFO: none }; + ok(old, ['worker', 'ch-04-d'], mac); + assert.ok(!findings(mac).some((f) => f.kind === 'memory-full')); + } finally { F.done(); } +});