From f99c63ecdbc97c395e90b0bcc9d4dae346c04e14 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 19 Nov 2026 19:04:21 +0000 Subject: [PATCH] fix(server): agents on hosts without swap are OOM-killed instead of throttled for hours Without swap, reclaim at the agents slice's MemoryHigh can only drop page cache, so every agent stalls indefinitely and MemoryMax never kills the runaway. Set MemoryHigh=infinity when SwapTotal is 0 and keep MemoryMax. Fixes pingdotgg/t3code#39195 Co-Authored-By: Claude Opus 5.5 --- apps/server/src/process/agentScope.test.ts | 15 ++++++++++--- apps/server/src/process/agentScope.ts | 25 ++++++++++++++-------- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index 49674354b2532..033d83bc5f5c0 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -21,7 +21,7 @@ import * as ProviderProcesses from "./providerProcesses.ts"; import { classifyScope, make, ownedAgentSlice, type ScopeState } from "./agentScope.ts"; const GiB = 1024 ** 3; -const memory = vi.hoisted(() => ({ total: 16 * 1024 ** 3 })); +const memory = vi.hoisted(() => ({ total: 16 * 1024 ** 3, swapKb: 2 * 1024 ** 2 })); vi.mock("node:os", async (importOriginal) => ({ ...(await importOriginal()), totalmem: () => memory.total, @@ -133,7 +133,9 @@ const withFakeSystemd = ( Effect.provideService(FileSystem.FileSystem, { ...fileSystem, readFileString: (path, encoding) => - fileSystem.readFileString(path.replace("/sys/fs/cgroup", cgroupRoot), encoding), + path === "/proc/meminfo" + ? Effect.sync(() => `MemTotal: 16384 kB\nSwapTotal: ${memory.swapKb} kB\n`) + : fileSystem.readFileString(path.replace("/sys/fs/cgroup", cgroupRoot), encoding), writeFileString: (path, data, options) => fileSystem.writeFileString(path.replace("/sys/fs/cgroup", cgroupRoot), data, options), }), @@ -331,8 +333,9 @@ describe("AgentScope service", () => { ), ); - it.effect("follows a live memory change at the next launch", () => { + it.effect("follows a live memory or swap change at the next launch", () => { memory.total = 16 * GiB; + memory.swapKb = 2 * 1024 ** 2; return withFakeSystemd(({ scope, calls, failingVerbs }) => Effect.gen(function* () { const launch = scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" }); @@ -361,6 +364,12 @@ describe("AgentScope service", () => { grown, grown, ]); + + // Without swap, MemoryHigh only throttles; MemoryMax must OOM-kill the runaway. + memory.swapKb = 0; + yield* launch; + expect(memoryLimits().at(-1)).toEqual(["MemoryHigh=infinity", `MemoryMax=${18 * GiB}`]); + memory.swapKb = 2 * 1024 ** 2; }), ); }); diff --git a/apps/server/src/process/agentScope.ts b/apps/server/src/process/agentScope.ts index 1e52e680b6f8d..4bbb7d159cf0b 100644 --- a/apps/server/src/process/agentScope.ts +++ b/apps/server/src/process/agentScope.ts @@ -36,14 +36,13 @@ export function ownedAgentSlice(stateDir: string): string { const GiB = 1024 ** 3; /** - * Agents together leave about 6 GB free before they are throttled and 4 GB - * before the kernel kills one, so the server, sshd, and tailscaled keep - * working. Small machines still give agents at least half of RAM before - * throttling and three quarters before a kill. + * Agents leave about 6 GB free before throttling and 4 GB before an OOM kill (at + * least half and three quarters of RAM go to agents) so the server and sshd keep + * working. Without swap, MemoryHigh stalls every agent and never reaches the kill. */ -function agentSliceMemoryLimits(totalBytes: number) { +function agentSliceMemoryLimits(totalBytes: number, hasSwap: boolean) { return { - high: Math.max(totalBytes - 6 * GiB, Math.floor(totalBytes / 2)), + high: hasSwap ? Math.max(totalBytes - 6 * GiB, Math.floor(totalBytes / 2)) : "infinity", max: Math.max(totalBytes - 4 * GiB, Math.floor(totalBytes * 0.75)), }; } @@ -204,13 +203,20 @@ export const make = Effect.gen(function* () { }); // VMs and containers can resize RAM live, so each launch re-checks the total // the limits came from. A failed update is retried by the next launch. + // An unreadable meminfo keeps the throttle, as before swap was checked. + const readHasSwap = fileSystem.readFileString("/proc/meminfo").pipe( + Effect.map((text) => /^SwapTotal:\s*0 kB$/m.exec(text) === null), + Effect.orElseSucceed(() => true), + ); let appliedTotal: number | undefined; + let appliedHasSwap: boolean | undefined; const memoryLimitsLock = yield* Semaphore.make(1); const applyMemoryLimits = memoryLimitsLock.withPermits(1)( Effect.gen(function* () { const total = NodeOS.totalmem(); - if (total === appliedTotal) return; - const limits = agentSliceMemoryLimits(total); + const hasSwap = yield* readHasSwap; + if (total === appliedTotal && hasSwap === appliedHasSwap) return; + const limits = agentSliceMemoryLimits(total, hasSwap); const applied = yield* setProperties(AGENT_SLICE, [ `MemoryHigh=${limits.high}`, `MemoryMax=${limits.max}`, @@ -225,13 +231,14 @@ export const make = Effect.gen(function* () { }); } appliedTotal = total; + appliedHasSwap = hasSwap; }), ); yield* applyMemoryLimits; yield* setProperties(AGENTS_PARENT_SLICE, AGENTS_PARENT_PROPERTIES); yield* Effect.logInfo("Agent scopes are on", { slice: AGENT_SLICE, - ...agentSliceMemoryLimits(NodeOS.totalmem()), + ...agentSliceMemoryLimits(NodeOS.totalmem(), yield* readHasSwap), oomPolicy: supportsOomPolicy, }); return {