diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index 49674354b2532..033d83bc5f5c0 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -21,7 +21,7 @@ import * as ProviderProcesses from "./providerProcesses.ts"; import { classifyScope, make, ownedAgentSlice, type ScopeState } from "./agentScope.ts"; const GiB = 1024 ** 3; -const memory = vi.hoisted(() => ({ total: 16 * 1024 ** 3 })); +const memory = vi.hoisted(() => ({ total: 16 * 1024 ** 3, swapKb: 2 * 1024 ** 2 })); vi.mock("node:os", async (importOriginal) => ({ ...(await importOriginal()), totalmem: () => memory.total, @@ -133,7 +133,9 @@ const withFakeSystemd = ( Effect.provideService(FileSystem.FileSystem, { ...fileSystem, readFileString: (path, encoding) => - fileSystem.readFileString(path.replace("/sys/fs/cgroup", cgroupRoot), encoding), + path === "/proc/meminfo" + ? Effect.sync(() => `MemTotal: 16384 kB\nSwapTotal: ${memory.swapKb} kB\n`) + : fileSystem.readFileString(path.replace("/sys/fs/cgroup", cgroupRoot), encoding), writeFileString: (path, data, options) => fileSystem.writeFileString(path.replace("/sys/fs/cgroup", cgroupRoot), data, options), }), @@ -331,8 +333,9 @@ describe("AgentScope service", () => { ), ); - it.effect("follows a live memory change at the next launch", () => { + it.effect("follows a live memory or swap change at the next launch", () => { memory.total = 16 * GiB; + memory.swapKb = 2 * 1024 ** 2; return withFakeSystemd(({ scope, calls, failingVerbs }) => Effect.gen(function* () { const launch = scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" }); @@ -361,6 +364,12 @@ describe("AgentScope service", () => { grown, grown, ]); + + // Without swap, MemoryHigh only throttles; MemoryMax must OOM-kill the runaway. + memory.swapKb = 0; + yield* launch; + expect(memoryLimits().at(-1)).toEqual(["MemoryHigh=infinity", `MemoryMax=${18 * GiB}`]); + memory.swapKb = 2 * 1024 ** 2; }), ); }); diff --git a/apps/server/src/process/agentScope.ts b/apps/server/src/process/agentScope.ts index 1e52e680b6f8d..4bbb7d159cf0b 100644 --- a/apps/server/src/process/agentScope.ts +++ b/apps/server/src/process/agentScope.ts @@ -36,14 +36,13 @@ export function ownedAgentSlice(stateDir: string): string { const GiB = 1024 ** 3; /** - * Agents together leave about 6 GB free before they are throttled and 4 GB - * before the kernel kills one, so the server, sshd, and tailscaled keep - * working. Small machines still give agents at least half of RAM before - * throttling and three quarters before a kill. + * Agents leave about 6 GB free before throttling and 4 GB before an OOM kill (at + * least half and three quarters of RAM go to agents) so the server and sshd keep + * working. Without swap, MemoryHigh stalls every agent and never reaches the kill. */ -function agentSliceMemoryLimits(totalBytes: number) { +function agentSliceMemoryLimits(totalBytes: number, hasSwap: boolean) { return { - high: Math.max(totalBytes - 6 * GiB, Math.floor(totalBytes / 2)), + high: hasSwap ? Math.max(totalBytes - 6 * GiB, Math.floor(totalBytes / 2)) : "infinity", max: Math.max(totalBytes - 4 * GiB, Math.floor(totalBytes * 0.75)), }; } @@ -204,13 +203,20 @@ export const make = Effect.gen(function* () { }); // VMs and containers can resize RAM live, so each launch re-checks the total // the limits came from. A failed update is retried by the next launch. + // An unreadable meminfo keeps the throttle, as before swap was checked. + const readHasSwap = fileSystem.readFileString("/proc/meminfo").pipe( + Effect.map((text) => /^SwapTotal:\s*0 kB$/m.exec(text) === null), + Effect.orElseSucceed(() => true), + ); let appliedTotal: number | undefined; + let appliedHasSwap: boolean | undefined; const memoryLimitsLock = yield* Semaphore.make(1); const applyMemoryLimits = memoryLimitsLock.withPermits(1)( Effect.gen(function* () { const total = NodeOS.totalmem(); - if (total === appliedTotal) return; - const limits = agentSliceMemoryLimits(total); + const hasSwap = yield* readHasSwap; + if (total === appliedTotal && hasSwap === appliedHasSwap) return; + const limits = agentSliceMemoryLimits(total, hasSwap); const applied = yield* setProperties(AGENT_SLICE, [ `MemoryHigh=${limits.high}`, `MemoryMax=${limits.max}`, @@ -225,13 +231,14 @@ export const make = Effect.gen(function* () { }); } appliedTotal = total; + appliedHasSwap = hasSwap; }), ); yield* applyMemoryLimits; yield* setProperties(AGENTS_PARENT_SLICE, AGENTS_PARENT_PROPERTIES); yield* Effect.logInfo("Agent scopes are on", { slice: AGENT_SLICE, - ...agentSliceMemoryLimits(NodeOS.totalmem()), + ...agentSliceMemoryLimits(NodeOS.totalmem(), yield* readHasSwap), oomPolicy: supportsOomPolicy, }); return {