diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index 033d83bc5f5c..f7b448523e34 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -21,7 +21,11 @@ import * as ProviderProcesses from "./providerProcesses.ts"; import { classifyScope, make, ownedAgentSlice, type ScopeState } from "./agentScope.ts"; const GiB = 1024 ** 3; -const memory = vi.hoisted(() => ({ total: 16 * 1024 ** 3, swapKb: 2 * 1024 ** 2 })); +const memory = vi.hoisted(() => ({ + total: 16 * 1024 ** 3, + swapKb: 2 * 1024 ** 2, + freeSwapKb: 2 * 1024 ** 2, +})); vi.mock("node:os", async (importOriginal) => ({ ...(await importOriginal()), totalmem: () => memory.total, @@ -134,7 +138,10 @@ const withFakeSystemd = ( ...fileSystem, readFileString: (path, encoding) => path === "/proc/meminfo" - ? Effect.sync(() => `MemTotal: 16384 kB\nSwapTotal: ${memory.swapKb} kB\n`) + ? Effect.sync( + () => + `MemTotal: 16384 kB\nSwapTotal: ${memory.swapKb} kB\nSwapFree: ${memory.freeSwapKb} kB\n`, + ) : fileSystem.readFileString(path.replace("/sys/fs/cgroup", cgroupRoot), encoding), writeFileString: (path, data, options) => fileSystem.writeFileString(path.replace("/sys/fs/cgroup", cgroupRoot), data, options), @@ -336,6 +343,7 @@ describe("AgentScope service", () => { it.effect("follows a live memory or swap change at the next launch", () => { memory.total = 16 * GiB; memory.swapKb = 2 * 1024 ** 2; + memory.freeSwapKb = memory.swapKb; return withFakeSystemd(({ scope, calls, failingVerbs }) => Effect.gen(function* () { const launch = scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" }); @@ -365,11 +373,21 @@ describe("AgentScope service", () => { grown, ]); - // Without swap, MemoryHigh only throttles; MemoryMax must OOM-kill the runaway. + const unthrottled = ["MemoryHigh=infinity", `MemoryMax=${18 * GiB}`]; + memory.freeSwapKb = 0; + yield* launch; + expect(memoryLimits().at(-1)).toEqual(unthrottled); + + memory.freeSwapKb = 1; + yield* launch; + expect(memoryLimits().at(-1)).toEqual(grown); + memory.swapKb = 0; + memory.freeSwapKb = 0; yield* launch; - expect(memoryLimits().at(-1)).toEqual(["MemoryHigh=infinity", `MemoryMax=${18 * GiB}`]); + expect(memoryLimits().at(-1)).toEqual(unthrottled); memory.swapKb = 2 * 1024 ** 2; + memory.freeSwapKb = memory.swapKb; }), ); }); diff --git a/apps/server/src/process/agentScope.ts b/apps/server/src/process/agentScope.ts index 4bbb7d159cf0..8893389c8ee2 100644 --- a/apps/server/src/process/agentScope.ts +++ b/apps/server/src/process/agentScope.ts @@ -38,7 +38,7 @@ const GiB = 1024 ** 3; /** * Agents leave about 6 GB free before throttling and 4 GB before an OOM kill (at * least half and three quarters of RAM go to agents) so the server and sshd keep - * working. Without swap, MemoryHigh stalls every agent and never reaches the kill. + * working. Without available swap, MemoryHigh can stall agents below the hard limit. */ function agentSliceMemoryLimits(totalBytes: number, hasSwap: boolean) { return { @@ -205,7 +205,7 @@ export const make = Effect.gen(function* () { // the limits came from. A failed update is retried by the next launch. // An unreadable meminfo keeps the throttle, as before swap was checked. const readHasSwap = fileSystem.readFileString("/proc/meminfo").pipe( - Effect.map((text) => /^SwapTotal:\s*0 kB$/m.exec(text) === null), + Effect.map((text) => /^Swap(?:Total|Free):\s*0 kB$/m.exec(text) === null), Effect.orElseSucceed(() => true), ); let appliedTotal: number | undefined;