Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 22 additions & 4 deletions apps/server/src/process/agentScope.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,11 @@ import * as ProviderProcesses from "./providerProcesses.ts";
import { classifyScope, make, ownedAgentSlice, type ScopeState } from "./agentScope.ts";

const GiB = 1024 ** 3;
const memory = vi.hoisted(() => ({ total: 16 * 1024 ** 3, swapKb: 2 * 1024 ** 2 }));
const memory = vi.hoisted(() => ({
total: 16 * 1024 ** 3,
swapKb: 2 * 1024 ** 2,
freeSwapKb: 2 * 1024 ** 2,
}));
vi.mock("node:os", async (importOriginal) => ({
...(await importOriginal<typeof import("node:os")>()),
totalmem: () => memory.total,
Expand Down Expand Up @@ -134,7 +138,10 @@ const withFakeSystemd = <A, E>(
...fileSystem,
readFileString: (path, encoding) =>
path === "/proc/meminfo"
? Effect.sync(() => `MemTotal: 16384 kB\nSwapTotal: ${memory.swapKb} kB\n`)
? Effect.sync(
() =>
`MemTotal: 16384 kB\nSwapTotal: ${memory.swapKb} kB\nSwapFree: ${memory.freeSwapKb} kB\n`,
)
: fileSystem.readFileString(path.replace("/sys/fs/cgroup", cgroupRoot), encoding),
writeFileString: (path, data, options) =>
fileSystem.writeFileString(path.replace("/sys/fs/cgroup", cgroupRoot), data, options),
Expand Down Expand Up @@ -336,6 +343,7 @@ describe("AgentScope service", () => {
it.effect("follows a live memory or swap change at the next launch", () => {
memory.total = 16 * GiB;
memory.swapKb = 2 * 1024 ** 2;
memory.freeSwapKb = memory.swapKb;
return withFakeSystemd(({ scope, calls, failingVerbs }) =>
Effect.gen(function* () {
const launch = scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
Expand Down Expand Up @@ -365,11 +373,21 @@ describe("AgentScope service", () => {
grown,
]);

// Without swap, MemoryHigh only throttles; MemoryMax must OOM-kill the runaway.
const unthrottled = ["MemoryHigh=infinity", `MemoryMax=${18 * GiB}`];
memory.freeSwapKb = 0;
yield* launch;
expect(memoryLimits().at(-1)).toEqual(unthrottled);

memory.freeSwapKb = 1;
yield* launch;
expect(memoryLimits().at(-1)).toEqual(grown);

memory.swapKb = 0;
memory.freeSwapKb = 0;
yield* launch;
expect(memoryLimits().at(-1)).toEqual(["MemoryHigh=infinity", `MemoryMax=${18 * GiB}`]);
expect(memoryLimits().at(-1)).toEqual(unthrottled);
memory.swapKb = 2 * 1024 ** 2;
memory.freeSwapKb = memory.swapKb;
}),
);
});
Expand Down
4 changes: 2 additions & 2 deletions apps/server/src/process/agentScope.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ const GiB = 1024 ** 3;
/**
* Agents leave about 6 GB free before throttling and 4 GB before an OOM kill (at
* least half and three quarters of RAM go to agents) so the server and sshd keep
* working. Without swap, MemoryHigh stalls every agent and never reaches the kill.
* working. Without available swap, MemoryHigh can stall agents below the hard limit.
*/
function agentSliceMemoryLimits(totalBytes: number, hasSwap: boolean) {
return {
Expand Down Expand Up @@ -205,7 +205,7 @@ export const make = Effect.gen(function* () {
// the limits came from. A failed update is retried by the next launch.
// An unreadable meminfo keeps the throttle, as before swap was checked.
const readHasSwap = fileSystem.readFileString("/proc/meminfo").pipe(
Effect.map((text) => /^SwapTotal:\s*0 kB$/m.exec(text) === null),
Effect.map((text) => /^Swap(?:Total|Free):\s*0 kB$/m.exec(text) === null),
Effect.orElseSucceed(() => true),
);
let appliedTotal: number | undefined;
Expand Down
Loading