Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 11 additions & 8 deletions apps/server/src/diagnostics/ProcessDiagnostics.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,9 @@ describe("ProcessDiagnostics", () => {
Effect.sync(() => {
const rows = ProcessDiagnostics.parsePosixProcessRows(
[
" 10 1 10 Ss 0.0 1024 01:02.03 /usr/bin/node server.js",
" 11 10 10 S+ 12.5 20480 00:04 codex app-server --config /tmp/one two",
// macOS prints CPU time as minutes and seconds, Linux as [days-]hours:minutes:seconds.
" 10 1 10 Ss 0.0 1024 01:02.03 398:51.57 /usr/bin/node server.js",
" 11 10 10 S+ 12.5 20480 00:04 1-02:03:04 codex app-server --config /tmp/one two",
].join("\n"),
);

Expand All @@ -61,6 +62,7 @@ describe("ProcessDiagnostics", () => {
rssBytes: 1024 * 1024,
elapsed: "01:02.03",
command: "/usr/bin/node server.js",
cpuTimeMs: (398 * 60 + 51.57) * 1000,
},
{
pid: 11,
Expand All @@ -71,6 +73,7 @@ describe("ProcessDiagnostics", () => {
rssBytes: 20480 * 1024,
elapsed: "00:04",
command: "codex app-server --config /tmp/one two",
cpuTimeMs: (((24 + 2) * 60 + 3) * 60 + 4) * 1000,
},
]);
}),
Expand Down Expand Up @@ -130,7 +133,7 @@ describe("ProcessDiagnostics", () => {
cpuPercent: 9,
rssBytes: 9_000,
elapsed: "00:00",
command: "ps -axo pid=,ppid=,pgid=,stat=,pcpu=,rss=,etime=,command=",
command: "ps -axo pid=,ppid=,pgid=,stat=,pcpu=,rss=,etime=,time=,command=",
},
],
});
Expand Down Expand Up @@ -222,8 +225,8 @@ describe("ProcessDiagnostics", () => {
return Effect.succeed(
mockHandle({
stdout: [
` ${process.pid} 1 ${process.pid} Ss 0.0 1024 01:02.03 t3 server`,
` 4242 ${process.pid} ${process.pid} S 1.5 2048 00:04 agent`,
` ${process.pid} 1 ${process.pid} Ss 0.0 1024 01:02.03 0:01.50 t3 server`,
` 4242 ${process.pid} ${process.pid} S 1.5 2048 00:04 0:00.06 agent`,
].join("\n"),
}),
);
Expand Down Expand Up @@ -254,7 +257,7 @@ describe("ProcessDiagnostics", () => {
expect(commands).toEqual([
{
command: "ps",
args: ["-axo", "pid=,ppid=,pgid=,stat=,pcpu=,rss=,etime=,command="],
args: ["-axo", "pid=,ppid=,pgid=,stat=,pcpu=,rss=,etime=,time=,command="],
},
]);
}
Expand Down Expand Up @@ -837,8 +840,8 @@ describe("ProcessDiagnostics", () => {
Effect.succeed(
mockHandle({
stdout: [
` ${process.pid} 1 ${process.pid} Ss 0.0 1024 01:02.03 t3 server`,
` 4242 ${process.pid} ${process.pid} R 1.5 2048 00:00 ps -axo pid=,ppid=,pgid=,stat=,pcpu=,rss=,etime=,command=`,
` ${process.pid} 1 ${process.pid} Ss 0.0 1024 01:02.03 0:01.50 t3 server`,
` 4242 ${process.pid} ${process.pid} R 1.5 2048 00:00 0:00.01 ps -axo pid=,ppid=,pgid=,stat=,pcpu=,rss=,etime=,time=,command=`,
].join("\n"),
}),
),
Expand Down
34 changes: 30 additions & 4 deletions apps/server/src/diagnostics/ProcessDiagnostics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,21 @@ export interface ProcessRow {
readonly rssBytes: number;
readonly elapsed: string;
readonly command: string;
/**
* All the CPU time the process has used since it started (POSIX `ps time`).
* Two reads of it give what the process used in between, which `cpuPercent`
* can't: on Linux that is an average over the process's whole life.
* Absent on Windows, where `cpuPercent` already comes from such a pair.
*/
readonly cpuTimeMs?: number | undefined;
}

const PROCESS_QUERY_TIMEOUT_MS = 15_000;
const POSIX_PROCESS_QUERY_COMMAND = "pid=,ppid=,pgid=,stat=,pcpu=,rss=,etime=,command=";
const POSIX_PROCESS_QUERY_COMMAND = "pid=,ppid=,pgid=,stat=,pcpu=,rss=,etime=,time=,command=";
/** Our own `ps` as it shows up in the process list it prints. */
const POSIX_PROCESS_QUERY_PATTERN = new RegExp(
`(?:^|[/\\\\])ps\\s+-axo\\s+${POSIX_PROCESS_QUERY_COMMAND}`,
);
const PROCESS_QUERY_MAX_OUTPUT_BYTES = 2 * 1024 * 1024;
const PROCESS_SNAPSHOT_TTL_MS = 2_000;
const PROCESS_SNAPSHOT_INITIAL_FAILURE_BACKOFF_MS = 2_000;
Expand Down Expand Up @@ -106,10 +117,22 @@ function parseNumber(value: string): number | null {
return Number.isFinite(parsed) ? parsed : null;
}

/**
* `ps time` in milliseconds: `398:51.57` (macOS, minutes and seconds) or
* `[1-]02:03:04` (Linux, whole seconds). Null when it reads as neither.
*/
export function parsePosixCpuTime(value: string): number | null {
const match = /^(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+(?:\.\d+)?)$/.exec(value);
if (!match) return null;
const [, days = "0", hours = "0", minutes = "0", seconds = "0"] = match;
const total = ((Number(days) * 24 + Number(hours)) * 60 + Number(minutes)) * 60 + Number(seconds);
return Number.isFinite(total) ? Math.round(total * 1000) : null;
}

export function parsePosixProcessRows(output: string): ReadonlyArray<ProcessRow> {
const rows: ProcessRow[] = [];
const rowPattern =
/^\s*(\d+)\s+(\d+)\s+(-?\d+)\s+(\S+)\s+([+-]?(?:\d+\.?\d*|\.\d+))\s+(\d+)\s+(\S+)\s+(.+)$/;
/^\s*(\d+)\s+(\d+)\s+(-?\d+)\s+(\S+)\s+([+-]?(?:\d+\.?\d*|\.\d+))\s+(\d+)\s+(\S+)\s+(\S+)\s+(.+)$/;

for (const line of output.split(/\r?\n/)) {
if (line.trim().length === 0) continue;
Expand All @@ -124,7 +147,8 @@ export function parsePosixProcessRows(output: string): ReadonlyArray<ProcessRow>
const cpuText = match[5];
const rssText = match[6];
const elapsed = match[7];
const command = match[8];
const cpuTimeText = match[8];
const command = match[9];
if (
pidText === undefined ||
ppidText === undefined ||
Expand All @@ -133,6 +157,7 @@ export function parsePosixProcessRows(output: string): ReadonlyArray<ProcessRow>
cpuText === undefined ||
rssText === undefined ||
elapsed === undefined ||
cpuTimeText === undefined ||
command === undefined
) {
continue;
Expand Down Expand Up @@ -165,6 +190,7 @@ export function parsePosixProcessRows(output: string): ReadonlyArray<ProcessRow>
rssBytes: rssKiB * 1024,
elapsed,
command,
cpuTimeMs: parsePosixCpuTime(cpuTimeText) ?? undefined,
});
}

Expand Down Expand Up @@ -426,7 +452,7 @@ export function isDiagnosticsQueryProcess(row: ProcessRow, serverPid: number): b

const command = row.command.trim();
return (
/(?:^|[/\\])ps\s+-axo\s+pid=,ppid=,pgid=,stat=,pcpu=,rss=,etime=,command=/.test(command) ||
POSIX_PROCESS_QUERY_PATTERN.test(command) ||
(/\bpowershell(?:\.exe)?\b/i.test(command) &&
/\bGet-CimInstance\s+Win32_Process\b/i.test(command))
);
Expand Down
49 changes: 48 additions & 1 deletion apps/server/src/diagnostics/ProcessResourceMonitor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,32 @@ interface MonitorState {
readonly samplerRunning: boolean;
}

/** One look at the process table, as the sampler took it. */
export interface ProcessTableRead {
readonly rows: ReadonlyArray<ProcessRow>;
readonly serverPid: number;
readonly sampledAtMs: number;
}

export interface ProcessResourceMonitorShape {
readonly readCurrent: Effect.Effect<ServerProcessDiagnosticsResult>;
readonly readHistory: (
input: ServerProcessResourceHistoryInput,
) => Effect.Effect<ServerProcessResourceHistoryResult>;
/**
* Keeps the sampler running for its active window, as `readHistory` does,
* and takes a read at once when the latest is older than one interval.
* Fails only when no read has ever succeeded.
*/
readonly requestSamples: Effect.Effect<void, string>;
/**
* Hands every read the sampler takes to `observer`, until the caller's
* scope closes. Runs only while something keeps the sampler going, so an
* observer adds no reads of its own.
*/
readonly observeReads: (
observer: (read: ProcessTableRead) => Effect.Effect<void>,
) => Effect.Effect<void, never, Scope.Scope>;
}

export class ProcessResourceMonitor extends Context.Service<
Expand Down Expand Up @@ -314,10 +335,17 @@ export const make = Effect.fn("makeProcessResourceMonitor")(function* () {
samplerRunning: false,
});

const observers = new Set<(read: ProcessTableRead) => Effect.Effect<void>>();

const sampleOnce = Effect.gen(function* () {
const sampledAt = yield* DateTime.now;
const sampledAtMs = DateTime.toEpochMillis(sampledAt);
const rows = yield* processSnapshot.readTree.pipe(Effect.withTracerEnabled(false));
for (const observer of observers) {
yield* Effect.suspend(() => observer({ rows, serverPid: process.pid, sampledAtMs })).pipe(
Effect.catchCause(() => Effect.void),
);
}
const samples = collectMonitoredSamples({
rows,
serverPid: process.pid,
Expand Down Expand Up @@ -420,7 +448,26 @@ export const make = Effect.fn("makeProcessResourceMonitor")(function* () {
});
});

return ProcessResourceMonitor.of({ readCurrent, readHistory });
const requestSamples: ProcessResourceMonitorShape["requestSamples"] = Effect.gen(function* () {
yield* ensureSamplerRunning;
const nowMs = DateTime.toEpochMillis(yield* DateTime.now);
const latest = (yield* Ref.get(state)).latestSampledAt;
if (latest === null || nowMs - DateTime.toEpochMillis(latest) >= SAMPLE_INTERVAL_MS) {
yield* sampleOnce;
}
const current = yield* Ref.get(state);
if (current.latestRows === null) {
return yield* Effect.fail(current.lastError ?? "The process list couldn't be read.");
}
});

const observeReads: ProcessResourceMonitorShape["observeReads"] = (observer) =>
Effect.acquireRelease(
Effect.sync(() => observers.add(observer)),
() => Effect.sync(() => observers.delete(observer)),
).pipe(Effect.asVoid);

return ProcessResourceMonitor.of({ readCurrent, readHistory, requestSamples, observeReads });
});

export const layer = Layer.effect(ProcessResourceMonitor, make());
Loading
Loading