From 3fc2b678c1f248bb83dd33aaf3f6baad62a7a323 Mon Sep 17 00:00:00 2001 From: Badcuban <108198679+badcuban@users.noreply.github.com> Date: Fri, 20 Nov 2026 23:53:49 -0500 Subject: [PATCH 1/4] perf(server): storage cleanup no longer freezes the server The cleanup job deleted old rows in 5,000-row transactions that ran back to back on the main thread, so the server could not answer for the whole prune (3 to 20 seconds on a 9.5 GB database). It now deletes 25 rows per transaction and yields a full event-loop turn after each one, batches the diff-blob delete too, and drops the final TRUNCATE checkpoint, which small batches make unnecessary. --- .../Layers/StorageMaintenance.test.ts | 43 +++++++++++ .../persistence/Layers/StorageMaintenance.ts | 71 +++++++++++++------ 2 files changed, 93 insertions(+), 21 deletions(-) diff --git a/apps/server/src/persistence/Layers/StorageMaintenance.test.ts b/apps/server/src/persistence/Layers/StorageMaintenance.test.ts index 16ab40844..3f5c56287 100644 --- a/apps/server/src/persistence/Layers/StorageMaintenance.test.ts +++ b/apps/server/src/persistence/Layers/StorageMaintenance.test.ts @@ -209,6 +209,49 @@ layer("StorageMaintenance", (it) => { }), ); + it.effect("lets the event loop turn between delete batches", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storageMaintenance = yield* StorageMaintenance; + + // Enough excess activity for several batches. + const base = 80_000; + const excess = 200; + for (let index = 0; index < MAX_THREAD_ACTIVITIES + excess; index += 1) { + yield* insertEvent(sql, { + sequence: base + index, + streamId: "thread-long", + eventType: "thread.activity-appended", + }); + } + yield* setProjectorCheckpoint(sql, base + MAX_THREAD_ACTIVITIES + excess); + + // Counts turns of the event loop while the prune runs: a callback that + // queues itself again runs once per turn, and only when nothing holds + // the loop. + let running = true; + let turns = 0; + const countTurn = () => { + if (!running) return; + turns += 1; + setImmediate(countTurn); + }; + setImmediate(countTurn); + + const report = yield* storageMaintenance.runOnce.pipe( + Effect.ensuring( + Effect.sync(() => { + running = false; + }), + ), + ); + + assert.equal(report.deletedActivityEvents, excess); + // All in one hold, the loop would not turn at all. + assert.isAtLeast(turns, 4); + }), + ); + it.effect("caps activity pruning at the projector checkpoint", () => Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; diff --git a/apps/server/src/persistence/Layers/StorageMaintenance.ts b/apps/server/src/persistence/Layers/StorageMaintenance.ts index d8df31cd0..60a8f992d 100644 --- a/apps/server/src/persistence/Layers/StorageMaintenance.ts +++ b/apps/server/src/persistence/Layers/StorageMaintenance.ts @@ -24,12 +24,18 @@ import { const RECEIPT_TTL = Duration.days(7); /** - * Rows deleted per statement. Each batch runs in its own short transaction - * so the synchronous driver never blocks the event loop for long, even when - * the first run on a long-lived database prunes hundreds of thousands of - * rows. + * Rows deleted per transaction. The SQLite driver is synchronous, so each + * batch holds the event loop from its DELETE through its COMMIT (and the WAL + * checkpoint a COMMIT can trigger), and `deleteAllBatches` gives the loop a + * turn after every batch. Old events are scattered across a big database, so + * each row costs a few random page reads: on a 9.5 GB one held in the OS + * cache, a 25-row batch held the loop under 55 ms, where 5,000-row batches + * ran back to back and froze the server for the whole prune (3 to 20 + * seconds). A cold cache still makes single batches slower. */ -const DELETE_BATCH_SIZE = 5_000; +const DELETE_BATCH_SIZE = 25; +/** Diff blobs can each run to megabytes, so fewer go per batch. */ +const DIFF_BLOB_DELETE_BATCH_SIZE = 20; const INITIAL_DELAY = Duration.minutes(2); const RUN_INTERVAL = Duration.hours(6); @@ -55,13 +61,25 @@ const makeStorageMaintenance = Effect.gen(function* () { ), ); - const deleteAllBatches = (makeStatement: () => Effect.Effect) => + /** + * Runs `makeStatement` (a DELETE of at most `batchSize` rows) until a batch + * comes up short. After every batch the fiber yields, which on Effect's + * default scheduler is a `setImmediate`: a full turn of the event loop, so + * timers, sockets and other fibers' queries run between batches. Without + * it Effect yields only every couple of thousand operations, and the + * batches ran back to back. + */ + const deleteAllBatches = ( + batchSize: number, + makeStatement: (limit: number) => Effect.Effect, + ) => Effect.gen(function* () { let total = 0; for (;;) { - const deleted = yield* deleteBatch(makeStatement()); + const deleted = yield* deleteBatch(makeStatement(batchSize)); total += deleted; - if (deleted < DELETE_BATCH_SIZE) { + yield* Effect.yieldNow; + if (deleted < batchSize) { return total; } } @@ -73,13 +91,14 @@ const makeStorageMaintenance = Effect.gen(function* () { DateTime.makeUnsafe(nowMillis - Duration.toMillis(RECEIPT_TTL)), ); return yield* deleteAllBatches( - () => sql` + DELETE_BATCH_SIZE, + (limit) => sql` DELETE FROM orchestration_command_receipts WHERE command_id IN ( SELECT command_id FROM orchestration_command_receipts WHERE accepted_at < ${cutoffIso} - LIMIT ${DELETE_BATCH_SIZE} + LIMIT ${limit} ) `, ); @@ -113,7 +132,8 @@ const makeStorageMaintenance = Effect.gen(function* () { for (const thread of deletedThreads) { deletedEvents += yield* deleteAllBatches( - () => sql` + DELETE_BATCH_SIZE, + (limit) => sql` DELETE FROM orchestration_events WHERE sequence IN ( SELECT sequence @@ -121,12 +141,21 @@ const makeStorageMaintenance = Effect.gen(function* () { WHERE aggregate_kind = 'thread' AND stream_id = ${thread.threadId} AND sequence < ${thread.deletedSequence} - LIMIT ${DELETE_BATCH_SIZE} + LIMIT ${limit} ) `, ); - deletedDiffBlobs += yield* deleteBatch( - sql`DELETE FROM checkpoint_diff_blobs WHERE thread_id = ${thread.threadId}`, + deletedDiffBlobs += yield* deleteAllBatches( + DIFF_BLOB_DELETE_BATCH_SIZE, + (limit) => sql` + DELETE FROM checkpoint_diff_blobs + WHERE rowid IN ( + SELECT rowid + FROM checkpoint_diff_blobs + WHERE thread_id = ${thread.threadId} + LIMIT ${limit} + ) + `, ); } @@ -169,7 +198,8 @@ const makeStorageMaintenance = Effect.gen(function* () { } deletedEvents += yield* deleteAllBatches( - () => sql` + DELETE_BATCH_SIZE, + (limit) => sql` DELETE FROM orchestration_events WHERE sequence IN ( SELECT sequence @@ -179,7 +209,7 @@ const makeStorageMaintenance = Effect.gen(function* () { AND event_type = 'thread.activity-appended' AND sequence < ${cutoffSequence} AND sequence <= ${minAppliedSequence} - LIMIT ${DELETE_BATCH_SIZE} + LIMIT ${limit} ) `, ); @@ -222,13 +252,12 @@ const makeStorageMaintenance = Effect.gen(function* () { deletedActivityEvents = yield* pruneActivityEventsBeyondCap(minAppliedSequence); } + // No explicit WAL checkpoint: batches this small keep the WAL near + // SQLite's 1,000-page autocheckpoint, so a prune of any size leaves it + // about as large as everyday writes do. A TRUNCATE checkpoint here was + // one more synchronous call, and one that waits out busy readers. const anythingDeleted = deletedReceipts + deletedThreadEvents + deletedActivityEvents + deletedDiffBlobs > 0; - if (anythingDeleted) { - // Fold the (potentially large) WAL back into the main file so a big - // prune does not leave a WAL the same size as the pruned data. - yield* sql`PRAGMA wal_checkpoint(TRUNCATE);`.pipe(Effect.ignore); - } const pageStats = yield* readPageStats; const report: StorageMaintenanceReport = { From b9a6ad5dddfec4ec36ff509a149fa3bf19532e6b Mon Sep 17 00:00:00 2001 From: Badcuban <108198679+badcuban@users.noreply.github.com> Date: Fri, 20 Nov 2026 23:53:49 -0500 Subject: [PATCH 2/4] feat(server): freeze log catches 2-second freezes and names busy programs The stall monitor now logs freezes from 2 seconds (clients still hear of them from 5). When a freeze's cause is overloaded or unknown, the log line also lists the five busiest programs by CPU, read with ps after the freeze: program names only, never arguments or paths. --- .../src/observability/StallMonitor.test.ts | 36 ++++++ apps/server/src/observability/StallMonitor.ts | 107 +++++++++++++++--- 2 files changed, 125 insertions(+), 18 deletions(-) diff --git a/apps/server/src/observability/StallMonitor.test.ts b/apps/server/src/observability/StallMonitor.test.ts index f225ca61c..f88e714de 100644 --- a/apps/server/src/observability/StallMonitor.test.ts +++ b/apps/server/src/observability/StallMonitor.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "@effect/vitest"; import { classifyStall, type MachineLoad, + parseTopProcesses, reportableStall, STALL_REPORT_WINDOW_MS, type StallSample, @@ -80,6 +81,11 @@ describe("classifyStall", () => { durationMs: 9_000, load: { average1m: 219.4, cpuCount: 18 }, }); + // From two seconds on, a freeze is caught (for the log; clients hear of it from five). + expect(classifyStall(START, after(2_500, { loop: 0.98, cpu: 0.95 }), idleMachine)).toEqual({ + cause: "busy", + durationMs: 2_500, + }); // Occupied without the CPU to show for it, on a quiet computer: a slow disk, say. expect(classifyStall(START, after(6_000, { loop: 0.9, cpu: 0.1 }), idleMachine)).toEqual({ cause: "unknown", @@ -89,6 +95,9 @@ describe("classifyStall", () => { it("says nothing when the server could answer, or when it slept", () => { expect(classifyStall(START, after(500, { loop: 0.2, cpu: 0.1 }), idleMachine)).toBeNull(); + expect(classifyStall(START, after(1_900, { loop: 1, cpu: 1 }), idleMachine)).toBeNull(); + // A short Linux sleep, or the clock set forward a few seconds, mid-freeze. + expect(classifyStall(START, after(3_000, { loop: 1, cpu: 1 }, 2_500), idleMachine)).toBeNull(); // The wall clock ran on while the monotonic one stood still: a Linux sleep, // or the clock being set. No freeze. expect( @@ -142,3 +151,30 @@ describe("reportableStall", () => { expect(reportableStall(recorded, { monoMs: 2_000, wallMs: 5_000_000 + 3_600_000 })).toBeNull(); }); }); + +describe("parseTopProcesses", () => { + it("names the busiest programs without their paths", () => { + const macOs = [ + " 12.0 /usr/libexec/logd", + " 85.0 /Applications/Google Chrome.app/Contents/MacOS/Google Chrome Helper (Renderer)", + " 0.0 /Users/someone/secret-project/bin/tool", + " 72.4 /Users/someone/Library/App.app/App", + " 3.1 kernel_task", + " 40.5 /usr/bin/node", + " 9.9 /sbin/launchd", + "", + ].join("\n"); + expect(parseTopProcesses(macOs)).toEqual([ + "Google Chrome Helper (Renderer) 85.0%", + "App 72.4%", + "node 40.5%", + "logd 12.0%", + "launchd 9.9%", + ]); + // Linux gives the kernel's short name, which can hold a slash of its own. + expect(parseTopProcesses(" 1.5 kworker/u8:2\n 30.0 node\ngarbage line\n")).toEqual([ + "node 30.0%", + "kworker/u8:2 1.5%", + ]); + }); +}); diff --git a/apps/server/src/observability/StallMonitor.ts b/apps/server/src/observability/StallMonitor.ts index 4d0866bfc..d2a38b1e0 100644 --- a/apps/server/src/observability/StallMonitor.ts +++ b/apps/server/src/observability/StallMonitor.ts @@ -3,7 +3,9 @@ * it is back (`subscribeServerStalls`). * * A timer samples the clocks every half second. When the next sample comes - * five seconds or more late, the server couldn't answer for that long: + * two seconds or more late, the server couldn't answer for that long. Every + * such freeze goes to the log; clients hear only of those of five seconds or + * more, about where their heartbeat gives up. A freeze is classified as: * * - its event loop was occupied the whole time: Threadlines was busy with its * own work when its database calls or its CPU time cover the gap (the @@ -35,8 +37,12 @@ * Only the latest freeze is kept, and only reported for ten minutes after it * ended. Sampling reads clocks and counters only; the load average is read * once a gap is found (`os.cpus()` is too slow on some Linux machines to read - * twice a second). + * twice a second). When the cause lies outside Threadlines (`overloaded`, + * `unknown`), the log line also names the busiest programs on the computer, + * read with `ps` after the freeze: program names and CPU shares only, never + * their arguments, and only into the local log. */ +import { execFile } from "node:child_process"; import { randomUUID } from "node:crypto"; import os from "node:os"; import { performance } from "node:perf_hooks"; @@ -55,7 +61,9 @@ import { readDatabaseCallMs } from "./databaseCallTime.ts"; /** How often the clocks are read. */ export const STALL_SAMPLE_INTERVAL_MS = 500; -/** A gap this long is a freeze: about where a client's heartbeat gives up. */ +/** A gap this long is a freeze, and goes to the log. */ +export const STALL_LOG_MIN_MS = 2_000; +/** A freeze this long is reported to clients: about where their heartbeat gives up. */ export const STALL_MIN_MS = 5_000; /** Past this, only Threadlines' own CPU time makes a gap a freeze (see the module comment). */ export const STALL_UNEXPLAINED_MAX_MS = 2 * 60_000; @@ -105,9 +113,10 @@ export function classifyStall( ): ClassifiedStall | null { const wallGap = next.wallMs - previous.wallMs; const monoGap = next.monoMs - previous.monoMs; - if (monoGap < STALL_MIN_MS) return null; - // The wall clock ran well ahead: asleep (Linux) or the clock was set. - if (wallGap - monoGap >= STALL_MIN_MS) return null; + if (monoGap < STALL_LOG_MIN_MS) return null; + // The wall clock ran ahead by as much as a freeze: asleep (Linux) or the + // clock was set, and the gap can't be read as a freeze. + if (wallGap - monoGap >= STALL_LOG_MIN_MS) return null; const durationMs = Math.round(monoGap); const occupied = next.loopActiveMs - previous.loopActiveMs >= OCCUPIED_SHARE * monoGap; @@ -166,6 +175,52 @@ function readMachineLoad(): MachineLoad { }; } +/** How many of the busiest programs a freeze's log line names. */ +const TOP_PROCESS_COUNT = 5; +/** How long reading the process table may take before the line is logged without it. */ +const TOP_PROCESS_TIMEOUT_MS = 1_500; + +/** + * The busiest programs in `ps -A -o pcpu=,comm=` output, as "name 12.3%". + * `comm` never holds arguments, and of a full path (as macOS gives it) only + * the last part is kept: a path can name the user or a project. + */ +export function parseTopProcesses(output: string, limit = TOP_PROCESS_COUNT): string[] { + const processes: { readonly name: string; readonly cpu: number }[] = []; + for (const line of output.split("\n")) { + const match = /^\s*(\d+(?:\.\d+)?)\s+(.+?)\s*$/.exec(line); + if (!match) continue; + const command = match[2]!; + // A Linux `comm` can hold a slash of its own ("kworker/u8:2"). + const name = command.startsWith("/") ? (command.split("/").at(-1) ?? "") : command; + if (name.length === 0) continue; + processes.push({ name, cpu: Number(match[1]) }); + } + return processes + .toSorted((left, right) => right.cpu - left.cpu) + .slice(0, limit) + .map((entry) => `${entry.name} ${entry.cpu.toFixed(1)}%`); +} + +/** The busiest programs right now, or none on Windows or when `ps` fails or is slow. */ +const readTopProcesses: Effect.Effect> = + process.platform === "win32" + ? Effect.succeed([]) + : Effect.callback>((resume) => { + const child = execFile( + "ps", + ["-A", "-o", "pcpu=,comm="], + { + timeout: TOP_PROCESS_TIMEOUT_MS, + maxBuffer: 4 * 1024 * 1024, + // A decimal point, not a locale's comma. + env: { ...process.env, LC_ALL: "C" }, + }, + (error, stdout) => resume(Effect.succeed(error ? [] : parseTopProcesses(stdout))), + ); + return Effect.sync(() => child.kill()); + }); + export interface RecordedStall extends ClassifiedStall { readonly id: string; /** When it ended, on each clock. */ @@ -229,6 +284,33 @@ const make = Effect.gen(function* () { : Effect.void, ); + const logStall = (stall: RecordedStall, topProcesses: ReadonlyArray) => + Effect.logWarning("server froze", { + cause: stall.cause, + durationMs: stall.durationMs, + ...(stall.databaseMs !== undefined ? { databaseMs: stall.databaseMs } : {}), + ...(stall.load ? { load: stall.load } : {}), + ...(topProcesses.length > 0 ? { topProcesses } : {}), + }); + + // When something outside Threadlines froze it, the log names the busiest + // programs. The process table is read in its own fiber, so the sampling + // and what clients are told never wait on it. + const record = (stall: RecordedStall) => + Effect.gen(function* () { + if (stall.cause === "overloaded" || stall.cause === "unknown") { + yield* Effect.forkIn( + Effect.flatMap(readTopProcesses, (top) => logStall(stall, top)), + scope, + ); + } else { + yield* logStall(stall, []); + } + if (stall.durationMs >= STALL_MIN_MS) { + yield* SubscriptionRef.set(latest, Option.some(stall)); + } + }); + const watch = Effect.sync(() => { const next = readSample(); const stall = classifyStall(previous, next, readMachineLoad); @@ -242,18 +324,7 @@ const make = Effect.gen(function* () { endedAtWallMs: next.wallMs, }); }).pipe( - Effect.flatMap((recorded) => - Option.isSome(recorded) - ? Effect.logWarning("server froze", { - cause: recorded.value.cause, - durationMs: recorded.value.durationMs, - ...(recorded.value.databaseMs !== undefined - ? { databaseMs: recorded.value.databaseMs } - : {}), - ...(recorded.value.load ? { load: recorded.value.load } : {}), - }).pipe(Effect.andThen(SubscriptionRef.set(latest, recorded))) - : expireOld, - ), + Effect.flatMap((recorded) => (Option.isSome(recorded) ? record(recorded.value) : expireOld)), Effect.repeat(Schedule.spaced(Duration.millis(STALL_SAMPLE_INTERVAL_MS))), ); From 3edbb912c36319c7a4b741a2e41aea5ee2a47b88 Mon Sep 17 00:00:00 2001 From: Badcuban <108198679+badcuban@users.noreply.github.com> Date: Fri, 20 Nov 2026 23:57:18 -0500 Subject: [PATCH 3/4] fix(server): a long reader no longer leaves the WAL file large Sets journal_size_limit to 32 MiB so SQLite trims the WAL once a checkpoint resets it. Without it, a read held open across many commits (such as during a storage prune) grew the WAL to hundreds of megabytes and it stayed that size. The cleanup test now counts one loop turn per delete transaction, the partial last one included. --- apps/server/src/persistence/Layers/Sqlite.ts | 5 +++++ .../src/persistence/Layers/StorageMaintenance.test.ts | 9 +++++---- apps/server/src/persistence/Layers/StorageMaintenance.ts | 6 +++--- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/apps/server/src/persistence/Layers/Sqlite.ts b/apps/server/src/persistence/Layers/Sqlite.ts index f7b449894..7738dec5e 100644 --- a/apps/server/src/persistence/Layers/Sqlite.ts +++ b/apps/server/src/persistence/Layers/Sqlite.ts @@ -51,6 +51,11 @@ const setup = (readAhead?: Effect.Effect) => // pages the next one needed: repeated thread loads on payload-heavy chats // took about twice as long. yield* sql`PRAGMA cache_size = -32768;`; + // The WAL file never shrinks on its own: a reader that holds a snapshot + // while many commits land (a long read, a storage prune) grows it to + // hundreds of megabytes and it stays that size. With a limit, SQLite + // trims it back once a checkpoint lets the WAL start over. + yield* sql`PRAGMA journal_size_limit = 33554432;`; yield* runMigrations({ readAhead }); yield* Effect.addFinalizer(() => Effect.ignore(sql`PRAGMA optimize;`)); }), diff --git a/apps/server/src/persistence/Layers/StorageMaintenance.test.ts b/apps/server/src/persistence/Layers/StorageMaintenance.test.ts index 3f5c56287..14e9edf71 100644 --- a/apps/server/src/persistence/Layers/StorageMaintenance.test.ts +++ b/apps/server/src/persistence/Layers/StorageMaintenance.test.ts @@ -7,7 +7,7 @@ import * as Layer from "effect/Layer"; import * as SqlClient from "effect/sql/SqlClient"; import { StorageMaintenance } from "../Services/StorageMaintenance.ts"; -import { StorageMaintenanceLive } from "./StorageMaintenance.ts"; +import { DELETE_BATCH_SIZE, StorageMaintenanceLive } from "./StorageMaintenance.ts"; import { SqlitePersistenceMemory } from "./Sqlite.ts"; const layer = it.layer(StorageMaintenanceLive.pipe(Layer.provideMerge(SqlitePersistenceMemory))); @@ -214,9 +214,9 @@ layer("StorageMaintenance", (it) => { const sql = yield* SqlClient.SqlClient; const storageMaintenance = yield* StorageMaintenance; - // Enough excess activity for several batches. + // Several full batches and a last partial one. const base = 80_000; - const excess = 200; + const excess = DELETE_BATCH_SIZE * 8 + 1; for (let index = 0; index < MAX_THREAD_ACTIVITIES + excess; index += 1) { yield* insertEvent(sql, { sequence: base + index, @@ -247,8 +247,9 @@ layer("StorageMaintenance", (it) => { ); assert.equal(report.deletedActivityEvents, excess); + // One turn after every delete transaction, the partial last one too. // All in one hold, the loop would not turn at all. - assert.isAtLeast(turns, 4); + assert.isAtLeast(turns, Math.ceil(excess / DELETE_BATCH_SIZE)); }), ); diff --git a/apps/server/src/persistence/Layers/StorageMaintenance.ts b/apps/server/src/persistence/Layers/StorageMaintenance.ts index 60a8f992d..de886ed4d 100644 --- a/apps/server/src/persistence/Layers/StorageMaintenance.ts +++ b/apps/server/src/persistence/Layers/StorageMaintenance.ts @@ -33,7 +33,7 @@ const RECEIPT_TTL = Duration.days(7); * ran back to back and froze the server for the whole prune (3 to 20 * seconds). A cold cache still makes single batches slower. */ -const DELETE_BATCH_SIZE = 25; +export const DELETE_BATCH_SIZE = 25; /** Diff blobs can each run to megabytes, so fewer go per batch. */ const DIFF_BLOB_DELETE_BATCH_SIZE = 20; @@ -253,8 +253,8 @@ const makeStorageMaintenance = Effect.gen(function* () { } // No explicit WAL checkpoint: batches this small keep the WAL near - // SQLite's 1,000-page autocheckpoint, so a prune of any size leaves it - // about as large as everyday writes do. A TRUNCATE checkpoint here was + // SQLite's 1,000-page autocheckpoint, and `journal_size_limit` (Sqlite.ts) + // trims a WAL that a long reader let grow. A TRUNCATE checkpoint here was // one more synchronous call, and one that waits out busy readers. const anythingDeleted = deletedReceipts + deletedThreadEvents + deletedActivityEvents + deletedDiffBlobs > 0; From dc530bfc5d7a23e105fcc073b4cad2e38135a49a Mon Sep 17 00:00:00 2001 From: Badcuban <108198679+badcuban@users.noreply.github.com> Date: Fri, 20 Nov 2026 23:57:18 -0500 Subject: [PATCH 4/4] fix(server): a failing ps can't drop a freeze log line Catches a spawn that throws, gives up after a hard timeout, kills the child with SIGKILL, and cuts relative paths down to the program name. --- .../src/observability/StallMonitor.test.ts | 2 + apps/server/src/observability/StallMonitor.ts | 47 ++++++++++++------- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/apps/server/src/observability/StallMonitor.test.ts b/apps/server/src/observability/StallMonitor.test.ts index f88e714de..5ff59d6c3 100644 --- a/apps/server/src/observability/StallMonitor.test.ts +++ b/apps/server/src/observability/StallMonitor.test.ts @@ -176,5 +176,7 @@ describe("parseTopProcesses", () => { "node 30.0%", "kworker/u8:2 1.5%", ]); + // A process can name itself with a relative path. + expect(parseTopProcesses(" 91.0 ./private/tool\n")).toEqual(["tool 91.0%"]); }); }); diff --git a/apps/server/src/observability/StallMonitor.ts b/apps/server/src/observability/StallMonitor.ts index d2a38b1e0..8dd349160 100644 --- a/apps/server/src/observability/StallMonitor.ts +++ b/apps/server/src/observability/StallMonitor.ts @@ -191,8 +191,10 @@ export function parseTopProcesses(output: string, limit = TOP_PROCESS_COUNT): st const match = /^\s*(\d+(?:\.\d+)?)\s+(.+?)\s*$/.exec(line); if (!match) continue; const command = match[2]!; - // A Linux `comm` can hold a slash of its own ("kworker/u8:2"). - const name = command.startsWith("/") ? (command.split("/").at(-1) ?? "") : command; + // A Linux `comm` can hold a slash of its own ("kworker/u8:2"); only a + // path, absolute or relative, is cut down to its last part. + const isPath = command.startsWith("/") || command.startsWith("."); + const name = isPath ? (command.split("/").at(-1) ?? "") : command; if (name.length === 0) continue; processes.push({ name, cpu: Number(match[1]) }); } @@ -202,24 +204,37 @@ export function parseTopProcesses(output: string, limit = TOP_PROCESS_COUNT): st .map((entry) => `${entry.name} ${entry.cpu.toFixed(1)}%`); } -/** The busiest programs right now, or none on Windows or when `ps` fails or is slow. */ +/** + * The busiest programs right now, or none on Windows or when `ps` can't + * start, fails, or is slow. Gives up after the timeout even when the child + * won't exit, and kills it on the way out. + */ const readTopProcesses: Effect.Effect> = process.platform === "win32" ? Effect.succeed([]) : Effect.callback>((resume) => { - const child = execFile( - "ps", - ["-A", "-o", "pcpu=,comm="], - { - timeout: TOP_PROCESS_TIMEOUT_MS, - maxBuffer: 4 * 1024 * 1024, - // A decimal point, not a locale's comma. - env: { ...process.env, LC_ALL: "C" }, - }, - (error, stdout) => resume(Effect.succeed(error ? [] : parseTopProcesses(stdout))), - ); - return Effect.sync(() => child.kill()); - }); + try { + const child = execFile( + "ps", + ["-A", "-o", "pcpu=,comm="], + { + timeout: TOP_PROCESS_TIMEOUT_MS, + killSignal: "SIGKILL", + maxBuffer: 4 * 1024 * 1024, + // A decimal point, not a locale's comma. + env: { ...process.env, LC_ALL: "C" }, + }, + (error, stdout) => resume(Effect.succeed(error ? [] : parseTopProcesses(stdout))), + ); + return Effect.sync(() => child.kill("SIGKILL")); + } catch { + resume(Effect.succeed([])); + return; + } + }).pipe( + Effect.timeoutOption(Duration.millis(TOP_PROCESS_TIMEOUT_MS + 500)), + Effect.map(Option.getOrElse((): ReadonlyArray => [])), + ); export interface RecordedStall extends ClassifiedStall { readonly id: string;