Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions apps/server/src/observability/StallMonitor.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { describe, expect, it } from "@effect/vitest";
import {
classifyStall,
type MachineLoad,
parseTopProcesses,
reportableStall,
STALL_REPORT_WINDOW_MS,
type StallSample,
Expand Down Expand Up @@ -80,6 +81,11 @@ describe("classifyStall", () => {
durationMs: 9_000,
load: { average1m: 219.4, cpuCount: 18 },
});
// From two seconds on, a freeze is caught (for the log; clients hear of it from five).
expect(classifyStall(START, after(2_500, { loop: 0.98, cpu: 0.95 }), idleMachine)).toEqual({
cause: "busy",
durationMs: 2_500,
});
// Occupied without the CPU to show for it, on a quiet computer: a slow disk, say.
expect(classifyStall(START, after(6_000, { loop: 0.9, cpu: 0.1 }), idleMachine)).toEqual({
cause: "unknown",
Expand All @@ -89,6 +95,9 @@ describe("classifyStall", () => {

it("says nothing when the server could answer, or when it slept", () => {
expect(classifyStall(START, after(500, { loop: 0.2, cpu: 0.1 }), idleMachine)).toBeNull();
expect(classifyStall(START, after(1_900, { loop: 1, cpu: 1 }), idleMachine)).toBeNull();
// A short Linux sleep, or the clock set forward a few seconds, mid-freeze.
expect(classifyStall(START, after(3_000, { loop: 1, cpu: 1 }, 2_500), idleMachine)).toBeNull();
// The wall clock ran on while the monotonic one stood still: a Linux sleep,
// or the clock being set. No freeze.
expect(
Expand Down Expand Up @@ -142,3 +151,32 @@ describe("reportableStall", () => {
expect(reportableStall(recorded, { monoMs: 2_000, wallMs: 5_000_000 + 3_600_000 })).toBeNull();
});
});

describe("parseTopProcesses", () => {
it("names the busiest programs without their paths", () => {
const macOs = [
" 12.0 /usr/libexec/logd",
" 85.0 /Applications/Google Chrome.app/Contents/MacOS/Google Chrome Helper (Renderer)",
" 0.0 /Users/someone/secret-project/bin/tool",
" 72.4 /Users/someone/Library/App.app/App",
" 3.1 kernel_task",
" 40.5 /usr/bin/node",
" 9.9 /sbin/launchd",
"",
].join("\n");
expect(parseTopProcesses(macOs)).toEqual([
"Google Chrome Helper (Renderer) 85.0%",
"App 72.4%",
"node 40.5%",
"logd 12.0%",
"launchd 9.9%",
]);
// Linux gives the kernel's short name, which can hold a slash of its own.
expect(parseTopProcesses(" 1.5 kworker/u8:2\n 30.0 node\ngarbage line\n")).toEqual([
"node 30.0%",
"kworker/u8:2 1.5%",
]);
// A process can name itself with a relative path.
expect(parseTopProcesses(" 91.0 ./private/tool\n")).toEqual(["tool 91.0%"]);
});
});
122 changes: 104 additions & 18 deletions apps/server/src/observability/StallMonitor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@
* it is back (`subscribeServerStalls`).
*
* A timer samples the clocks every half second. When the next sample comes
* five seconds or more late, the server couldn't answer for that long:
* two seconds or more late, the server couldn't answer for that long. Every
* such freeze goes to the log; clients hear only of those of five seconds or
* more, about where their heartbeat gives up. A freeze is classified as:
*
* - its event loop was occupied the whole time: Threadlines was busy with its
* own work when its database calls or its CPU time cover the gap (the
Expand Down Expand Up @@ -35,8 +37,12 @@
* Only the latest freeze is kept, and only reported for ten minutes after it
* ended. Sampling reads clocks and counters only; the load average is read
* once a gap is found (`os.cpus()` is too slow on some Linux machines to read
* twice a second).
* twice a second). When the cause lies outside Threadlines (`overloaded`,
* `unknown`), the log line also names the busiest programs on the computer,
* read with `ps` after the freeze: program names and CPU shares only, never
* their arguments, and only into the local log.
*/
import { execFile } from "node:child_process";
import { randomUUID } from "node:crypto";
import os from "node:os";
import { performance } from "node:perf_hooks";
Expand All @@ -55,7 +61,9 @@ import { readDatabaseCallMs } from "./databaseCallTime.ts";

/** How often the clocks are read. */
export const STALL_SAMPLE_INTERVAL_MS = 500;
/** A gap this long is a freeze: about where a client's heartbeat gives up. */
/** A gap this long is a freeze, and goes to the log. */
export const STALL_LOG_MIN_MS = 2_000;
/** A freeze this long is reported to clients: about where their heartbeat gives up. */
export const STALL_MIN_MS = 5_000;
/** Past this, only Threadlines' own CPU time makes a gap a freeze (see the module comment). */
export const STALL_UNEXPLAINED_MAX_MS = 2 * 60_000;
Expand Down Expand Up @@ -105,9 +113,10 @@ export function classifyStall(
): ClassifiedStall | null {
const wallGap = next.wallMs - previous.wallMs;
const monoGap = next.monoMs - previous.monoMs;
if (monoGap < STALL_MIN_MS) return null;
// The wall clock ran well ahead: asleep (Linux) or the clock was set.
if (wallGap - monoGap >= STALL_MIN_MS) return null;
if (monoGap < STALL_LOG_MIN_MS) return null;
// The wall clock ran ahead by as much as a freeze: asleep (Linux) or the
// clock was set, and the gap can't be read as a freeze.
if (wallGap - monoGap >= STALL_LOG_MIN_MS) return null;

const durationMs = Math.round(monoGap);
const occupied = next.loopActiveMs - previous.loopActiveMs >= OCCUPIED_SHARE * monoGap;
Expand Down Expand Up @@ -166,6 +175,67 @@ function readMachineLoad(): MachineLoad {
};
}

/** How many of the busiest programs a freeze's log line names. */
const TOP_PROCESS_COUNT = 5;
/** How long reading the process table may take before the line is logged without it. */
const TOP_PROCESS_TIMEOUT_MS = 1_500;

/**
* The busiest programs in `ps -A -o pcpu=,comm=` output, as "name 12.3%".
* `comm` never holds arguments, and of a full path (as macOS gives it) only
* the last part is kept: a path can name the user or a project.
*/
export function parseTopProcesses(output: string, limit = TOP_PROCESS_COUNT): string[] {
const processes: { readonly name: string; readonly cpu: number }[] = [];
for (const line of output.split("\n")) {
const match = /^\s*(\d+(?:\.\d+)?)\s+(.+?)\s*$/.exec(line);
if (!match) continue;
const command = match[2]!;
// A Linux `comm` can hold a slash of its own ("kworker/u8:2"); only a
// path, absolute or relative, is cut down to its last part.
const isPath = command.startsWith("/") || command.startsWith(".");
const name = isPath ? (command.split("/").at(-1) ?? "") : command;
if (name.length === 0) continue;
processes.push({ name, cpu: Number(match[1]) });
}
return processes
.toSorted((left, right) => right.cpu - left.cpu)
.slice(0, limit)
.map((entry) => `${entry.name} ${entry.cpu.toFixed(1)}%`);
}

/**
* The busiest programs right now, or none on Windows or when `ps` can't
* start, fails, or is slow. Gives up after the timeout even when the child
* won't exit, and kills it on the way out.
*/
const readTopProcesses: Effect.Effect<ReadonlyArray<string>> =
process.platform === "win32"
? Effect.succeed([])
: Effect.callback<ReadonlyArray<string>>((resume) => {
try {
const child = execFile(
"ps",
["-A", "-o", "pcpu=,comm="],
{
timeout: TOP_PROCESS_TIMEOUT_MS,
killSignal: "SIGKILL",
maxBuffer: 4 * 1024 * 1024,
// A decimal point, not a locale's comma.
env: { ...process.env, LC_ALL: "C" },
},
(error, stdout) => resume(Effect.succeed(error ? [] : parseTopProcesses(stdout))),
);
return Effect.sync(() => child.kill("SIGKILL"));
} catch {
resume(Effect.succeed([]));
return;
}
}).pipe(
Effect.timeoutOption(Duration.millis(TOP_PROCESS_TIMEOUT_MS + 500)),
Effect.map(Option.getOrElse((): ReadonlyArray<string> => [])),
);

export interface RecordedStall extends ClassifiedStall {
readonly id: string;
/** When it ended, on each clock. */
Expand Down Expand Up @@ -229,6 +299,33 @@ const make = Effect.gen(function* () {
: Effect.void,
);

const logStall = (stall: RecordedStall, topProcesses: ReadonlyArray<string>) =>
Effect.logWarning("server froze", {
cause: stall.cause,
durationMs: stall.durationMs,
...(stall.databaseMs !== undefined ? { databaseMs: stall.databaseMs } : {}),
...(stall.load ? { load: stall.load } : {}),
...(topProcesses.length > 0 ? { topProcesses } : {}),
});

// When something outside Threadlines froze it, the log names the busiest
// programs. The process table is read in its own fiber, so the sampling
// and what clients are told never wait on it.
const record = (stall: RecordedStall) =>
Effect.gen(function* () {
if (stall.cause === "overloaded" || stall.cause === "unknown") {
yield* Effect.forkIn(
Effect.flatMap(readTopProcesses, (top) => logStall(stall, top)),
scope,
);
} else {
yield* logStall(stall, []);
}
if (stall.durationMs >= STALL_MIN_MS) {
yield* SubscriptionRef.set(latest, Option.some(stall));
}
});

const watch = Effect.sync(() => {
const next = readSample();
const stall = classifyStall(previous, next, readMachineLoad);
Expand All @@ -242,18 +339,7 @@ const make = Effect.gen(function* () {
endedAtWallMs: next.wallMs,
});
}).pipe(
Effect.flatMap((recorded) =>
Option.isSome(recorded)
? Effect.logWarning("server froze", {
cause: recorded.value.cause,
durationMs: recorded.value.durationMs,
...(recorded.value.databaseMs !== undefined
? { databaseMs: recorded.value.databaseMs }
: {}),
...(recorded.value.load ? { load: recorded.value.load } : {}),
}).pipe(Effect.andThen(SubscriptionRef.set(latest, recorded)))
: expireOld,
),
Effect.flatMap((recorded) => (Option.isSome(recorded) ? record(recorded.value) : expireOld)),
Effect.repeat(Schedule.spaced(Duration.millis(STALL_SAMPLE_INTERVAL_MS))),
);

Expand Down
5 changes: 5 additions & 0 deletions apps/server/src/persistence/Layers/Sqlite.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,11 @@ const setup = (readAhead?: Effect.Effect<void>) =>
// pages the next one needed: repeated thread loads on payload-heavy chats
// took about twice as long.
yield* sql`PRAGMA cache_size = -32768;`;
// The WAL file never shrinks on its own: a reader that holds a snapshot
// while many commits land (a long read, a storage prune) grows it to
// hundreds of megabytes and it stays that size. With a limit, SQLite
// trims it back once a checkpoint lets the WAL start over.
yield* sql`PRAGMA journal_size_limit = 33554432;`;
yield* runMigrations({ readAhead });
yield* Effect.addFinalizer(() => Effect.ignore(sql`PRAGMA optimize;`));
}),
Expand Down
46 changes: 45 additions & 1 deletion apps/server/src/persistence/Layers/StorageMaintenance.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import * as Layer from "effect/Layer";
import * as SqlClient from "effect/sql/SqlClient";

import { StorageMaintenance } from "../Services/StorageMaintenance.ts";
import { StorageMaintenanceLive } from "./StorageMaintenance.ts";
import { DELETE_BATCH_SIZE, StorageMaintenanceLive } from "./StorageMaintenance.ts";
import { SqlitePersistenceMemory } from "./Sqlite.ts";

const layer = it.layer(StorageMaintenanceLive.pipe(Layer.provideMerge(SqlitePersistenceMemory)));
Expand Down Expand Up @@ -209,6 +209,50 @@ layer("StorageMaintenance", (it) => {
}),
);

it.effect("lets the event loop turn between delete batches", () =>
Effect.gen(function* () {
const sql = yield* SqlClient.SqlClient;
const storageMaintenance = yield* StorageMaintenance;

// Several full batches and a last partial one.
const base = 80_000;
const excess = DELETE_BATCH_SIZE * 8 + 1;
for (let index = 0; index < MAX_THREAD_ACTIVITIES + excess; index += 1) {
yield* insertEvent(sql, {
sequence: base + index,
streamId: "thread-long",
eventType: "thread.activity-appended",
});
}
yield* setProjectorCheckpoint(sql, base + MAX_THREAD_ACTIVITIES + excess);

// Counts turns of the event loop while the prune runs: a callback that
// queues itself again runs once per turn, and only when nothing holds
// the loop.
let running = true;
let turns = 0;
const countTurn = () => {
if (!running) return;
turns += 1;
setImmediate(countTurn);
};
setImmediate(countTurn);

const report = yield* storageMaintenance.runOnce.pipe(
Effect.ensuring(
Effect.sync(() => {
running = false;
}),
),
);

assert.equal(report.deletedActivityEvents, excess);
// One turn after every delete transaction, the partial last one too.
// All in one hold, the loop would not turn at all.
assert.isAtLeast(turns, Math.ceil(excess / DELETE_BATCH_SIZE));
}),
);

it.effect("caps activity pruning at the projector checkpoint", () =>
Effect.gen(function* () {
const sql = yield* SqlClient.SqlClient;
Expand Down
Loading
Loading