From 6ae917e3a8176ffb4b2dd3e7563f0196b104564b Mon Sep 17 00:00:00 2001 From: Badcuban <108198679+badcuban@users.noreply.github.com> Date: Sat, 21 Nov 2026 03:02:48 -0500 Subject: [PATCH] feat: Settings > Diagnostics shows which thread is using the CPU Nothing in the app said which thread's agents were using the machine. The Diagnostics page listed the server's child processes, but none were tied to a thread. A new "CPU by thread" section at the top of the page has one row per thread, one for Threadlines itself, and one for a provider whose single process serves all its threads (OpenCode). Rows are sorted by the average over the last minute, a thread's name opens it, and a row expands to its busiest processes. Ownership comes from what the server already tracks: each driver's agent process ledger (now listed through ProviderAdapterShape.listProcesses and ProviderService.listAgentProcesses) and each terminal's shell. A process belongs to the nearest tracked process above it, or to the process group of an agent that leads one. CPU is what each process used since the read before, from `ps time`. `ps pcpu` is an average over a process's whole life on Linux, so the POSIX process query gained a `time=` column. It adds no timer: the existing on-demand sampler hands each read to the new view, and the page asks only while the section is on screen and the window is in use. --- .../diagnostics/ProcessDiagnostics.test.ts | 19 +- .../src/diagnostics/ProcessDiagnostics.ts | 34 +- .../src/diagnostics/ProcessResourceMonitor.ts | 49 ++- .../src/diagnostics/ThreadCpuUsage.test.ts | 228 ++++++++++ apps/server/src/diagnostics/ThreadCpuUsage.ts | 403 ++++++++++++++++++ apps/server/src/guide/topics.ts | 10 + .../Layers/CheckpointReactor.test.ts | 1 + .../Layers/ProviderCommandReactor.test.ts | 1 + .../Layers/ProviderRuntimeIngestion.test.ts | 1 + .../Layers/ProjectSetupScriptRunner.test.ts | 2 + .../src/provider/Layers/ClaudeAdapter.ts | 1 + .../src/provider/Layers/CodexAdapter.ts | 1 + .../server/src/provider/Layers/MuseAdapter.ts | 1 + .../provider/Layers/OpenCodeAdapter.test.ts | 2 + .../src/provider/Layers/OpenCodeAdapter.ts | 7 + apps/server/src/provider/Layers/PiAdapter.ts | 1 + .../provider/Layers/ProviderRegistry.test.ts | 2 + .../src/provider/Layers/ProviderService.ts | 14 + .../Layers/ProviderSessionReaper.test.ts | 1 + .../src/provider/Services/ProviderAdapter.ts | 24 ++ .../src/provider/Services/ProviderService.ts | 14 + apps/server/src/provider/acp/AcpAdapter.ts | 1 + .../server/src/provider/agentProcessLedger.ts | 20 +- .../src/provider/opencode/OpenCodeServer.ts | 5 +- .../opencode/OpenCodeServerManager.ts | 3 + apps/server/src/server.test.ts | 11 + apps/server/src/server.ts | 8 +- apps/server/src/terminal/Layers/Manager.ts | 9 + apps/server/src/terminal/Services/Manager.ts | 8 + apps/server/src/ws.ts | 6 + .../settings/DiagnosticsSettings.tsx | 2 + .../settings/SettingsPanels.browser.tsx | 76 +++- .../settings/ThreadCpuUsageSection.tsx | 351 +++++++++++++++ apps/web/src/localApi.ts | 1 + apps/web/src/rpc/wsRpcClient.ts | 5 + packages/contracts/src/ipc.ts | 3 + packages/contracts/src/rpc.ts | 8 + packages/contracts/src/server.ts | 56 +++ packages/shared/src/appPlaces.ts | 12 +- 39 files changed, 1381 insertions(+), 20 deletions(-) create mode 100644 apps/server/src/diagnostics/ThreadCpuUsage.test.ts create mode 100644 apps/server/src/diagnostics/ThreadCpuUsage.ts create mode 100644 apps/web/src/components/settings/ThreadCpuUsageSection.tsx diff --git a/apps/server/src/diagnostics/ProcessDiagnostics.test.ts b/apps/server/src/diagnostics/ProcessDiagnostics.test.ts index 6923d0d20..9e0b467b5 100644 --- a/apps/server/src/diagnostics/ProcessDiagnostics.test.ts +++ b/apps/server/src/diagnostics/ProcessDiagnostics.test.ts @@ -46,8 +46,9 @@ describe("ProcessDiagnostics", () => { Effect.sync(() => { const rows = ProcessDiagnostics.parsePosixProcessRows( [ - " 10 1 10 Ss 0.0 1024 01:02.03 /usr/bin/node server.js", - " 11 10 10 S+ 12.5 20480 00:04 codex app-server --config /tmp/one two", + // macOS prints CPU time as minutes and seconds, Linux as [days-]hours:minutes:seconds. + " 10 1 10 Ss 0.0 1024 01:02.03 398:51.57 /usr/bin/node server.js", + " 11 10 10 S+ 12.5 20480 00:04 1-02:03:04 codex app-server --config /tmp/one two", ].join("\n"), ); @@ -61,6 +62,7 @@ describe("ProcessDiagnostics", () => { rssBytes: 1024 * 1024, elapsed: "01:02.03", command: "/usr/bin/node server.js", + cpuTimeMs: (398 * 60 + 51.57) * 1000, }, { pid: 11, @@ -71,6 +73,7 @@ describe("ProcessDiagnostics", () => { rssBytes: 20480 * 1024, elapsed: "00:04", command: "codex app-server --config /tmp/one two", + cpuTimeMs: (((24 + 2) * 60 + 3) * 60 + 4) * 1000, }, ]); }), @@ -130,7 +133,7 @@ describe("ProcessDiagnostics", () => { cpuPercent: 9, rssBytes: 9_000, elapsed: "00:00", - command: "ps -axo pid=,ppid=,pgid=,stat=,pcpu=,rss=,etime=,command=", + command: "ps -axo pid=,ppid=,pgid=,stat=,pcpu=,rss=,etime=,time=,command=", }, ], }); @@ -222,8 +225,8 @@ describe("ProcessDiagnostics", () => { return Effect.succeed( mockHandle({ stdout: [ - ` ${process.pid} 1 ${process.pid} Ss 0.0 1024 01:02.03 t3 server`, - ` 4242 ${process.pid} ${process.pid} S 1.5 2048 00:04 agent`, + ` ${process.pid} 1 ${process.pid} Ss 0.0 1024 01:02.03 0:01.50 t3 server`, + ` 4242 ${process.pid} ${process.pid} S 1.5 2048 00:04 0:00.06 agent`, ].join("\n"), }), ); @@ -254,7 +257,7 @@ describe("ProcessDiagnostics", () => { expect(commands).toEqual([ { command: "ps", - args: ["-axo", "pid=,ppid=,pgid=,stat=,pcpu=,rss=,etime=,command="], + args: ["-axo", "pid=,ppid=,pgid=,stat=,pcpu=,rss=,etime=,time=,command="], }, ]); } @@ -837,8 +840,8 @@ describe("ProcessDiagnostics", () => { Effect.succeed( mockHandle({ stdout: [ - ` ${process.pid} 1 ${process.pid} Ss 0.0 1024 01:02.03 t3 server`, - ` 4242 ${process.pid} ${process.pid} R 1.5 2048 00:00 ps -axo pid=,ppid=,pgid=,stat=,pcpu=,rss=,etime=,command=`, + ` ${process.pid} 1 ${process.pid} Ss 0.0 1024 01:02.03 0:01.50 t3 server`, + ` 4242 ${process.pid} ${process.pid} R 1.5 2048 00:00 0:00.01 ps -axo pid=,ppid=,pgid=,stat=,pcpu=,rss=,etime=,time=,command=`, ].join("\n"), }), ), diff --git a/apps/server/src/diagnostics/ProcessDiagnostics.ts b/apps/server/src/diagnostics/ProcessDiagnostics.ts index b18462775..581491c13 100644 --- a/apps/server/src/diagnostics/ProcessDiagnostics.ts +++ b/apps/server/src/diagnostics/ProcessDiagnostics.ts @@ -30,10 +30,21 @@ export interface ProcessRow { readonly rssBytes: number; readonly elapsed: string; readonly command: string; + /** + * All the CPU time the process has used since it started (POSIX `ps time`). + * Two reads of it give what the process used in between, which `cpuPercent` + * can't: on Linux that is an average over the process's whole life. + * Absent on Windows, where `cpuPercent` already comes from such a pair. + */ + readonly cpuTimeMs?: number | undefined; } const PROCESS_QUERY_TIMEOUT_MS = 15_000; -const POSIX_PROCESS_QUERY_COMMAND = "pid=,ppid=,pgid=,stat=,pcpu=,rss=,etime=,command="; +const POSIX_PROCESS_QUERY_COMMAND = "pid=,ppid=,pgid=,stat=,pcpu=,rss=,etime=,time=,command="; +/** Our own `ps` as it shows up in the process list it prints. */ +const POSIX_PROCESS_QUERY_PATTERN = new RegExp( + `(?:^|[/\\\\])ps\\s+-axo\\s+${POSIX_PROCESS_QUERY_COMMAND}`, +); const PROCESS_QUERY_MAX_OUTPUT_BYTES = 2 * 1024 * 1024; const PROCESS_SNAPSHOT_TTL_MS = 2_000; const PROCESS_SNAPSHOT_INITIAL_FAILURE_BACKOFF_MS = 2_000; @@ -106,10 +117,22 @@ function parseNumber(value: string): number | null { return Number.isFinite(parsed) ? parsed : null; } +/** + * `ps time` in milliseconds: `398:51.57` (macOS, minutes and seconds) or + * `[1-]02:03:04` (Linux, whole seconds). Null when it reads as neither. + */ +export function parsePosixCpuTime(value: string): number | null { + const match = /^(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+(?:\.\d+)?)$/.exec(value); + if (!match) return null; + const [, days = "0", hours = "0", minutes = "0", seconds = "0"] = match; + const total = ((Number(days) * 24 + Number(hours)) * 60 + Number(minutes)) * 60 + Number(seconds); + return Number.isFinite(total) ? Math.round(total * 1000) : null; +} + export function parsePosixProcessRows(output: string): ReadonlyArray { const rows: ProcessRow[] = []; const rowPattern = - /^\s*(\d+)\s+(\d+)\s+(-?\d+)\s+(\S+)\s+([+-]?(?:\d+\.?\d*|\.\d+))\s+(\d+)\s+(\S+)\s+(.+)$/; + /^\s*(\d+)\s+(\d+)\s+(-?\d+)\s+(\S+)\s+([+-]?(?:\d+\.?\d*|\.\d+))\s+(\d+)\s+(\S+)\s+(\S+)\s+(.+)$/; for (const line of output.split(/\r?\n/)) { if (line.trim().length === 0) continue; @@ -124,7 +147,8 @@ export function parsePosixProcessRows(output: string): ReadonlyArray const cpuText = match[5]; const rssText = match[6]; const elapsed = match[7]; - const command = match[8]; + const cpuTimeText = match[8]; + const command = match[9]; if ( pidText === undefined || ppidText === undefined || @@ -133,6 +157,7 @@ export function parsePosixProcessRows(output: string): ReadonlyArray cpuText === undefined || rssText === undefined || elapsed === undefined || + cpuTimeText === undefined || command === undefined ) { continue; @@ -165,6 +190,7 @@ export function parsePosixProcessRows(output: string): ReadonlyArray rssBytes: rssKiB * 1024, elapsed, command, + cpuTimeMs: parsePosixCpuTime(cpuTimeText) ?? undefined, }); } @@ -426,7 +452,7 @@ export function isDiagnosticsQueryProcess(row: ProcessRow, serverPid: number): b const command = row.command.trim(); return ( - /(?:^|[/\\])ps\s+-axo\s+pid=,ppid=,pgid=,stat=,pcpu=,rss=,etime=,command=/.test(command) || + POSIX_PROCESS_QUERY_PATTERN.test(command) || (/\bpowershell(?:\.exe)?\b/i.test(command) && /\bGet-CimInstance\s+Win32_Process\b/i.test(command)) ); diff --git a/apps/server/src/diagnostics/ProcessResourceMonitor.ts b/apps/server/src/diagnostics/ProcessResourceMonitor.ts index 4632446a4..519800cc9 100644 --- a/apps/server/src/diagnostics/ProcessResourceMonitor.ts +++ b/apps/server/src/diagnostics/ProcessResourceMonitor.ts @@ -50,11 +50,32 @@ interface MonitorState { readonly samplerRunning: boolean; } +/** One look at the process table, as the sampler took it. */ +export interface ProcessTableRead { + readonly rows: ReadonlyArray; + readonly serverPid: number; + readonly sampledAtMs: number; +} + export interface ProcessResourceMonitorShape { readonly readCurrent: Effect.Effect; readonly readHistory: ( input: ServerProcessResourceHistoryInput, ) => Effect.Effect; + /** + * Keeps the sampler running for its active window, as `readHistory` does, + * and takes a read at once when the latest is older than one interval. + * Fails only when no read has ever succeeded. + */ + readonly requestSamples: Effect.Effect; + /** + * Hands every read the sampler takes to `observer`, until the caller's + * scope closes. Runs only while something keeps the sampler going, so an + * observer adds no reads of its own. + */ + readonly observeReads: ( + observer: (read: ProcessTableRead) => Effect.Effect, + ) => Effect.Effect; } export class ProcessResourceMonitor extends Context.Service< @@ -314,10 +335,17 @@ export const make = Effect.fn("makeProcessResourceMonitor")(function* () { samplerRunning: false, }); + const observers = new Set<(read: ProcessTableRead) => Effect.Effect>(); + const sampleOnce = Effect.gen(function* () { const sampledAt = yield* DateTime.now; const sampledAtMs = DateTime.toEpochMillis(sampledAt); const rows = yield* processSnapshot.readTree.pipe(Effect.withTracerEnabled(false)); + for (const observer of observers) { + yield* Effect.suspend(() => observer({ rows, serverPid: process.pid, sampledAtMs })).pipe( + Effect.catchCause(() => Effect.void), + ); + } const samples = collectMonitoredSamples({ rows, serverPid: process.pid, @@ -420,7 +448,26 @@ export const make = Effect.fn("makeProcessResourceMonitor")(function* () { }); }); - return ProcessResourceMonitor.of({ readCurrent, readHistory }); + const requestSamples: ProcessResourceMonitorShape["requestSamples"] = Effect.gen(function* () { + yield* ensureSamplerRunning; + const nowMs = DateTime.toEpochMillis(yield* DateTime.now); + const latest = (yield* Ref.get(state)).latestSampledAt; + if (latest === null || nowMs - DateTime.toEpochMillis(latest) >= SAMPLE_INTERVAL_MS) { + yield* sampleOnce; + } + const current = yield* Ref.get(state); + if (current.latestRows === null) { + return yield* Effect.fail(current.lastError ?? "The process list couldn't be read."); + } + }); + + const observeReads: ProcessResourceMonitorShape["observeReads"] = (observer) => + Effect.acquireRelease( + Effect.sync(() => observers.add(observer)), + () => Effect.sync(() => observers.delete(observer)), + ).pipe(Effect.asVoid); + + return ProcessResourceMonitor.of({ readCurrent, readHistory, requestSamples, observeReads }); }); export const layer = Layer.effect(ProcessResourceMonitor, make()); diff --git a/apps/server/src/diagnostics/ThreadCpuUsage.test.ts b/apps/server/src/diagnostics/ThreadCpuUsage.test.ts new file mode 100644 index 000000000..2f5f44b7e --- /dev/null +++ b/apps/server/src/diagnostics/ThreadCpuUsage.test.ts @@ -0,0 +1,228 @@ +import { describe, expect, it } from "vitest"; + +import type { ProcessRow } from "./ProcessDiagnostics.ts"; +import { + groupProcessRead, + makeIntervalCpu, + summarizeGroupedReads, + type ProcessGroupRead, + type ProcessOwners, +} from "./ThreadCpuUsage.ts"; + +const SERVER = 100; +const APP = 50; + +const row = ( + pid: number, + ppid: number, + cpuPercent: number, + command: string, + pgid: number | null = pid, +): ProcessRow => ({ + pid, + ppid, + pgid, + status: "S", + cpuPercent, + rssBytes: 1_000 * pid, + elapsed: "01:00", + command, +}); + +const owners = (overrides: Partial = {}): ProcessOwners => ({ + agents: [], + shared: [], + shells: [], + ...overrides, +}); + +const byKey = (groups: ReadonlyArray) => + Object.fromEntries( + groups.map((group) => [ + group.key.kind === "thread" + ? group.key.threadId + : group.key.kind === "shared" + ? `shared:${group.key.provider}` + : "threadlines", + { + cpu: group.cpuPercent, + pids: group.topProcesses.map((process) => process.pid).toSorted((a, b) => a - b), + }, + ]), + ); + +describe("groupProcessRead", () => { + const rows = [ + row(1, 0, 0, "launchd"), + row(APP, 1, 12, "Threadlines"), + row(51, APP, 20, "Threadlines Helper (Renderer)"), + row(SERVER, APP, 5, "node server.mjs", APP), + row(101, SERVER, 1, "git status", SERVER), + // Thread A: two agents in one room. + row(200, SERVER, 10, "claude --resume a1"), + row(201, 200, 300, "npm test", 200), + row(210, SERVER, 4, "codex app-server"), + // Agent B, started by a command of agent A's: the nearest tracked agent wins. + row(220, 201, 7, "claude --resume b1"), + row(221, 220, 50, "vitest", 220), + // A browser left behind by agent A's tests, handed to pid 1. + row(230, 1, 400, "chrome-headless-shell", 200), + // Agent C exited; what it left in its group still runs, reparented. + row(240, 1, 90, "playwright worker", 300), + // OpenCode's one server, for every thread of it. + row(250, SERVER, 30, "opencode serve"), + row(251, 250, 15, "bash -c make", 250), + // A terminal of thread D. + row(260, SERVER, 0, "/bin/zsh -l"), + row(261, 260, 80, "cargo build", 260), + // Not ours. + row(900, 1, 500, "Spotlight"), + // The diagnostics query itself. + row(102, SERVER, 3, "ps -axo pid=,ppid=,pgid=,stat=,pcpu=,rss=,etime=,time=,command="), + ]; + const tracked = owners({ + agents: [ + { threadId: "thread-a", pid: 200, ownGroup: true, alive: true }, + { threadId: "thread-a", pid: 210, ownGroup: true, alive: true }, + { threadId: "thread-b", pid: 220, ownGroup: true, alive: true }, + { threadId: "thread-c", pid: 300, ownGroup: true, alive: false }, + ], + shared: [{ provider: "opencode" as never, pid: 250 }], + shells: [{ threadId: "thread-d", pid: 260 }], + }); + + it("puts every process in one group by who started it", () => { + const groups = byKey( + groupProcessRead({ rows, serverPid: SERVER, owners: tracked, includeAppProcesses: true }), + ); + expect(groups).toEqual({ + threadlines: { cpu: 12 + 20 + 5 + 1, pids: [APP, 51, SERVER, 101] }, + "thread-a": { cpu: 10 + 300 + 4 + 400, pids: [200, 201, 210, 230] }, + "thread-b": { cpu: 7 + 50, pids: [220, 221] }, + "thread-c": { cpu: 90, pids: [240] }, + "shared:opencode": { cpu: 30 + 15, pids: [250, 251] }, + "thread-d": { cpu: 80, pids: [260, 261] }, + }); + }); + + it("keeps a tracked agent that left the server's tree, and drops a reused group id", () => { + const groups = byKey( + groupProcessRead({ + rows: [ + row(SERVER, APP, 5, "node server.mjs", APP), + row(200, SERVER, 10, "claude --resume a1"), + // Agent B, in A's group, reparented when the shell between them ended. + row(220, 1, 7, "claude --resume b1", 200), + row(221, 220, 50, "vitest", 200), + // Agent C exited and its group emptied; the number now leads a stranger's group. + row(300, 1, 70, "Xcode build", 300), + row(301, 300, 30, "clang", 300), + ], + serverPid: SERVER, + owners: owners({ + agents: [ + { threadId: "thread-a", pid: 200, ownGroup: true, alive: true }, + { threadId: "thread-b", pid: 220, ownGroup: true, alive: true }, + { threadId: "thread-c", pid: 300, ownGroup: true, alive: false }, + ], + }), + includeAppProcesses: false, + }), + ); + expect(groups["thread-b"]).toEqual({ cpu: 57, pids: [220, 221] }); + expect(groups["thread-a"]).toEqual({ cpu: 10, pids: [200] }); + expect(groups["thread-c"]).toBeUndefined(); + }); + + it("leaves the app's own processes out unless the desktop app started the server", () => { + const groups = byKey( + groupProcessRead({ rows, serverPid: SERVER, owners: tracked, includeAppProcesses: false }), + ); + expect(groups.threadlines).toEqual({ cpu: 5 + 1, pids: [SERVER, 101] }); + }); + + it("finds only the tree where processes carry no group (Windows)", () => { + const windowsRows = rows.map((entry) => ({ ...entry, pgid: null })); + const groups = byKey( + groupProcessRead({ + rows: windowsRows, + serverPid: SERVER, + owners: tracked, + includeAppProcesses: false, + }), + ); + expect(groups["thread-a"]?.pids).toEqual([200, 201, 210]); + expect(groups["thread-c"]).toBeUndefined(); + }); +}); + +describe("summarizeGroupedReads", () => { + const group = (threadId: string, cpuPercent: number): ProcessGroupRead => ({ + key: { kind: "thread", threadId }, + cpuPercent, + rssBytes: 1_000, + processCount: 1, + topProcesses: [{ pid: 1, command: "work", cpuPercent, rssBytes: 1_000 }], + }); + + it("averages over every read and keeps a group whose processes ended since", () => { + const summary = summarizeGroupedReads([ + { sampledAtMs: 0, groups: [group("busy-earlier", 400), group("steady", 100)] }, + { sampledAtMs: 5_000, groups: [group("steady", 100), group("spike", 600)] }, + ]); + expect( + summary.map((entry) => ({ + threadId: entry.kind === "thread" ? entry.threadId : null, + cpu: entry.cpuPercent, + avg: entry.avgCpuPercent, + processes: entry.processCount, + shown: entry.topProcesses.length, + })), + ).toEqual([ + { threadId: "spike", cpu: 600, avg: 300, processes: 1, shown: 1 }, + { threadId: "busy-earlier", cpu: 0, avg: 200, processes: 0, shown: 0 }, + { threadId: "steady", cpu: 100, avg: 100, processes: 1, shown: 1 }, + ]); + }); +}); + +describe("makeIntervalCpu", () => { + const timed = (pid: number, cpuPercent: number, cpuTimeMs?: number): ProcessRow => ({ + ...row(pid, SERVER, cpuPercent, `process ${pid}`), + cpuTimeMs, + }); + const cpu = (rows: ReadonlyArray) => rows.map((entry) => entry.cpuPercent); + + it("reads what each process used since the read before, not its lifetime average", () => { + const intervalCpu = makeIntervalCpu(); + // An hour-old agent that just filled a core: `ps` on Linux still says 2%. + expect(cpu(intervalCpu([timed(1, 2, 72_000), timed(2, 40, 500)], 10_000))).toEqual([2, 40]); + expect(cpu(intervalCpu([timed(1, 2, 77_000), timed(2, 40, 500)], 15_000))).toEqual([100, 0]); + // Two reads at almost the same moment: the earlier figure stands. + expect(cpu(intervalCpu([timed(1, 2, 77_400), timed(2, 40, 500)], 15_400))).toEqual([100, 0]); + expect(cpu(intervalCpu([timed(1, 2, 79_500), timed(2, 40, 1_500)], 20_000))).toEqual([50, 20]); + }); + + it("keeps the figure it was given when there is nothing to compare with", () => { + const intervalCpu = makeIntervalCpu(); + intervalCpu([timed(1, 9, 50_000), timed(2, 9, 50_000), timed(3, 9)], 10_000); + expect( + cpu( + intervalCpu( + [ + // Its number was reused: the total went backwards. + timed(1, 30, 200), + timed(2, 9, 55_000), + // Windows gives no total. + timed(3, 7), + // New since the last read. + timed(4, 60, 3_000), + ], + 15_000, + ), + ), + ).toEqual([30, 100, 7, 60]); + // The sampler stopped and started again: the old totals say nothing about now. + expect(cpu(intervalCpu([timed(2, 9, 999_000)], 500_000))).toEqual([9]); + }); +}); diff --git a/apps/server/src/diagnostics/ThreadCpuUsage.ts b/apps/server/src/diagnostics/ThreadCpuUsage.ts new file mode 100644 index 000000000..ed43d49b7 --- /dev/null +++ b/apps/server/src/diagnostics/ThreadCpuUsage.ts @@ -0,0 +1,403 @@ +// @effect-diagnostics nodeBuiltinImport:off +/** + * CPU by thread (Settings > Diagnostics): which thread's agents and terminals + * use the machine, next to Threadlines itself and provider processes that + * serve several threads. + * + * Ownership comes from what the server already tracks, never from command + * lines or folders: each driver's agent processes (ProviderService + * `listAgentProcesses`, from the ledger it keeps to confirm a thread stopped) + * and each terminal's shell (TerminalManager `listShells`). Every read the + * process monitor takes is grouped as it arrives, so a process that ran + * between two looks still counts where it ran. Nothing here reads on its own: + * reads happen only while someone keeps the monitor's sampler going. + */ +import * as OS from "node:os"; + +import type { + ProviderDriverKind, + ServerThreadCpuUsageGroup, + ServerThreadCpuUsageProcess, + ServerThreadCpuUsageResult, + ThreadId, +} from "@threadlines/contracts"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; + +import { ServerConfig } from "../config.ts"; +import { ProviderService } from "../provider/Services/ProviderService.ts"; +import { TerminalManager } from "../terminal/Services/Manager.ts"; +import { isDiagnosticsQueryProcess, type ProcessRow } from "./ProcessDiagnostics.ts"; +import { ProcessResourceMonitor, type ProcessTableRead } from "./ProcessResourceMonitor.ts"; + +/** How far back the average reaches. */ +export const THREAD_CPU_WINDOW_MS = 60_000; +const TOP_PROCESSES = 5; +const MAX_COMMAND_LENGTH = 300; +/** Reads further apart than this say too little about now to compare. */ +const MACHINE_CPU_MAX_GAP_MS = 15_000; + +/** What the server knows about who started which process. */ +export interface ProcessOwners { + readonly agents: ReadonlyArray<{ + readonly threadId: string; + readonly pid: number; + /** It leads a process group of its own (POSIX), so the whole group is its. */ + readonly ownGroup: boolean; + /** False once the process exited; its group may still run. */ + readonly alive: boolean; + }>; + /** Provider processes serving every thread of their provider (OpenCode's server). */ + readonly shared: ReadonlyArray<{ readonly provider: ProviderDriverKind; readonly pid: number }>; + readonly shells: ReadonlyArray<{ readonly threadId: string; readonly pid: number }>; +} + +export type ProcessGroupKey = + | { readonly kind: "thread"; readonly threadId: string } + | { readonly kind: "threadlines" } + | { readonly kind: "shared"; readonly provider: ProviderDriverKind }; + +export interface ProcessGroupRead { + readonly key: ProcessGroupKey; + readonly cpuPercent: number; + readonly rssBytes: number; + readonly processCount: number; + readonly topProcesses: ReadonlyArray; +} + +const keyId = (key: ProcessGroupKey) => + key.kind === "thread" + ? `thread:${key.threadId}` + : key.kind === "shared" + ? `shared:${key.provider}` + : "threadlines"; + +const THREADLINES: ProcessGroupKey = { kind: "threadlines" }; + +/** + * Puts each process of one read in exactly one group. + * + * - A thread: the tree under each of its agents and terminal shells, also + * where one left the server's tree (its parent died). When tracked + * processes nest, the nearest one wins. An agent that leads its own process + * group also gets the group's processes that left the tree (handed to pid 1 + * when their parent died), and keeps them after it exits itself, until + * another process holds its number. Rows carry no group on Windows, so + * there only the tree counts. + * - Shared: the tree under a provider process that serves several threads. + * - Threadlines: the server and the rest of its tree, plus, with + * `includeAppProcesses` (the desktop app started the server), the server's + * parent and that parent's other children. + * + * Processes that are none of these are left out. + */ +export function groupProcessRead(input: { + readonly rows: ReadonlyArray; + readonly serverPid: number; + readonly owners: ProcessOwners; + readonly includeAppProcesses: boolean; +}): ReadonlyArray { + const rowsByPid = new Map(); + const childrenByParent = new Map(); + for (const row of input.rows) { + if (isDiagnosticsQueryProcess(row, input.serverPid)) continue; + rowsByPid.set(row.pid, row); + const siblings = childrenByParent.get(row.ppid); + if (siblings) siblings.push(row); + else childrenByParent.set(row.ppid, [row]); + } + + const anchors = new Map(); + for (const shell of input.owners.shells) { + anchors.set(shell.pid, { kind: "thread", threadId: shell.threadId }); + } + for (const shared of input.owners.shared) { + anchors.set(shared.pid, { kind: "shared", provider: shared.provider }); + } + const groupOwners = new Map(); + for (const agent of input.owners.agents) { + const key: ProcessGroupKey = { kind: "thread", threadId: agent.threadId }; + // A pid whose process exited may already be someone else's. + if (agent.alive) anchors.set(agent.pid, key); + // Its group id can't be reused while any member lives, but once they are + // all gone a new process may take the number and lead a group of its + // own: a group whose leader's number is in use again is someone else's. + if (agent.ownGroup && (agent.alive || !rowsByPid.has(agent.pid))) { + groupOwners.set(agent.pid, key); + } + } + + const owner = new Map(); + /** Claims `root` and its unclaimed tree for `key`, nearest anchor first. */ + const claimTree = (root: ProcessRow, key: ProcessGroupKey) => { + const stack: Array<{ readonly row: ProcessRow; readonly key: ProcessGroupKey }> = [ + { row: root, key }, + ]; + while (stack.length > 0) { + const item = stack.pop()!; + if (owner.has(item.row.pid)) continue; + owner.set(item.row.pid, item.key); + for (const child of childrenByParent.get(item.row.pid) ?? []) { + stack.push({ row: child, key: anchors.get(child.pid) ?? item.key }); + } + } + }; + + const server = rowsByPid.get(input.serverPid); + if (server) claimTree(server, THREADLINES); + else { + // Windows reads only the server's tree; its own row may be missing. + for (const child of childrenByParent.get(input.serverPid) ?? []) { + claimTree(child, anchors.get(child.pid) ?? THREADLINES); + } + } + for (const [pid, key] of anchors) { + const row = rowsByPid.get(pid); + if (row && !owner.has(pid)) claimTree(row, key); + } + for (const row of rowsByPid.values()) { + if (owner.has(row.pid) || row.pgid === null) continue; + const key = groupOwners.get(row.pgid); + if (key) claimTree(row, key); + } + if (input.includeAppProcesses && server && server.ppid > 1) { + const app = rowsByPid.get(server.ppid); + if (app) claimTree(app, THREADLINES); + } + + const groups = new Map(); + for (const [pid, key] of owner) { + const row = rowsByPid.get(pid)!; + const id = keyId(key); + const group = groups.get(id); + if (group) group.rows.push(row); + else groups.set(id, { key, rows: [row] }); + } + return [...groups.values()].map(({ key, rows }) => ({ + key, + cpuPercent: rows.reduce((total, row) => total + row.cpuPercent, 0), + rssBytes: rows.reduce((total, row) => total + row.rssBytes, 0), + processCount: rows.length, + topProcesses: rows + .toSorted( + (left, right) => right.cpuPercent - left.cpuPercent || right.rssBytes - left.rssBytes, + ) + .slice(0, TOP_PROCESSES) + .map((row) => ({ + pid: row.pid, + command: row.command.trim().slice(0, MAX_COMMAND_LENGTH) || `pid ${row.pid}`, + cpuPercent: row.cpuPercent, + rssBytes: row.rssBytes, + })), + })); +} + +/** Two reads closer than this give an unsteady figure. */ +const INTERVAL_CPU_MIN_GAP_MS = 1_000; + +/** + * Turns each row's `cpuPercent` into what the process used since the read + * before, from its running total of CPU time (`cpuTimeMs`). That is the same + * measure on every system: left alone, `ps` reports a recent average on + * macOS and an average over the process's whole life on Linux, where an + * agent that sat idle for an hour and then filled a core would read 2%. + * + * A row keeps the figure `ps` gave it when there is nothing to compare with: + * a process seen for the first time, one whose total went backwards (its + * number was reused), a read after a gap, or a row without a total + * (Windows, where the figure is already made this way). Linux counts CPU + * time in whole seconds, so there one process's figure moves in steps of + * 100% divided by the seconds between reads; a group's sum and its average + * over the minute even that out. + */ +export function makeIntervalCpu(): ( + rows: ReadonlyArray, + atMs: number, +) => ReadonlyArray { + let previous = new Map< + number, + { readonly cpuTimeMs: number; readonly atMs: number; readonly cpuPercent: number | null } + >(); + return (rows, atMs) => { + const next: typeof previous = new Map(); + const result = rows.map((row) => { + if (row.cpuTimeMs === undefined) return row; + const last = previous.get(row.pid); + const gapMs = last ? atMs - last.atMs : 0; + if (!last || row.cpuTimeMs < last.cpuTimeMs || gapMs > MACHINE_CPU_MAX_GAP_MS) { + next.set(row.pid, { cpuTimeMs: row.cpuTimeMs, atMs, cpuPercent: null }); + return row; + } + if (gapMs < INTERVAL_CPU_MIN_GAP_MS) { + // Too soon after the last read to compare: its total and figure stand. + next.set(row.pid, last); + return last.cpuPercent === null ? row : { ...row, cpuPercent: last.cpuPercent }; + } + const cpuPercent = Math.round(((row.cpuTimeMs - last.cpuTimeMs) / gapMs) * 1000) / 10; + next.set(row.pid, { cpuTimeMs: row.cpuTimeMs, atMs, cpuPercent }); + return { ...row, cpuPercent }; + }); + previous = next; + return result; + }; +} + +export interface GroupedRead { + readonly sampledAtMs: number; + readonly groups: ReadonlyArray; +} + +/** + * Each group seen in the window, with its latest numbers and its average over + * every read in the window (a read it was absent from counts as 0), busiest + * on average first. + */ +export function summarizeGroupedReads( + reads: ReadonlyArray, +): ReadonlyArray { + const latestRead = reads.at(-1); + if (!latestRead) return []; + const summaries = new Map< + string, + { latestSeen: ProcessGroupRead; cpuTotal: number; inLatest: boolean } + >(); + for (const read of reads) { + for (const group of read.groups) { + const id = keyId(group.key); + const summary = summaries.get(id); + if (summary) { + summary.latestSeen = group; + summary.cpuTotal += group.cpuPercent; + } else { + summaries.set(id, { latestSeen: group, cpuTotal: group.cpuPercent, inLatest: false }); + } + } + } + for (const group of latestRead.groups) summaries.get(keyId(group.key))!.inLatest = true; + + return [...summaries.values()] + .map(({ latestSeen, cpuTotal, inLatest }): ServerThreadCpuUsageGroup => { + const numbers = { + cpuPercent: inLatest ? latestSeen.cpuPercent : 0, + avgCpuPercent: cpuTotal / reads.length, + rssBytes: inLatest ? Math.round(latestSeen.rssBytes) : 0, + processCount: inLatest ? latestSeen.processCount : 0, + topProcesses: inLatest ? latestSeen.topProcesses : [], + }; + const key = latestSeen.key; + return key.kind === "thread" + ? { kind: "thread", threadId: key.threadId as ThreadId, ...numbers } + : key.kind === "shared" + ? { kind: "shared", provider: key.provider, ...numbers } + : { kind: "threadlines", ...numbers }; + }) + .toSorted((left, right) => right.avgCpuPercent - left.avgCpuPercent); +} + +/** + * Busy time across all cores since the last read, in percent of one core. + * Null for the first read, and after a gap (the sampler stopped), whose + * average would say little about now. + */ +function makeMachineCpuMeter() { + let previous: { busy: number; total: number; atMs: number } | null = null; + return (atMs: number) => { + let busy = 0; + let total = 0; + for (const cpu of OS.cpus()) { + const { user, nice, sys, idle, irq } = cpu.times; + busy += user + nice + sys + irq; + total += user + nice + sys + idle + irq; + } + const last = previous; + previous = { busy, total, atMs }; + if (!last || atMs - last.atMs > MACHINE_CPU_MAX_GAP_MS || total <= last.total) return null; + const cores = Math.max(1, OS.cpus().length); + return ((busy - last.busy) / (total - last.total)) * cores * 100; + }; +} + +export interface ThreadCpuUsageShape { + /** Keeps the sampler going for another two minutes and answers what it has. */ + readonly read: Effect.Effect; +} + +export class ThreadCpuUsage extends Context.Service()( + "threadlines/diagnostics/ThreadCpuUsage", +) {} + +export const make = Effect.fn("makeThreadCpuUsage")(function* () { + const monitor = yield* ProcessResourceMonitor; + const providers = yield* ProviderService; + const terminals = yield* TerminalManager; + const config = yield* ServerConfig; + const includeAppProcesses = config.mode === "desktop"; + const machineCpu = makeMachineCpuMeter(); + const intervalCpu = makeIntervalCpu(); + let reads: ReadonlyArray = []; + let lastRows: ReadonlyArray | null = null; + let machineCpuPercent: number | null = null; + + const readOwners = Effect.gen(function* () { + const [listed, shells] = yield* Effect.all([ + providers.listAgentProcesses, + terminals.listShells, + ]); + return { + agents: listed.flatMap((owner) => (owner.kind === "thread" ? [owner] : [])), + shared: listed.flatMap((owner) => + owner.kind === "shared" ? [{ provider: owner.provider, pid: owner.pid }] : [], + ), + shells, + } satisfies ProcessOwners; + }); + + const observe = (read: ProcessTableRead) => + Effect.gen(function* () { + // Two requests at once can take one cached snapshot twice; counting it + // twice would weigh that moment double in the average. + if (read.rows === lastRows) return; + lastRows = read.rows; + const owners = yield* readOwners; + const groups = groupProcessRead({ + rows: intervalCpu(read.rows, read.sampledAtMs), + serverPid: read.serverPid, + owners, + includeAppProcesses, + }); + machineCpuPercent = machineCpu(read.sampledAtMs); + reads = [ + ...reads.filter((kept) => kept.sampledAtMs > read.sampledAtMs - THREAD_CPU_WINDOW_MS), + { sampledAtMs: read.sampledAtMs, groups }, + ]; + }); + yield* monitor.observeReads(observe); + + const read: ThreadCpuUsageShape["read"] = Effect.gen(function* () { + const error = yield* monitor.requestSamples.pipe( + Effect.as(Option.none<{ readonly message: string }>()), + Effect.catch((message) => Effect.succeed(Option.some({ message }))), + ); + const readAt = yield* DateTime.now; + const nowMs = DateTime.toEpochMillis(readAt); + // Reads older than the window are dropped here too, for a sampler that + // stopped and started again. + const recent = reads.filter((kept) => kept.sampledAtMs > nowMs - THREAD_CPU_WINDOW_MS); + return { + readAt, + windowMs: THREAD_CPU_WINDOW_MS, + // The same cores the machine figure counts. + coreCount: Math.max(1, OS.cpus().length), + machineCpuPercent: recent.length > 0 ? machineCpuPercent : null, + groups: summarizeGroupedReads(recent), + error, + } satisfies ServerThreadCpuUsageResult; + }); + + return ThreadCpuUsage.of({ read }); +}); + +export const layer = Layer.effect(ThreadCpuUsage, make()); diff --git a/apps/server/src/guide/topics.ts b/apps/server/src/guide/topics.ts index aa1180b99..f8c71366e 100644 --- a/apps/server/src/guide/topics.ts +++ b/apps/server/src/guide/topics.ts @@ -617,6 +617,16 @@ Settings that give agents more reach ({place:threads.agentThreads}, {place:threa What this promises: Threadlines doesn't put the key in the chat, its stored events, logs or other apps. What it doesn't: the agent runs as you on that computer, so it can still read the file, and if it prints the key, the model sees it. A key pasted into chat is already in the conversation; replace it if that matters. Answering needs an encrypted connection to that computer. Archiving, moving or deleting the thread removes its private files, and a saved row for a private file has Remove now. If the agent later prints a saved key, or a tool reads it, Threadlines replaces it with [NAME hidden] before it stores, logs or shows that text, and a row in the thread says so. An agent's message to another agent or thread, a page, or a command it asks you to run is refused while it holds one. The model may still have seen it, and the provider's own transcript keeps it; replace the key if that matters. Keys under 8 characters, or made only of letters or only of digits, aren't hidden. Your terminal, files in the project and voice audio aren't covered, and a key changed by hand in the env file is hidden from the next restart.`, + }, + { + id: "cpu-by-thread", + title: "Which thread is using the CPU", + keywords: ["cpu", "battery", "drain", "slow", "fan", "hot", "processes", "energy", "usage"], + body: `{place:diagnostics.cpuByThread} lists, busiest first, where this computer's CPU goes: one row per thread, one for Threadlines itself, and one for a provider whose single process serves all its threads. Its first line says how many of the computer's cores are busy in total. + +A thread's row counts its agents, what they started (test runs, browsers, builds, also ones left running after their parent ended) and its terminals. Rows are sorted by the average over the last minute, and CPU reads the way Activity Monitor shows it: 100% is one full core. Clicking a thread's name opens it; the arrow shows its busiest processes. + +It measures only while the section is on screen, so it can't show what ran before it was opened. OpenCode runs one shared process for every thread, so its work shows as one OpenCode row, not per thread. On Windows, a process that left its agent's process tree isn't found.`, }, { id: "updates", diff --git a/apps/server/src/orchestration/Layers/CheckpointReactor.test.ts b/apps/server/src/orchestration/Layers/CheckpointReactor.test.ts index c0e4f7331..a1daa1a74 100644 --- a/apps/server/src/orchestration/Layers/CheckpointReactor.test.ts +++ b/apps/server/src/orchestration/Layers/CheckpointReactor.test.ts @@ -179,6 +179,7 @@ function createProviderServiceHarness( agentSession: () => Effect.succeed(null), sendSubagentInput: () => unsupported(), deleteThread: () => unsupported(), + listAgentProcesses: Effect.succeed([]), get streamEvents() { return Stream.fromPubSub(runtimeEventPubSub); }, diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts index 5916d70e9..768103ca6 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts @@ -612,6 +612,7 @@ describe("ProviderCommandReactor", () => { agentSession: () => Effect.succeed(null), sendSubagentInput: () => unsupported(), deleteThread: () => unsupported(), + listAgentProcesses: Effect.succeed([]), get streamEvents() { return Stream.fromPubSub(runtimeEventPubSub); }, diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts index 1bc5c15a4..48de77782 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts @@ -185,6 +185,7 @@ function createProviderServiceHarness() { agentSession: () => Effect.succeed(null), sendSubagentInput: () => unsupported(), deleteThread: () => unsupported(), + listAgentProcesses: Effect.succeed([]), get streamEvents() { return Stream.fromPubSub(runtimeEventPubSub); }, diff --git a/apps/server/src/project/Layers/ProjectSetupScriptRunner.test.ts b/apps/server/src/project/Layers/ProjectSetupScriptRunner.test.ts index c6952b162..d0660936b 100644 --- a/apps/server/src/project/Layers/ProjectSetupScriptRunner.test.ts +++ b/apps/server/src/project/Layers/ProjectSetupScriptRunner.test.ts @@ -76,6 +76,7 @@ describe("ProjectSetupScriptRunner", () => { subscribe: () => Effect.succeed(() => undefined), startCommandRun: () => Effect.die(new Error("unused")), readCommandRunOutput: () => Effect.succeed(null), + listShells: Effect.succeed([]), subscribeCommandRuns: () => Effect.succeed(() => undefined), subscribeMetadata: () => Effect.succeed(() => undefined), subscribeThread: () => Effect.succeed(() => undefined), @@ -144,6 +145,7 @@ describe("ProjectSetupScriptRunner", () => { subscribe: () => Effect.succeed(() => undefined), startCommandRun: () => Effect.die(new Error("unused")), readCommandRunOutput: () => Effect.succeed(null), + listShells: Effect.succeed([]), subscribeCommandRuns: () => Effect.succeed(() => undefined), subscribeMetadata: () => Effect.succeed(() => undefined), subscribeThread: () => Effect.succeed(() => undefined), diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.ts b/apps/server/src/provider/Layers/ClaudeAdapter.ts index 3357298d5..1c7e5099f 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.ts @@ -11106,6 +11106,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( configDir: resolveClaudeConfigDir(claudeEnvironment, path), }), noteBeforeStop: processes.beforeStop, + listProcesses: processes.owners, confirmThreadStopped: (threadId) => [...unreportedProcesses].some((key) => parseSessionKey(key).threadId === threadId) ? Effect.succeed({ diff --git a/apps/server/src/provider/Layers/CodexAdapter.ts b/apps/server/src/provider/Layers/CodexAdapter.ts index 1db5ccff1..366a6e6c0 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.ts @@ -4679,6 +4679,7 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( }), noteBeforeStop: processes.beforeStop, confirmThreadStopped: processes.confirm, + listProcesses: processes.owners, get streamEvents() { return Stream.fromQueue(runtimeEventQueue); }, diff --git a/apps/server/src/provider/Layers/MuseAdapter.ts b/apps/server/src/provider/Layers/MuseAdapter.ts index 921f07b68..f3a161c09 100644 --- a/apps/server/src/provider/Layers/MuseAdapter.ts +++ b/apps/server/src/provider/Layers/MuseAdapter.ts @@ -2946,6 +2946,7 @@ export function makeMuseAdapter(options: MuseAdapterOptions) { stopAll, noteBeforeStop: processes.beforeStop, confirmThreadStopped: processes.confirm, + listProcesses: processes.owners, streamEvents: Stream.fromPubSub(runtimeEvents), } satisfies MuseAdapterShape; }); diff --git a/apps/server/src/provider/Layers/OpenCodeAdapter.test.ts b/apps/server/src/provider/Layers/OpenCodeAdapter.test.ts index 4a64a3c40..f4822d66e 100644 --- a/apps/server/src/provider/Layers/OpenCodeAdapter.test.ts +++ b/apps/server/src/provider/Layers/OpenCodeAdapter.test.ts @@ -113,6 +113,7 @@ const harnessWith = (overrides: Record unknown> = {}) => version: "2.0.22", external: false, exited: Effect.never, + pid: null, }, streamReady: Effect.void, }; @@ -122,6 +123,7 @@ const harnessWith = (overrides: Record unknown> = {}) => retire: Effect.void, withServer: (use) => Effect.scoped(use(active)), signals: Stream.fromPubSub(signals), + running: Effect.succeed(active.server), }; const adapter = yield* makeOpenCodeAdapter({ instanceId: INSTANCE_ID, diff --git a/apps/server/src/provider/Layers/OpenCodeAdapter.ts b/apps/server/src/provider/Layers/OpenCodeAdapter.ts index dd5c13243..811910940 100644 --- a/apps/server/src/provider/Layers/OpenCodeAdapter.ts +++ b/apps/server/src/provider/Layers/OpenCodeAdapter.ts @@ -3372,6 +3372,13 @@ export function makeOpenCodeAdapter(options: OpenCodeAdapterOptions) { readSubagentTranscript, stopAll, confirmThreadStopped, + // One server runs every thread's agent, so its work can't be split. + listProcesses: () => + manager.running.pipe( + Effect.map((server) => + server?.pid ? [{ kind: "shared" as const, pid: server.pid }] : [], + ), + ), streamEvents: Stream.fromPubSub(runtimeEvents), } satisfies OpenCodeAdapterShape; }); diff --git a/apps/server/src/provider/Layers/PiAdapter.ts b/apps/server/src/provider/Layers/PiAdapter.ts index b390afaf7..3d3d3008e 100644 --- a/apps/server/src/provider/Layers/PiAdapter.ts +++ b/apps/server/src/provider/Layers/PiAdapter.ts @@ -2439,6 +2439,7 @@ export function makePiAdapter(options: PiAdapterOptions) { importPiNativeSession({ ...input, agentDir: instanceAgentDir() }), noteBeforeStop: processes.beforeStop, confirmThreadStopped: processes.confirm, + listProcesses: processes.owners, streamEvents: Stream.fromPubSub(runtimeEvents), } satisfies PiAdapterShape; }); diff --git a/apps/server/src/provider/Layers/ProviderRegistry.test.ts b/apps/server/src/provider/Layers/ProviderRegistry.test.ts index 8e958ba21..5bb07927a 100644 --- a/apps/server/src/provider/Layers/ProviderRegistry.test.ts +++ b/apps/server/src/provider/Layers/ProviderRegistry.test.ts @@ -3299,6 +3299,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, ServerSettingsService.layerTest(), T retire: Effect.die("A turned-off OpenCode must not retire its server"), withServer: () => Effect.die("A turned-off OpenCode must not use its server"), signals: Stream.empty, + running: Effect.succeed(undefined), }; const check = (settings: Partial) => checkOpenCodeProviderStatus({ @@ -3341,6 +3342,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, ServerSettingsService.layerTest(), T retire: Effect.die("A broken OpenCode must not retire its server"), withServer: () => Effect.die("A broken OpenCode must not use its server"), signals: Stream.empty, + running: Effect.succeed(undefined), }, cwd: process.cwd(), environment: { PATH: "" }, diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts index 5692c8025..f8435d129 100644 --- a/apps/server/src/provider/Layers/ProviderService.ts +++ b/apps/server/src/provider/Layers/ProviderService.ts @@ -3381,6 +3381,20 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( agentSession, sendSubagentInput, deleteThread, + listAgentProcesses: getAdapterEntries.pipe( + Effect.flatMap((entries) => + Effect.forEach(entries, ([, adapter]) => + // One driver that can't answer leaves only its own processes out. + Effect.suspend(() => adapter.listProcesses?.() ?? Effect.succeed([])).pipe( + Effect.catchCause(() => Effect.succeed([])), + Effect.map((owners) => + owners.map((owner) => ({ ...owner, provider: adapter.provider })), + ), + ), + ), + ), + Effect.map((lists) => lists.flat()), + ), // Each access creates a fresh PubSub subscription so that multiple // consumers (ProviderRuntimeIngestion, CheckpointReactor, etc.) each // independently receive all runtime events. diff --git a/apps/server/src/provider/Layers/ProviderSessionReaper.test.ts b/apps/server/src/provider/Layers/ProviderSessionReaper.test.ts index 01609d8aa..a12374df9 100644 --- a/apps/server/src/provider/Layers/ProviderSessionReaper.test.ts +++ b/apps/server/src/provider/Layers/ProviderSessionReaper.test.ts @@ -274,6 +274,7 @@ describe("ProviderSessionReaper", () => { agentSession: () => Effect.succeed(null), sendSubagentInput: () => unsupported(), deleteThread: () => unsupported(), + listAgentProcesses: Effect.succeed([]), streamEvents: Stream.empty, }; diff --git a/apps/server/src/provider/Services/ProviderAdapter.ts b/apps/server/src/provider/Services/ProviderAdapter.ts index 6a8300108..d98f2bfbe 100644 --- a/apps/server/src/provider/Services/ProviderAdapter.ts +++ b/apps/server/src/provider/Services/ProviderAdapter.ts @@ -537,12 +537,36 @@ export interface ProviderAdapterShape { */ readonly noteBeforeStop?: (sessionKey: ThreadId) => Effect.Effect; + /** + * The agent processes this adapter runs right now, and whose they are, for + * the CPU-by-thread view in Settings > Diagnostics. Read from what the + * driver already tracks; never starts anything. Absent: the driver can't + * say, so its processes count as Threadlines' own. + */ + readonly listProcesses?: () => Effect.Effect>; + /** * Canonical runtime event stream emitted by this adapter. */ readonly streamEvents: Stream.Stream; } +/** + * One process a driver runs (ProviderAdapterShape.listProcesses): a thread's + * agent, or one serving every thread of the provider (OpenCode's server). + */ +export type ProviderProcessOwner = + | { + readonly kind: "thread"; + readonly threadId: ThreadId; + readonly pid: number; + /** It leads a process group of its own (POSIX), so the whole group is its. */ + readonly ownGroup: boolean; + /** False once the process itself exited while its group lives on. */ + readonly alive: boolean; + } + | { readonly kind: "shared"; readonly pid: number }; + /** One session a driver ran, as the session directory keeps it (confirmThreadStopped). */ export interface ProviderStoppedSessionHint { readonly sessionKey: ThreadId; diff --git a/apps/server/src/provider/Services/ProviderService.ts b/apps/server/src/provider/Services/ProviderService.ts index 47e07b99d..5f092bbf8 100644 --- a/apps/server/src/provider/Services/ProviderService.ts +++ b/apps/server/src/provider/Services/ProviderService.ts @@ -51,6 +51,7 @@ import type { WorkflowSaveCommandResult, RuntimeTaskId, RuntimeMode, + ProviderDriverKind, } from "@threadlines/contracts"; import * as Context from "effect/Context"; import type * as Effect from "effect/Effect"; @@ -67,6 +68,7 @@ import type { ProviderAdapterShape, ProviderStopConfirmation, } from "./ProviderAdapter.ts"; +import type { ProviderProcessOwner } from "./ProviderAdapter.ts"; import type { ProviderInstanceRoutingInfo } from "./ProviderAdapterRegistry.ts"; /** @@ -454,6 +456,13 @@ export interface ProviderServiceShape { readonly threadId: ThreadId; }) => Effect.Effect; + /** + * Every agent process the drivers run now, and whose it is, with the + * driver that runs it (ProviderAdapterShape.listProcesses). A driver that + * can't say contributes nothing. + */ + readonly listAgentProcesses: Effect.Effect>; + /** * Canonical provider runtime event stream. * @@ -462,6 +471,11 @@ export interface ProviderServiceShape { readonly streamEvents: Stream.Stream; } +/** One process from {@link ProviderServiceShape.listAgentProcesses}. */ +export type ProviderListedProcess = ProviderProcessOwner & { + readonly provider: ProviderDriverKind; +}; + /** * ProviderService - Service tag for provider orchestration. */ diff --git a/apps/server/src/provider/acp/AcpAdapter.ts b/apps/server/src/provider/acp/AcpAdapter.ts index e46e96484..625dc6ded 100644 --- a/apps/server/src/provider/acp/AcpAdapter.ts +++ b/apps/server/src/provider/acp/AcpAdapter.ts @@ -2647,6 +2647,7 @@ export function makeAcpAdapter( stopAll, noteBeforeStop: processes.beforeStop, confirmThreadStopped: processes.confirm, + listProcesses: processes.owners, streamEvents: Stream.fromPubSub(runtimeEventPubSub), } satisfies AcpAdapterShape; }); diff --git a/apps/server/src/provider/agentProcessLedger.ts b/apps/server/src/provider/agentProcessLedger.ts index 9ec1d41bc..51c0e1b63 100644 --- a/apps/server/src/provider/agentProcessLedger.ts +++ b/apps/server/src/provider/agentProcessLedger.ts @@ -26,7 +26,7 @@ import * as Option from "effect/Option"; import type * as Scope from "effect/Scope"; -import type { ProviderStopConfirmation } from "./Services/ProviderAdapter.ts"; +import type { ProviderProcessOwner, ProviderStopConfirmation } from "./Services/ProviderAdapter.ts"; /** An agent process a driver started. */ export interface TrackedAgentProcess { @@ -123,6 +123,11 @@ export interface AgentProcessLedger { readonly beforeStop: (sessionKey: ThreadId) => Effect.Effect; /** Everything tracked for `threadId`'s session keys is gone, or why that can't be said. */ readonly confirm: (threadId: ThreadId) => Effect.Effect; + /** + * Every process tracked now, by thread (ProviderAdapterShape.listProcesses). + * One whose own process exited stays listed while its group may live on. + */ + readonly owners: () => Effect.Effect>; } export interface AgentProcessLedgerOptions { @@ -288,5 +293,16 @@ export const makeAgentProcessLedger = ( : ({ confirmed: false, reason: all.join("; ") } as const); }); - return { track, beforeStop, confirm } satisfies AgentProcessLedger; + const owners: AgentProcessLedger["owners"] = () => + Effect.sync(() => + [...entries].map((entry): ProviderProcessOwner => ({ + kind: "thread", + threadId: parseSessionKey(entry.sessionKey).threadId, + pid: entry.pid, + ownGroup: posix && entry.ownGroup, + alive: !entry.leaderGone, + })), + ); + + return { track, beforeStop, confirm, owners } satisfies AgentProcessLedger; }); diff --git a/apps/server/src/provider/opencode/OpenCodeServer.ts b/apps/server/src/provider/opencode/OpenCodeServer.ts index d9a6e2a62..667ef2ca3 100644 --- a/apps/server/src/provider/opencode/OpenCodeServer.ts +++ b/apps/server/src/provider/opencode/OpenCodeServer.ts @@ -59,6 +59,8 @@ export interface OpenCodeServer { readonly external: boolean; /** Resolves when an owned server process exits, for any reason. Never resolves for external servers. */ readonly exited: Effect.Effect; + /** The server process, when Threadlines started it; null for an external server. */ + readonly pid: number | null; } /** @@ -285,6 +287,7 @@ export const startOpenCodeServer = (input: { version, external: false, exited: Deferred.await(exited).pipe(Effect.asVoid), + pid: Number(child.pid), } satisfies OpenCodeServer; }); @@ -318,7 +321,7 @@ export const connectOpenCodeServer = (input: { ), ); yield* requireSupportedVersion(version, operation); - return { url: input.url, client, version, external: true, exited: Effect.never }; + return { url: input.url, client, version, external: true, exited: Effect.never, pid: null }; }); /** `opencode --version`, without starting a server. */ diff --git a/apps/server/src/provider/opencode/OpenCodeServerManager.ts b/apps/server/src/provider/opencode/OpenCodeServerManager.ts index ea38d75a0..b5a8e3fcc 100644 --- a/apps/server/src/provider/opencode/OpenCodeServerManager.ts +++ b/apps/server/src/provider/opencode/OpenCodeServerManager.ts @@ -81,6 +81,8 @@ export interface OpenCodeServerManagerShape { ) => Effect.Effect>; /** Every signal from every generation, for as long as the manager lives. */ readonly signals: Stream.Stream; + /** The server running now, if any. Never starts one. */ + readonly running: Effect.Effect; } export interface OpenCodeServerManagerOptions { @@ -371,5 +373,6 @@ export const makeOpenCodeServerManager = ( retire, withServer, signals: Stream.fromPubSub(signals), + running: Ref.get(currentRef).pipe(Effect.map((current) => current?.active.server)), } satisfies OpenCodeServerManagerShape; }); diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 8308e51fb..ff1fbac5d 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -202,6 +202,7 @@ import { ServerAuthLive } from "./auth/Layers/ServerAuth.ts"; import { SessionCredentialService } from "./auth/Services/SessionCredentialService.ts"; import * as ProcessDiagnostics from "./diagnostics/ProcessDiagnostics.ts"; import * as ProcessResourceMonitor from "./diagnostics/ProcessResourceMonitor.ts"; +import * as ThreadCpuUsage from "./diagnostics/ThreadCpuUsage.ts"; import * as StallMonitor from "./observability/StallMonitor.ts"; import * as TraceDiagnostics from "./diagnostics/TraceDiagnostics.ts"; import * as Data from "effect/Data"; @@ -961,6 +962,16 @@ const buildAppUnderTest = (options?: { Layer.provide( Layer.mergeAll( Layer.mock(StallMonitor.StallMonitor)({ stalls: Stream.empty }), + Layer.mock(ThreadCpuUsage.ThreadCpuUsage)({ + read: Effect.succeed({ + readAt: TEST_EPOCH, + windowMs: 60_000, + coreCount: 1, + machineCpuPercent: null, + groups: [], + error: Option.none(), + }), + }), Layer.mock(ProcessResourceMonitor.ProcessResourceMonitor)({ readCurrent: Effect.succeed({ serverPid: process.pid, diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 2ca612a9c..257f26382 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -195,6 +195,7 @@ import { formatHostForUrl, isWildcardHost } from "./startupAccess.ts"; import { ServerAuthLive } from "./auth/Layers/ServerAuth.ts"; import * as ProcessDiagnostics from "./diagnostics/ProcessDiagnostics.ts"; import * as ProcessResourceMonitor from "./diagnostics/ProcessResourceMonitor.ts"; +import * as ThreadCpuUsage from "./diagnostics/ThreadCpuUsage.ts"; import * as StallMonitor from "./observability/StallMonitor.ts"; import * as TraceDiagnostics from "./diagnostics/TraceDiagnostics.ts"; import { OrchestrationLayerLive } from "./orchestration/runtimeLayer.ts"; @@ -609,8 +610,11 @@ const ProviderRuntimeLayerLive = Layer.mergeAll( const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe( // Core Services. ThreadBootstrap sets threads up: the reactor starts child // threads with it, and the WebSocket routes run first-send bootstraps - // through the same instance. - Layer.provideMerge(Layer.mergeAll(CheckpointingLayerLive, ThreadBootstrapLive)), + // through the same instance. CPU by thread (Settings > Diagnostics) needs + // the providers' and the terminals' processes, so it sits above both. + Layer.provideMerge( + Layer.mergeAll(CheckpointingLayerLive, ThreadBootstrapLive, ThreadCpuUsage.layer), + ), Layer.provideMerge( Layer.mergeAll(SourceControlProviderRegistryLayerLive, PullRequestServiceLayerLive), ), diff --git a/apps/server/src/terminal/Layers/Manager.ts b/apps/server/src/terminal/Layers/Manager.ts index f884349be..e4f5ba0e3 100644 --- a/apps/server/src/terminal/Layers/Manager.ts +++ b/apps/server/src/terminal/Layers/Manager.ts @@ -4841,6 +4841,15 @@ export const makeTerminalManagerWithOptions = Effect.fn("makeTerminalManagerWith resolvePortInUse, startCommandRun, readCommandRunOutput, + listShells: readManagerState.pipe( + Effect.map((state) => + [...state.sessions.values()].flatMap((session) => + session.status === "running" && session.pid !== null + ? [{ threadId: session.threadId, pid: session.pid }] + : [], + ), + ), + ), subscribeCommandRuns: (listener) => Effect.sync(() => { commandRunListeners.add(listener); diff --git a/apps/server/src/terminal/Services/Manager.ts b/apps/server/src/terminal/Services/Manager.ts index d6adb237b..e2149c3b4 100644 --- a/apps/server/src/terminal/Services/Manager.ts +++ b/apps/server/src/terminal/Services/Manager.ts @@ -276,6 +276,14 @@ export interface TerminalManagerShape { */ readonly readCommandRunOutput: (runId: string) => Effect.Effect; + /** + * The shell process of every running terminal, by thread, for the + * CPU-by-thread view in Settings > Diagnostics. Reads memory only. + */ + readonly listShells: Effect.Effect< + ReadonlyArray<{ readonly threadId: string; readonly pid: number }> + >; + /** Subscribe to run updates. Returns an unsubscribe function. */ readonly subscribeCommandRuns: ( listener: (update: TerminalCommandRunUpdate) => Effect.Effect, diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 3fad00147..c875b714b 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -203,6 +203,7 @@ import { SecretScrubber } from "./secrets/SecretScrubber.ts"; import { makeWorkflowRunRequests } from "./orchestration/workflowRunRequests.ts"; import * as ProcessDiagnostics from "./diagnostics/ProcessDiagnostics.ts"; import * as ProcessResourceMonitor from "./diagnostics/ProcessResourceMonitor.ts"; +import { ThreadCpuUsage } from "./diagnostics/ThreadCpuUsage.ts"; import * as StallMonitor from "./observability/StallMonitor.ts"; import * as TraceDiagnostics from "./diagnostics/TraceDiagnostics.ts"; import * as SourceControlDiscoveryLayer from "./sourceControl/SourceControlDiscovery.ts"; @@ -1017,6 +1018,7 @@ const makeCoreRpcLayer = ( providerService, }); const processResourceMonitor = yield* ProcessResourceMonitor.ProcessResourceMonitor; + const threadCpuUsage = yield* ThreadCpuUsage; const stallMonitor = yield* StallMonitor.StallMonitor; const loadAuthAccessSnapshot = () => @@ -2196,6 +2198,10 @@ const makeCoreRpcLayer = ( "rpc.aggregate": "server", }, ), + [WS_METHODS.serverGetThreadCpuUsage]: (_input) => + observeRpcEffect(WS_METHODS.serverGetThreadCpuUsage, threadCpuUsage.read, { + "rpc.aggregate": "server", + }), [WS_METHODS.serverSignalProcess]: (input) => observeRpcEffect(WS_METHODS.serverSignalProcess, processDiagnostics.signal(input), { "rpc.aggregate": "server", diff --git a/apps/web/src/components/settings/DiagnosticsSettings.tsx b/apps/web/src/components/settings/DiagnosticsSettings.tsx index 55bb54e32..6d863695d 100644 --- a/apps/web/src/components/settings/DiagnosticsSettings.tsx +++ b/apps/web/src/components/settings/DiagnosticsSettings.tsx @@ -42,6 +42,7 @@ import { Skeleton } from "../ui/skeleton"; import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip"; import { failureNotice, toastManager } from "../ui/toast"; import { SettingsPageContainer, SettingsPageHeader, SettingsSection } from "./settingsLayout"; +import { ThreadCpuUsageSection } from "./ThreadCpuUsageSection"; const NUMBER_FORMAT = new Intl.NumberFormat(); @@ -1151,6 +1152,7 @@ export function DiagnosticsSettingsPanel() { Loading diagnostics ) : null} + { getProcessResourceHistory: vi .fn() .mockResolvedValue(createEmptyProcessResourceHistoryResult()), + getThreadCpuUsage: vi.fn().mockReturnValue(new Promise(() => {})), getTraceDiagnostics: vi.fn().mockResolvedValue({ traceFilePath: "/repo/project/.threadlines/traces.jsonl", scannedFilePaths: ["/repo/project/.threadlines/traces.jsonl"], @@ -2650,6 +2653,76 @@ describe("GeneralSettingsPanel observability", () => { expect(openInEditor).toHaveBeenCalledWith("/repo/project/.threadlines/logs", "cursor"); }); + it("orders CPU by thread by the last minute's average, not by each refresh", async () => { + const focus = vi.spyOn(document, "hasFocus").mockReturnValue(true); + onTestFinished(() => focus.mockRestore()); + const read = ( + groups: ReadonlyArray<{ + readonly id: "threadlines" | "opencode" | "thread"; + readonly avg: number; + readonly now: number; + }>, + ): ServerThreadCpuUsageResult => ({ + readAt: makeUtc("2036-04-07T00:00:00.000Z"), + windowMs: 60_000, + coreCount: 18, + machineCpuPercent: 940, + groups: groups.map(({ id, avg, now }) => { + const numbers = { + cpuPercent: now, + avgCpuPercent: avg, + rssBytes: 64 * 1024 * 1024, + processCount: 2, + topProcesses: [ + { pid: 42, command: "/usr/local/bin/node vitest run", cpuPercent: now, rssBytes: 1024 }, + ], + }; + return id === "thread" + ? { kind: "thread", threadId: ThreadId.make("thread-gone"), ...numbers } + : id === "opencode" + ? { kind: "shared", provider: ProviderDriverKind.make("opencode"), ...numbers } + : { kind: "threadlines", ...numbers }; + }), + error: Option.none(), + }); + const getThreadCpuUsage = vi + .fn() + .mockResolvedValueOnce( + read([ + { id: "threadlines", avg: 50, now: 10 }, + { id: "thread", avg: 300, now: 5 }, + { id: "opencode", avg: 120, now: 400 }, + ]), + ) + .mockResolvedValue( + read([ + { id: "threadlines", avg: 60, now: 900 }, + { id: "thread", avg: 290, now: 0 }, + { id: "opencode", avg: 130, now: 20 }, + ]), + ); + window.nativeApi = { server: { getThreadCpuUsage } } as unknown as LocalApi; + + mounted = await renderWithTestRouter( + + + , + ); + + const order = async () => + (await page.getByTestId("thread-cpu-row").all()).map((row) => + row.element().getAttribute("data-group"), + ); + await expect.element(page.getByText("9.4 of 18 cores busy", { exact: false })).toBeVisible(); + await expect.element(page.getByText("OpenCode", { exact: true })).toBeVisible(); + await expect.element(page.getByText("A thread not listed here")).toBeVisible(); + expect(await order()).toEqual(["thread:thread-gone", "shared:opencode", "threadlines"]); + + await vi.waitFor(() => expect(getThreadCpuUsage.mock.calls.length).toBeGreaterThanOrEqual(2)); + await expect.element(page.getByText("900%", { exact: true })).toBeVisible(); + expect(await order()).toEqual(["thread:thread-gone", "shared:opencode", "threadlines"]); + }); + it("uses native-layout skeletons while diagnostics are initially loading", async () => { const pendingDiagnostics = new Promise(() => {}); window.nativeApi = { @@ -2661,6 +2734,7 @@ describe("GeneralSettingsPanel observability", () => { server: { getProcessDiagnostics: vi.fn().mockReturnValue(pendingDiagnostics), getProcessResourceHistory: vi.fn().mockReturnValue(pendingDiagnostics), + getThreadCpuUsage: vi.fn().mockReturnValue(pendingDiagnostics), getTraceDiagnostics: vi.fn().mockReturnValue(pendingDiagnostics), }, } as unknown as LocalApi; diff --git a/apps/web/src/components/settings/ThreadCpuUsageSection.tsx b/apps/web/src/components/settings/ThreadCpuUsageSection.tsx new file mode 100644 index 000000000..f1df8aa37 --- /dev/null +++ b/apps/web/src/components/settings/ThreadCpuUsageSection.tsx @@ -0,0 +1,351 @@ +import { scopeThreadRef } from "@threadlines/client-runtime"; +import { + TERMINALS_PAGE_THREAD_ID, + type EnvironmentId, + type ServerThreadCpuUsageGroup, + type ServerThreadCpuUsageResult, +} from "@threadlines/contracts"; +import { useNavigate } from "@tanstack/react-router"; +import * as Option from "effect/Option"; +import { ChevronDownIcon, ChevronRightIcon } from "lucide-react"; +import { useEffect, useState } from "react"; + +import { usePrimaryEnvironmentId } from "../../environments/primary/context"; +import { formatBytes } from "../../lib/formatBytes"; +import { usePageInUse } from "../../lib/pageAttention"; +import { ensureLocalApi } from "../../localApi"; +import { selectProjectByRef, selectSidebarThreadSummaryByRef, useStore } from "../../store"; +import { buildThreadRouteParams } from "../../threadRoutes"; +import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip"; +import { getDriverOption } from "./providerDriverMeta"; +import { SettingsSection } from "./settingsLayout"; + +/** How often the section asks while it is on screen and the window is in use. */ +const REFRESH_MS = 5_000; + +const groupId = (group: ServerThreadCpuUsageGroup) => + group.kind === "thread" + ? `thread:${group.threadId}` + : group.kind === "shared" + ? `shared:${group.provider}` + : "threadlines"; + +/** + * Busiest over the last minute first. The minute's average moves slowly, so + * rows keep their places between refreshes unless one really overtakes + * another; the latest instant value never reorders them. + */ +export function orderThreadCpuGroups( + groups: ReadonlyArray, +): ReadonlyArray { + return groups.toSorted( + (left, right) => + right.avgCpuPercent - left.avgCpuPercent || groupId(left).localeCompare(groupId(right)), + ); +} + +/** The command's program name: `/usr/bin/node build.js` reads `node`. */ +function programName(command: string): string { + const first = + command + .trim() + .split(/\s+/)[0] + ?.replace(/^['"]|['"]$/g, "") ?? command; + return first.split(/[\\/]/).findLast((segment) => segment.length > 0) ?? first; +} + +const formatCpu = (percent: number) => `${Math.round(percent)}%`; +const formatCores = (percent: number) => (percent / 100).toFixed(1); + +function useThreadCpuUsage(active: boolean, refreshMs: number) { + const [state, setState] = useState<{ + readonly data: ServerThreadCpuUsageResult | null; + readonly error: string | null; + }>({ data: null, error: null }); + useEffect(() => { + if (!active) return; + let cancelled = false; + let timer: ReturnType | undefined; + const read = () => { + ensureLocalApi() + .server.getThreadCpuUsage() + .then( + (data) => { + if (!cancelled) setState({ data, error: null }); + }, + (error: unknown) => { + if (cancelled) return; + setState((current) => ({ + data: current.data, + error: error instanceof Error ? error.message : "Couldn't read CPU use.", + })); + }, + ) + .finally(() => { + if (!cancelled) timer = setTimeout(read, refreshMs); + }); + }; + read(); + return () => { + cancelled = true; + if (timer !== undefined) clearTimeout(timer); + }; + }, [active, refreshMs]); + return state; +} + +/** + * Whether `element` is on screen: false until the browser says so, true + * where it can't tell. + */ +function useOnScreen(element: HTMLElement | null): boolean { + const [onScreen, setOnScreen] = useState(() => typeof IntersectionObserver === "undefined"); + useEffect(() => { + if (!element || typeof IntersectionObserver === "undefined") return; + const observer = new IntersectionObserver((entries) => { + const entry = entries.at(-1); + if (entry) setOnScreen(entry.isIntersecting); + }); + observer.observe(element); + return () => observer.disconnect(); + }, [element]); + return onScreen; +} + +function GroupName({ + group, + environmentId, +}: { + group: ServerThreadCpuUsageGroup; + environmentId: EnvironmentId | null; +}) { + const navigate = useNavigate(); + const threadRef = + group.kind === "thread" && environmentId !== null + ? scopeThreadRef(environmentId, group.threadId) + : null; + const thread = useStore((state) => selectSidebarThreadSummaryByRef(state, threadRef)); + const project = useStore((state) => + thread + ? selectProjectByRef(state, { + environmentId: thread.environmentId, + projectId: thread.projectId, + }) + : undefined, + ); + + if (group.kind === "threadlines") { + return Threadlines; + } + if (group.kind === "shared") { + const name = getDriverOption(group.provider)?.label ?? group.provider; + return ( + + {name} + Shared by all its threads + + ); + } + if (group.threadId === TERMINALS_PAGE_THREAD_ID) { + return Terminals page; + } + if (!thread || !threadRef) { + return ( + A thread not listed here + ); + } + return ( + + + {project ? {project.name} : null} + + ); +} + +function GroupRow({ + group, + environmentId, + expanded, + onToggle, +}: { + group: ServerThreadCpuUsageGroup; + environmentId: EnvironmentId | null; + expanded: boolean; + onToggle: () => void; +}) { + const busiest = group.topProcesses[0]; + const Chevron = expanded ? ChevronDownIcon : ChevronRightIcon; + return ( +
  • +
    + +
    + + {busiest ? ( + + {programName(busiest.command)} + {group.processCount > 1 ? ` + ${group.processCount - 1} more` : ""} + + ) : ( + Ended + )} +
    + + {formatCpu(group.avgCpuPercent)} + + + {formatCpu(group.cpuPercent)} + + + {group.rssBytes > 0 ? formatBytes(group.rssBytes) : "–"} + +
    + {expanded ? ( +
      + {group.topProcesses.map((process) => ( +
    • + + } + /> + + {process.command} (pid {process.pid}) + + +
    • + ))} +
    + ) : null} +
  • + ); +} + +/** + * CPU by thread, at the top of Settings > Diagnostics: where this computer's + * CPU goes, by thread. Asks the server every few seconds only while it is on + * screen and someone is using the window, so the server's sampler winds down + * when nobody looks. + */ +export function ThreadCpuUsageSection({ + refreshMs = REFRESH_MS, +}: { + /** How often to ask; tests shorten it. */ + refreshMs?: number; +} = {}) { + const [element, setElement] = useState(null); + const onScreen = useOnScreen(element); + // In use, not just in front: a window left focused in front of an empty + // chair would otherwise keep the server's sampler going for hours. + const inUse = usePageInUse(); + const { data, error } = useThreadCpuUsage(onScreen && inUse, refreshMs); + const environmentId = usePrimaryEnvironmentId(); + const [expanded, setExpanded] = useState>(() => new Set()); + + const groups = data ? orderThreadCpuGroups(data.groups) : []; + const ownCores = groups.reduce((total, group) => total + group.cpuPercent, 0); + const serverError = data ? (Option.getOrNull(data.error)?.message ?? null) : null; + const summary = data + ? data.machineCpuPercent === null + ? `Measuring. This computer has ${data.coreCount} cores.` + : `${formatCores(data.machineCpuPercent)} of ${data.coreCount} cores busy, ${formatCores(ownCores)} of them from Threadlines and its agents.` + : null; + + return ( + +
    + {error !== null || serverError !== null ? ( +
    + {serverError ?? error} +
    + ) : null} + {data === null ? ( +
    + {error === null ? "Reading processes…" : null} +
    + ) : groups.length === 0 ? ( +
    + Nothing measured yet. +
    + ) : ( +
    +
    + } + /> + + Average over the last minute. 100% is one full core. + + + Now + Memory +
    +
      + {groups.map((group) => { + const id = groupId(group); + return ( + + setExpanded((current) => { + const next = new Set(current); + if (!next.delete(id)) next.add(id); + return next; + }) + } + /> + ); + })} +
    +
    + )} +
    +
    + ); +} diff --git a/apps/web/src/localApi.ts b/apps/web/src/localApi.ts index 33b841687..a23192e60 100644 --- a/apps/web/src/localApi.ts +++ b/apps/web/src/localApi.ts @@ -173,6 +173,7 @@ function createBrowserLocalApi(resolveRpcClient?: () => WsRpcClient | null): Loc getProcessDiagnostics: () => withServer((server) => server.getProcessDiagnostics()), getProcessResourceHistory: (input) => withServer((server) => server.getProcessResourceHistory(input)), + getThreadCpuUsage: () => withServer((server) => server.getThreadCpuUsage()), signalProcess: (input) => withServer((server) => server.signalProcess(input)), resolveBackgroundRuns: (input) => withServer((server) => server.resolveBackgroundRuns(input)), stopBackgroundRun: (input) => withServer((server) => server.stopBackgroundRun(input)), diff --git a/apps/web/src/rpc/wsRpcClient.ts b/apps/web/src/rpc/wsRpcClient.ts index c462bf261..8320a7d16 100644 --- a/apps/web/src/rpc/wsRpcClient.ts +++ b/apps/web/src/rpc/wsRpcClient.ts @@ -325,6 +325,7 @@ export interface WsRpcClient { readonly getProcessResourceHistory: RpcUnaryMethod< typeof WS_METHODS.serverGetProcessResourceHistory >; + readonly getThreadCpuUsage: RpcUnaryNoArgMethod; readonly signalProcess: RpcUnaryMethod; readonly resolveBackgroundRuns: RpcUnaryMethod; readonly readBackgroundRunOutput: RpcUnaryMethod< @@ -1089,6 +1090,10 @@ export function createWsRpcClient(transport: WsTransport): WsRpcClient { Effect.withTracerEnabled(false), ), ), + getThreadCpuUsage: () => + transport.request((client) => + client[WS_METHODS.serverGetThreadCpuUsage]({}).pipe(Effect.withTracerEnabled(false)), + ), signalProcess: (input) => transport.request((client) => client[WS_METHODS.serverSignalProcess](input).pipe(Effect.withTracerEnabled(false)), diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index 950152fc1..35aa801bc 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -167,6 +167,7 @@ import type { ServerProcessDiagnosticsResult, ServerProcessResourceHistoryInput, ServerProcessResourceHistoryResult, + ServerThreadCpuUsageResult, ServerProviderDataSharingSetInput, ServerProviderDataSharingSetResult, ServerProviderRateLimitResetCreditConsumeInput, @@ -1809,6 +1810,8 @@ export interface LocalApi { getProcessResourceHistory: ( input: ServerProcessResourceHistoryInput, ) => Promise; + /** CPU by thread over the last minute; keeps the server's sampler going. */ + getThreadCpuUsage: () => Promise; signalProcess: (input: ServerSignalProcessInput) => Promise; resolveBackgroundRuns: ( input: ServerResolveBackgroundRunsInput, diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index a5c7f1421..4fe926a9f 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -512,6 +512,7 @@ import { ServerProcessDiagnosticsResult, ServerProcessResourceHistoryInput, ServerProcessResourceHistoryResult, + ServerThreadCpuUsageResult, ServerSignalProcessInput, ServerSignalProcessResult, ServerResolveBackgroundRunsInput, @@ -703,6 +704,7 @@ export const WS_METHODS = { serverGetTraceDiagnostics: "server.getTraceDiagnostics", serverGetProcessDiagnostics: "server.getProcessDiagnostics", serverGetProcessResourceHistory: "server.getProcessResourceHistory", + serverGetThreadCpuUsage: "server.getThreadCpuUsage", serverSignalProcess: "server.signalProcess", serverResolveBackgroundRuns: "server.resolveBackgroundRuns", serverStopBackgroundRun: "server.stopBackgroundRun", @@ -1094,6 +1096,11 @@ export const WsServerGetProcessResourceHistoryRpc = Rpc.make( }, ); +export const WsServerGetThreadCpuUsageRpc = Rpc.make(WS_METHODS.serverGetThreadCpuUsage, { + payload: Schema.Struct({}), + success: ServerThreadCpuUsageResult, +}); + export const WsServerSignalProcessRpc = Rpc.make(WS_METHODS.serverSignalProcess, { payload: ServerSignalProcessInput, success: ServerSignalProcessResult, @@ -2941,6 +2948,7 @@ export const WsCoreRpcGroup = RpcGroup.make( WsServerGetTraceDiagnosticsRpc, WsServerGetProcessDiagnosticsRpc, WsServerGetProcessResourceHistoryRpc, + WsServerGetThreadCpuUsageRpc, WsServerSignalProcessRpc, WsServerResolveBackgroundRunsRpc, WsServerStopBackgroundRunRpc, diff --git a/packages/contracts/src/server.ts b/packages/contracts/src/server.ts index da6eac768..df5cc8fe1 100644 --- a/packages/contracts/src/server.ts +++ b/packages/contracts/src/server.ts @@ -805,6 +805,62 @@ export const ServerProcessResourceHistoryResult = Schema.Struct({ }); export type ServerProcessResourceHistoryResult = typeof ServerProcessResourceHistoryResult.Type; +/** One process inside a {@link ServerThreadCpuUsageGroup}, from the latest read. */ +export const ServerThreadCpuUsageProcess = Schema.Struct({ + pid: PositiveInt, + command: TrimmedNonEmptyString, + /** Percent of one core, as Activity Monitor shows it. */ + cpuPercent: Schema.Number, + rssBytes: NonNegativeInt, +}); +export type ServerThreadCpuUsageProcess = typeof ServerThreadCpuUsageProcess.Type; + +const ServerThreadCpuUsageNumbers = { + /** Percent of one core in the latest read; 0 once all its processes ended. */ + cpuPercent: Schema.Number, + /** Percent of one core averaged over every read in the window. */ + avgCpuPercent: Schema.Number, + rssBytes: NonNegativeInt, + processCount: NonNegativeInt, + /** Its busiest processes in the latest read it had any, busiest first. */ + topProcesses: Schema.Array(ServerThreadCpuUsageProcess), +}; + +/** + * Where a share of the machine's CPU goes: a thread's agents and terminals, + * Threadlines itself, or a provider process that serves several threads. + */ +export const ServerThreadCpuUsageGroup = Schema.Union([ + Schema.Struct({ + kind: Schema.Literal("thread"), + threadId: ThreadId, + ...ServerThreadCpuUsageNumbers, + }), + Schema.Struct({ kind: Schema.Literal("threadlines"), ...ServerThreadCpuUsageNumbers }), + Schema.Struct({ + kind: Schema.Literal("shared"), + provider: ProviderDriverKind, + ...ServerThreadCpuUsageNumbers, + }), +]); +export type ServerThreadCpuUsageGroup = typeof ServerThreadCpuUsageGroup.Type; + +export const ServerThreadCpuUsageResult = Schema.Struct({ + readAt: Schema.DateTimeUtc, + /** How far back `avgCpuPercent` reaches. */ + windowMs: NonNegativeInt, + coreCount: PositiveInt, + /** The whole machine's use, in percent of one core; null before two reads. */ + machineCpuPercent: Schema.NullOr(Schema.Number), + groups: Schema.Array(ServerThreadCpuUsageGroup), + error: Schema.Option( + Schema.Struct({ + message: TrimmedNonEmptyString, + }), + ), +}); +export type ServerThreadCpuUsageResult = typeof ServerThreadCpuUsageResult.Type; + export const ServerSignalProcessInput = Schema.Struct({ pid: PositiveInt, signal: ServerProcessSignal, diff --git a/packages/shared/src/appPlaces.ts b/packages/shared/src/appPlaces.ts index fd39f78a3..f30065de9 100644 --- a/packages/shared/src/appPlaces.ts +++ b/packages/shared/src/appPlaces.ts @@ -505,7 +505,7 @@ const PAGES = [ id: "settings.diagnostics", kind: "page", title: "Diagnostics", - about: "Slow requests, live processes and recent failures on this computer.", + about: "CPU by thread, slow requests, live processes and recent failures on this computer.", route: { to: "/settings/diagnostics" }, }, { @@ -566,6 +566,16 @@ const PAGES = [ // ----- sections -------------------------------------------------------------------- const SECTIONS = [ + // Diagnostics + { + id: "diagnostics.cpuByThread", + kind: "section", + parent: "settings.diagnostics", + title: "CPU by thread", + about: + "Which threads' agents and terminals are using this computer's CPU now and over the last minute, next to Threadlines itself.", + keywords: ["cpu", "battery", "slow", "fan", "hot", "processes", "usage", "energy"], + }, // General { id: "general.alerts",