From ca0ff85ecc25c18508787991bfbefb3e4ca0476b Mon Sep 17 00:00:00 2001 From: Gabe Debes Date: Sun, 18 Oct 2026 09:29:51 -0700 Subject: [PATCH 1/2] [build-tools] Add opt-in guest resource telemetry --- .changeset/duo-guest-metrics.md | 5 + .github/workflows/test-guest-metrics.yml | 32 +++ .../build-tools/resources/guest-metrics.md | 11 + .../__tests__/collectServeSimMetrics.test.ts | 31 +++ .../__tests__/startServeSimMetrics.test.ts | 19 ++ .../steps/functions/collectServeSimMetrics.ts | 21 ++ .../steps/functions/startServeSimMetrics.ts | 10 +- .../__tests__/guestMetricsRecorder.test.ts | 25 ++ .../src/steps/utils/guestMetricsRecorder.ts | 254 ++++++++++++++++++ 9 files changed, 407 insertions(+), 1 deletion(-) create mode 100644 .changeset/duo-guest-metrics.md create mode 100644 .github/workflows/test-guest-metrics.yml create mode 100644 packages/build-tools/resources/guest-metrics.md create mode 100644 packages/build-tools/src/steps/utils/__tests__/guestMetricsRecorder.test.ts create mode 100644 packages/build-tools/src/steps/utils/guestMetricsRecorder.ts diff --git a/.changeset/duo-guest-metrics.md b/.changeset/duo-guest-metrics.md new file mode 100644 index 0000000000..1527911e01 --- /dev/null +++ b/.changeset/duo-guest-metrics.md @@ -0,0 +1,5 @@ +--- +'@expo/build-tools': patch +--- + +Add opt-in guest resource telemetry to Simulator sessions for performance investigations. diff --git a/.github/workflows/test-guest-metrics.yml b/.github/workflows/test-guest-metrics.yml new file mode 100644 index 0000000000..f2bdd8e90f --- /dev/null +++ b/.github/workflows/test-guest-metrics.yml @@ -0,0 +1,32 @@ +name: Guest metrics artifact + +on: + pull_request: + types: [opened, synchronize] + paths: + - packages/build-tools/src/steps/utils/guestMetricsRecorder.ts + - .github/workflows/test-guest-metrics.yml + +permissions: + contents: read + +jobs: + artifact: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - uses: ./.github/actions/setup-mise + - run: yarn install --immutable + - name: Compile standalone collector for hosted profiling + run: | + yarn exec tsc packages/build-tools/src/steps/utils/guestMetricsRecorder.ts --target es2022 --module commonjs --esModuleInterop --skipLibCheck --types node --outDir guest-metrics-artifact + git rev-parse HEAD > guest-metrics-artifact/source-commit.txt + sha256sum guest-metrics-artifact/guestMetricsRecorder.js > guest-metrics-artifact/SHA256SUMS + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: guest-metrics + path: guest-metrics-artifact + retention-days: 7 diff --git a/packages/build-tools/resources/guest-metrics.md b/packages/build-tools/resources/guest-metrics.md new file mode 100644 index 0000000000..19d6a7a030 --- /dev/null +++ b/packages/build-tools/resources/guest-metrics.md @@ -0,0 +1,11 @@ +# Guest resource profiling + +Set `EAS_DEVICE_RUN_GUEST_METRICS=1` in the device session's worker step environment to collect `guest-metrics.ndjson` alongside app metrics. The existing start/collect serve-sim metrics steps own its lifetime, including Simulator startup. This is opt-in while overhead and retention are evaluated. + +The file contains metadata, one sample per second (best effort), and a terminal health record. Each sample includes aggregate busy core equivalents (1 = one fully busy vCPU), cumulative process CPU and interval deltas, PID/start-time identity, parent PID, RSS, raw `vm_stat`, swap and memory-pressure counters. IOAccelerator registry data is recorded every five samples when exposed by the guest. Executable paths are included; command arguments and environment are not recorded. The file is private and capped at approximately 128 MiB. Commands time out after two seconds and errors remain visible in samples. + +Check the terminal record, sample gaps, per-command errors and timestamps before interpreting a phase. A missing terminal record means incomplete collection. Failed process samples reset the delta baseline. Processes born or exited between samples cannot be fully attributed; residual core usage can be negative because process and aggregate counter boundaries differ. RSS contains shared pages and is not physical footprint. Free memory excludes reclaimable cache; interpret it with pressure, compression and swap. GPU allocations consume shared system memory and do not measure physical GPU or video encoder saturation. Physical host mapping, co-tenant contention, GPU/AVE utilization, process stacks and browser WebRTC statistics require separate collectors. + +For experiments, record the exact collector commit and artifact hash, app/package versions, image and actual guest CPU/RAM. Capture cold startup separately, wait for a settled baseline, then alternate recording off/on several times on the same VM. Match actual resources before comparing VM sizes. Preserve phase timestamps and recording encoder logs alongside telemetry. + +The Guest metrics artifact GitHub workflow compiles the standalone collector from the PR head and includes its source commit and checksum. A hosted experiment can load this module directly; that validates collector behavior on a hosted VM, while the worker upload integration still requires a worker containing this change. diff --git a/packages/build-tools/src/steps/functions/__tests__/collectServeSimMetrics.test.ts b/packages/build-tools/src/steps/functions/__tests__/collectServeSimMetrics.test.ts index c491162d7a..3906f1f388 100644 --- a/packages/build-tools/src/steps/functions/__tests__/collectServeSimMetrics.test.ts +++ b/packages/build-tools/src/steps/functions/__tests__/collectServeSimMetrics.test.ts @@ -7,11 +7,17 @@ import { getDeviceRunSessionIdOrThrow } from '../../utils/remoteDeviceRunSession import { uploadServeSimMetricsFileAsync } from '../../utils/serveSimMetricsArtifacts'; import { ServeSimMetricsRecorder } from '../../utils/serveSimMetricsRecorder'; import { createCollectServeSimMetricsBuildFunction } from '../collectServeSimMetrics'; +import { finishGuestMetricsAsync } from '../../utils/guestMetricsRecorder'; +import { uploadDeviceRunSessionArtifactAsync } from '../../utils/deviceRunSessionArtifacts'; +import { stat } from 'node:fs/promises'; jest.mock('../../utils/serveSimMetricsRecorder'); jest.mock('../../utils/serveSimMetricsArtifacts'); jest.mock('../../utils/remoteDeviceRunSession'); jest.mock('../../../sentry'); +jest.mock('../../utils/guestMetricsRecorder'); +jest.mock('../../utils/deviceRunSessionArtifacts'); +jest.mock('node:fs/promises', () => ({ stat: jest.fn() })); function createLoggerMock(): bunyan { return { info: jest.fn(), warn: jest.fn() } as unknown as bunyan; @@ -60,6 +66,31 @@ describe(createCollectServeSimMetricsBuildFunction, () => { expect(uploadServeSimMetricsFileAsync).not.toHaveBeenCalled(); }); + it('uploads guest metrics even without a connected serve-sim metrics stream', async () => { + jest.mocked(ServeSimMetricsRecorder.finishAsync).mockResolvedValue([]); + jest.mocked(finishGuestMetricsAsync).mockResolvedValue('/tmp/guest-metrics.ndjson'); + jest.mocked(stat).mockResolvedValue({ size: 123 } as Awaited>); + await runAsync(createLoggerMock()); + expect(uploadDeviceRunSessionArtifactAsync).toHaveBeenCalledWith( + ctx, + expect.objectContaining({ + artifactId: 'guest-metrics', + filename: 'guest-metrics.ndjson', + size: 123, + metadata: { schemaVersion: 1, scope: 'guest' }, + }) + ); + }); + + it('still collects app metrics if guest telemetry fails', async () => { + jest.mocked(finishGuestMetricsAsync).mockRejectedValue(new Error('disk failure')); + jest.mocked(ServeSimMetricsRecorder.finishAsync).mockResolvedValue([]); + const logger = createLoggerMock(); + await runAsync(logger); + expect(logger.warn).toHaveBeenCalledWith(expect.anything(), 'Could not collect guest metrics.'); + expect(ServeSimMetricsRecorder.finishAsync).toHaveBeenCalled(); + }); + it('warns instead of failing the session when the session id is missing', async () => { jest .mocked(ServeSimMetricsRecorder.finishAsync) diff --git a/packages/build-tools/src/steps/functions/__tests__/startServeSimMetrics.test.ts b/packages/build-tools/src/steps/functions/__tests__/startServeSimMetrics.test.ts index d14a58d9ba..4102af6954 100644 --- a/packages/build-tools/src/steps/functions/__tests__/startServeSimMetrics.test.ts +++ b/packages/build-tools/src/steps/functions/__tests__/startServeSimMetrics.test.ts @@ -2,9 +2,11 @@ import { type bunyan } from '@expo/logger'; import { BuildStepContext, BuildStepEnv } from '@expo/steps'; import { ServeSimMetricsRecorder } from '../../utils/serveSimMetricsRecorder'; +import { startGuestMetricsAsync } from '../../utils/guestMetricsRecorder'; import { createStartServeSimMetricsBuildFunction } from '../startServeSimMetrics'; jest.mock('../../utils/serveSimMetricsRecorder'); +jest.mock('../../utils/guestMetricsRecorder'); function createLoggerMock(): bunyan { return { info: jest.fn(), warn: jest.fn() } as unknown as bunyan; @@ -20,5 +22,22 @@ describe(createStartServeSimMetricsBuildFunction, () => { env: {} as BuildStepEnv, }); expect(ServeSimMetricsRecorder.startAsync).toHaveBeenCalledWith({ logger }); + expect(startGuestMetricsAsync).not.toHaveBeenCalled(); + }); + + it('starts guest metrics only when requested and tolerates collector failure', async () => { + jest.mocked(startGuestMetricsAsync).mockRejectedValueOnce(new Error('unavailable')); + const logger = createLoggerMock(); + await createStartServeSimMetricsBuildFunction().fn?.( + { logger } as unknown as BuildStepContext, + { + inputs: {}, + outputs: {}, + env: { EAS_DEVICE_RUN_GUEST_METRICS: '1' } as BuildStepEnv, + } + ); + expect(startGuestMetricsAsync).toHaveBeenCalledTimes(1); + expect(logger.warn).toHaveBeenCalled(); + expect(ServeSimMetricsRecorder.startAsync).toHaveBeenCalledWith({ logger }); }); }); diff --git a/packages/build-tools/src/steps/functions/collectServeSimMetrics.ts b/packages/build-tools/src/steps/functions/collectServeSimMetrics.ts index f7c965ba0d..6b6919b812 100644 --- a/packages/build-tools/src/steps/functions/collectServeSimMetrics.ts +++ b/packages/build-tools/src/steps/functions/collectServeSimMetrics.ts @@ -1,10 +1,14 @@ import { BuildFunction, BuildRuntimePlatform } from '@expo/steps'; +import { createReadStream } from 'node:fs'; +import { stat } from 'node:fs/promises'; import { type CustomBuildContext } from '../../customBuildContext'; import { Sentry } from '../../sentry'; import { getDeviceRunSessionIdOrThrow } from '../utils/remoteDeviceRunSession'; import { uploadServeSimMetricsFileAsync } from '../utils/serveSimMetricsArtifacts'; import { ServeSimMetricsRecorder } from '../utils/serveSimMetricsRecorder'; +import { finishGuestMetricsAsync } from '../utils/guestMetricsRecorder'; +import { uploadDeviceRunSessionArtifactAsync } from '../utils/deviceRunSessionArtifacts'; export function createCollectServeSimMetricsBuildFunction(ctx: CustomBuildContext): BuildFunction { return new BuildFunction({ @@ -14,6 +18,23 @@ export function createCollectServeSimMetricsBuildFunction(ctx: CustomBuildContex __metricsId: 'eas/collect_serve_sim_metrics', supportedRuntimePlatforms: [BuildRuntimePlatform.DARWIN], fn: async ({ logger }, { env }) => { + try { + const filePath = await finishGuestMetricsAsync(); + if (filePath) { + await uploadDeviceRunSessionArtifactAsync(ctx, { + deviceRunSessionId: getDeviceRunSessionIdOrThrow(env), + artifactId: 'guest-metrics', + name: 'Guest resource metrics', + filename: 'guest-metrics.ndjson', + kind: undefined, + metadata: { schemaVersion: 1, scope: 'guest' }, + size: (await stat(filePath)).size, + openStream: () => createReadStream(filePath), + }); + } + } catch (err) { + logger.warn({ err }, 'Could not collect guest metrics.'); + } const collected = await ServeSimMetricsRecorder.finishAsync({ logger }); if (collected.length === 0) { logger.info('No serve-sim metrics collected; skipping upload.'); diff --git a/packages/build-tools/src/steps/functions/startServeSimMetrics.ts b/packages/build-tools/src/steps/functions/startServeSimMetrics.ts index e50b62cdd5..7ea3d22edc 100644 --- a/packages/build-tools/src/steps/functions/startServeSimMetrics.ts +++ b/packages/build-tools/src/steps/functions/startServeSimMetrics.ts @@ -1,6 +1,7 @@ import { BuildFunction, BuildRuntimePlatform } from '@expo/steps'; import { ServeSimMetricsRecorder } from '../utils/serveSimMetricsRecorder'; +import { startGuestMetricsAsync } from '../utils/guestMetricsRecorder'; export function createStartServeSimMetricsBuildFunction(): BuildFunction { return new BuildFunction({ @@ -9,7 +10,14 @@ export function createStartServeSimMetricsBuildFunction(): BuildFunction { name: 'Start serve-sim metrics', __metricsId: 'eas/start_serve_sim_metrics', supportedRuntimePlatforms: [BuildRuntimePlatform.DARWIN], - fn: async ({ logger }) => { + fn: async ({ logger }, { env }) => { + if (env.EAS_DEVICE_RUN_GUEST_METRICS === '1') { + try { + await startGuestMetricsAsync(); + } catch (err) { + logger.warn({ err }, 'Could not start guest metrics.'); + } + } await ServeSimMetricsRecorder.startAsync({ logger }); }, }); diff --git a/packages/build-tools/src/steps/utils/__tests__/guestMetricsRecorder.test.ts b/packages/build-tools/src/steps/utils/__tests__/guestMetricsRecorder.test.ts new file mode 100644 index 0000000000..6f9f261787 --- /dev/null +++ b/packages/build-tools/src/steps/utils/__tests__/guestMetricsRecorder.test.ts @@ -0,0 +1,25 @@ +import { parseGuestProcesses } from '../guestMetricsRecorder'; + +describe(parseGuestProcesses, () => { + it('parses cumulative CPU, parentage, start identity and paths containing spaces', () => { + expect( + parseGuestProcesses( + ' 12 1 Thu Oct 1 14:51:02 2026 2-03:04:05.50 200 /Applications/My App.app/App' + )[0] + ).toEqual({ + pid: 12, + ppid: 1, + started: 'Thu Oct 1 14:51:02 2026', + cpuSeconds: 183845.5, + rssBytes: 204800, + executable: '/Applications/My App.app/App', + }); + expect( + parseGuestProcesses('12 1 Thu Oct 1 14:51:02 2026 3:05.25 200 /bin/app')[0].cpuSeconds + ).toBe(185.25); + }); + + it('rejects an unrecognized row rather than silently undercounting', () => { + expect(() => parseGuestProcesses('12 broken')).toThrow('Unrecognized ps row'); + }); +}); diff --git a/packages/build-tools/src/steps/utils/guestMetricsRecorder.ts b/packages/build-tools/src/steps/utils/guestMetricsRecorder.ts new file mode 100644 index 0000000000..afb57d9b19 --- /dev/null +++ b/packages/build-tools/src/steps/utils/guestMetricsRecorder.ts @@ -0,0 +1,254 @@ +import { execFile } from 'node:child_process'; +import { mkdtemp, open } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); +const INTERVAL_MS = 1_000; +const MAX_BYTES = 128 * 1024 * 1024; + +type ProcessSample = { + pid: number; + ppid: number; + started: string; + cpuSeconds: number; + rssBytes: number; + executable: string; +}; + +// RSS includes shared pages. Never sum it as physical guest memory usage. +export function parseGuestProcesses(output: string): ProcessSample[] { + return output + .trim() + .split('\n') + .filter(Boolean) + .map(line => { + const match = + /^\s*(\d+)\s+(\d+)\s+(\w+\s+\w+\s+\d+\s+[\d:]+\s+\d+)\s+([\d:.\-]+)\s+(\d+)\s+(.+)$/.exec( + line + ); + if (!match) { + throw new Error('Unrecognized ps row'); + } + const [, pid, ppid, started, cpu, rss, executable] = match; + const [days, clock] = cpu.includes('-') ? cpu.split('-') : ['0', cpu]; + const seconds = clock.split(':').reduce((total, part) => total * 60 + Number(part), 0); + return { + pid: Number(pid), + ppid: Number(ppid), + started, + cpuSeconds: Number(days) * 86400 + seconds, + rssBytes: Number(rss) * 1024, + executable, + }; + }); +} + +function cpuTotals(): { busy: number; total: number } { + return os.cpus().reduce( + (sum, { times }) => ({ + busy: sum.busy + times.user + times.nice + times.sys + times.irq, + total: sum.total + times.user + times.nice + times.sys + times.irq + times.idle, + }), + { busy: 0, total: 0 } + ); +} + +async function command(file: string, args: string[]): Promise<{ output?: string; error?: string }> { + try { + const { stdout } = await execFileAsync(file, args, { + timeout: 2_000, + maxBuffer: 8 * 1024 * 1024, + env: { ...process.env, LC_ALL: 'C' }, + }); + return { output: stdout }; + } catch (error) { + // Do not serialize commands, environment, or stderr into telemetry errors. + return { error: error instanceof Error ? error.name : 'CommandError' }; + } +} + +export class GuestMetricsRecorder { + private readonly abort = new AbortController(); + private readonly done: Promise; + private error: unknown; + + private constructor(public readonly filePath: string) { + this.done = this.recordAsync().catch(error => { + this.error = error; + }); + } + + static async startAsync(): Promise { + if (process.platform !== 'darwin') { + return undefined; + } + const directory = await mkdtemp(path.join(os.tmpdir(), 'guest-metrics-')); + return new GuestMetricsRecorder(path.join(directory, 'guest-metrics.ndjson')); + } + + async finishAsync(): Promise { + this.abort.abort(); + await this.done; + if (this.error) { + throw this.error; + } + return this.filePath; + } + + private async recordAsync(): Promise { + const file = await open(this.filePath, 'wx', 0o600); + let bytes = 0; + const write = async (record: object): Promise => { + const line = + JSON.stringify({ + time: new Date().toISOString(), + monotonicMs: performance.now(), + ...record, + }) + '\n'; + await file.write(line); + bytes += Buffer.byteLength(line); + }; + let count = 0; + let failures = 0; + let previousCpu = cpuTotals(); + let previousProcesses = new Map(); + let previousProcessTime: number | undefined; + let maxSampleGapMs = 0; + let previousStart: number | undefined; + let reason = 'stopped'; + try { + await write({ + type: 'meta', + schemaVersion: 1, + intervalMs: INTERVAL_MS, + cores: os.cpus().length, + memoryBytes: os.totalmem(), + bootTimeApprox: new Date(Date.now() - os.uptime() * 1000).toISOString(), + release: os.release(), + arch: os.arch(), + model: os.cpus()[0]?.model, + pid: process.pid, + jobId: process.env.EAS_BUILD_ID, + workflowJobId: process.env.EAS_WORKFLOW_JOB_ID, + limitations: [ + 'Guest metrics only; physical host GPU and video engine utilization unavailable.', + 'Process CPU excludes tasks that exit between polls; residual is not attributable.', + 'RSS includes shared pages; GPU allocations share system RAM.', + 'Process and aggregate counters have different sampling boundaries.', + ], + }); + while (!this.abort.signal.aborted && bytes < MAX_BYTES) { + const start = performance.now(); + if (previousStart !== undefined) { + maxSampleGapMs = Math.max(maxSampleGapMs, start - previousStart); + } + previousStart = start; + const cpu = cpuTotals(); + const totalDelta = cpu.total - previousCpu.total; + const busyCores = + totalDelta > 0 ? ((cpu.busy - previousCpu.busy) / totalDelta) * os.cpus().length : null; + previousCpu = cpu; + const [ps, vm, swap, pressure, gpu] = await Promise.all([ + command('/bin/ps', ['-axo', 'pid=,ppid=,lstart=,time=,rss=,comm=']), + command('/usr/bin/vm_stat', []), + command('/usr/sbin/sysctl', ['vm.swapusage']), + command('/usr/bin/memory_pressure', ['-Q']), + count % 5 === 0 + ? command('/usr/sbin/ioreg', ['-a', '-l', '-r', '-c', 'IOAccelerator']) + : Promise.resolve(undefined), + ]); + const processTime = performance.now(); + const errors: Record = {}; + let processes: (ProcessSample & { cpuCores: number | null })[] = []; + let exited: string[] = []; + try { + if (!ps.output) { + throw new Error('Process sample unavailable'); + } + const parsed = parseGuestProcesses(ps.output); + const current = new Map(parsed.map(p => [`${p.pid}:${p.started}`, p])); + processes = parsed.map(p => { + const prior = previousProcesses.get(`${p.pid}:${p.started}`); + return { + ...p, + cpuCores: + prior && previousProcessTime !== undefined + ? Math.max(0, p.cpuSeconds - prior.cpuSeconds) / + ((processTime - previousProcessTime) / 1000) + : null, + }; + }); + exited = [...previousProcesses.keys()].filter(key => !current.has(key)); + previousProcesses = current; + previousProcessTime = processTime; + } catch { + errors.processes = ps.error ?? 'ParseError'; + previousProcesses.clear(); + previousProcessTime = undefined; + } + for (const [key, result] of Object.entries({ vm, swap, pressure, gpu })) { + if (result?.error) { + errors[key] = result.error; + } + } + if (Object.keys(errors).length) { + failures++; + } + const attributedCores = processes.reduce((sum, p) => sum + (p.cpuCores ?? 0), 0); + await write({ + type: 'sample', + sequence: count++, + elapsedMs: performance.now() - start, + uptimeSeconds: os.uptime(), + freeBytes: os.freemem(), + busyCores: count > 1 ? busyCores : null, + attributedCores, + residualCores: count > 1 && busyCores !== null ? busyCores - attributedCores : null, + processes, + exited, + vm: vm.output, + swap: swap.output, + pressure: pressure.output, + gpu: gpu?.output, + errors, + }); + await delay(Math.max(0, INTERVAL_MS - (performance.now() - start)), undefined, { + signal: this.abort.signal, + }).catch(() => {}); + } + if (bytes >= MAX_BYTES) { + reason = 'size-limit'; + } + } catch (error) { + reason = 'collector-error'; + await write({ type: 'error', error: error instanceof Error ? error.name : 'Error' }); + } finally { + try { + await write({ + type: 'end', + reason, + samples: count, + samplesWithErrors: failures, + maxSampleGapMs, + }); + } finally { + await file.close(); + } + } + } +} + +let active: GuestMetricsRecorder | undefined; + +export async function startGuestMetricsAsync(): Promise { + active ??= await GuestMetricsRecorder.startAsync(); +} + +export async function finishGuestMetricsAsync(): Promise { + const recorder = active; + active = undefined; + return await recorder?.finishAsync(); +} From bc73c7171196595a31992fe5a0e3c8a305cb4849 Mon Sep 17 00:00:00 2001 From: Gabe Debes Date: Sun, 18 Oct 2026 09:50:25 -0700 Subject: [PATCH 2/2] [build-tools] Timestamp each guest process probe independently --- .../build-tools/resources/guest-metrics.md | 2 +- .../src/steps/utils/guestMetricsRecorder.ts | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/packages/build-tools/resources/guest-metrics.md b/packages/build-tools/resources/guest-metrics.md index 19d6a7a030..f39fb7b906 100644 --- a/packages/build-tools/resources/guest-metrics.md +++ b/packages/build-tools/resources/guest-metrics.md @@ -2,7 +2,7 @@ Set `EAS_DEVICE_RUN_GUEST_METRICS=1` in the device session's worker step environment to collect `guest-metrics.ndjson` alongside app metrics. The existing start/collect serve-sim metrics steps own its lifetime, including Simulator startup. This is opt-in while overhead and retention are evaluated. -The file contains metadata, one sample per second (best effort), and a terminal health record. Each sample includes aggregate busy core equivalents (1 = one fully busy vCPU), cumulative process CPU and interval deltas, PID/start-time identity, parent PID, RSS, raw `vm_stat`, swap and memory-pressure counters. IOAccelerator registry data is recorded every five samples when exposed by the guest. Executable paths are included; command arguments and environment are not recorded. The file is private and capped at approximately 128 MiB. Commands time out after two seconds and errors remain visible in samples. +The file contains metadata, one sample per second (best effort), and a terminal health record. Each sample includes aggregate busy core equivalents (1 = one fully busy vCPU), cumulative process CPU and interval deltas, PID/start-time identity, parent PID, RSS, raw `vm_stat`, swap and memory-pressure counters. IOAccelerator registry data is recorded every five samples when exposed by the guest. Executable paths are included; command arguments and environment are not recorded. The file is private and capped at approximately 128 MiB. Commands request a two-second timeout with SIGKILL; OS stalls can delay collection beyond that deadline. Errors and gaps remain visible. CPU and process probes have separate monotonic timestamps so a slow sibling probe does not distort process CPU intervals. Check the terminal record, sample gaps, per-command errors and timestamps before interpreting a phase. A missing terminal record means incomplete collection. Failed process samples reset the delta baseline. Processes born or exited between samples cannot be fully attributed; residual core usage can be negative because process and aggregate counter boundaries differ. RSS contains shared pages and is not physical footprint. Free memory excludes reclaimable cache; interpret it with pressure, compression and swap. GPU allocations consume shared system memory and do not measure physical GPU or video encoder saturation. Physical host mapping, co-tenant contention, GPU/AVE utilization, process stacks and browser WebRTC statistics require separate collectors. diff --git a/packages/build-tools/src/steps/utils/guestMetricsRecorder.ts b/packages/build-tools/src/steps/utils/guestMetricsRecorder.ts index afb57d9b19..9ecc714ce0 100644 --- a/packages/build-tools/src/steps/utils/guestMetricsRecorder.ts +++ b/packages/build-tools/src/steps/utils/guestMetricsRecorder.ts @@ -56,17 +56,25 @@ function cpuTotals(): { busy: number; total: number } { ); } -async function command(file: string, args: string[]): Promise<{ output?: string; error?: string }> { +async function command( + file: string, + args: string[] +): Promise<{ output?: string; error?: string; completedMonotonicMs: number }> { try { const { stdout } = await execFileAsync(file, args, { timeout: 2_000, + killSignal: 'SIGKILL', maxBuffer: 8 * 1024 * 1024, env: { ...process.env, LC_ALL: 'C' }, }); - return { output: stdout }; + return { output: stdout, completedMonotonicMs: performance.now() }; } catch (error) { // Do not serialize commands, environment, or stderr into telemetry errors. - return { error: error instanceof Error ? error.name : 'CommandError' }; + const details = error as { name?: string; code?: string | number; signal?: string }; + return { + error: `${details.name ?? 'CommandError'}:${details.code ?? details.signal ?? 'unknown'}`, + completedMonotonicMs: performance.now(), + }; } } @@ -160,7 +168,8 @@ export class GuestMetricsRecorder { ? command('/usr/sbin/ioreg', ['-a', '-l', '-r', '-c', 'IOAccelerator']) : Promise.resolve(undefined), ]); - const processTime = performance.now(); + // Use this probe's completion, not a slower sibling probe's completion. + const processTime = ps.completedMonotonicMs; const errors: Record = {}; let processes: (ProcessSample & { cpuCores: number | null })[] = []; let exited: string[] = []; @@ -202,6 +211,8 @@ export class GuestMetricsRecorder { type: 'sample', sequence: count++, elapsedMs: performance.now() - start, + cpuSampleMonotonicMs: start, + processSampleMonotonicMs: processTime, uptimeSeconds: os.uptime(), freeBytes: os.freemem(), busyCores: count > 1 ? busyCores : null,