[build-tools] Add guest telemetry for Simulator profiling

.changeset/duo-guest-metrics.md

@@ -0,0 +1,5 @@
+---
+'@expo/build-tools': patch
+---
+
+Add opt-in guest resource telemetry to Simulator sessions for performance investigations.

.github/workflows/test-guest-metrics.yml

@@ -0,0 +1,32 @@
+name: Guest metrics artifact
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+    paths:
+      - packages/build-tools/src/steps/utils/guestMetricsRecorder.ts
+      - .github/workflows/test-guest-metrics.yml
+
+permissions:
+  contents: read
+
+jobs:
+  artifact:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          persist-credentials: false
+      - uses: ./.github/actions/setup-mise
+      - run: yarn install --immutable
+      - name: Compile standalone collector for hosted profiling
+        run: |
+          yarn exec tsc packages/build-tools/src/steps/utils/guestMetricsRecorder.ts --target es2022 --module commonjs --esModuleInterop --skipLibCheck --types node --outDir guest-metrics-artifact
+          git rev-parse HEAD > guest-metrics-artifact/source-commit.txt
+          sha256sum guest-metrics-artifact/guestMetricsRecorder.js > guest-metrics-artifact/SHA256SUMS
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: guest-metrics
+          path: guest-metrics-artifact
+          retention-days: 7

packages/build-tools/resources/guest-metrics.md

@@ -0,0 +1,11 @@
+# Guest resource profiling
+
+Set `EAS_DEVICE_RUN_GUEST_METRICS=1` in the device session's worker step environment to collect `guest-metrics.ndjson` alongside app metrics. The existing start/collect serve-sim metrics steps own its lifetime, including Simulator startup. This is opt-in while overhead and retention are evaluated.
+
+The file contains metadata, one sample per second (best effort), and a terminal health record. Each sample includes aggregate busy core equivalents (1 = one fully busy vCPU), cumulative process CPU and interval deltas, PID/start-time identity, parent PID, RSS, raw `vm_stat`, swap and memory-pressure counters. IOAccelerator registry data is recorded every five samples when exposed by the guest. Executable paths are included; command arguments and environment are not recorded. The file is private and capped at approximately 128 MiB. Commands request a two-second timeout with SIGKILL; OS stalls can delay collection beyond that deadline. Errors and gaps remain visible. CPU and process probes have separate monotonic timestamps so a slow sibling probe does not distort process CPU intervals.
+
+Check the terminal record, sample gaps, per-command errors and timestamps before interpreting a phase. A missing terminal record means incomplete collection. Failed process samples reset the delta baseline. Processes born or exited between samples cannot be fully attributed; residual core usage can be negative because process and aggregate counter boundaries differ. RSS contains shared pages and is not physical footprint. Free memory excludes reclaimable cache; interpret it with pressure, compression and swap. GPU allocations consume shared system memory and do not measure physical GPU or video encoder saturation. Physical host mapping, co-tenant contention, GPU/AVE utilization, process stacks and browser WebRTC statistics require separate collectors.
+
+For experiments, record the exact collector commit and artifact hash, app/package versions, image and actual guest CPU/RAM. Capture cold startup separately, wait for a settled baseline, then alternate recording off/on several times on the same VM. Match actual resources before comparing VM sizes. Preserve phase timestamps and recording encoder logs alongside telemetry.
+
+The Guest metrics artifact GitHub workflow compiles the standalone collector from the PR head and includes its source commit and checksum. A hosted experiment can load this module directly; that validates collector behavior on a hosted VM, while the worker upload integration still requires a worker containing this change.

packages/build-tools/src/steps/functions/__tests__/collectServeSimMetrics.test.ts

@@ -7,11 +7,17 @@ import { getDeviceRunSessionIdOrThrow } from '../../utils/remoteDeviceRunSession
 import { uploadServeSimMetricsFileAsync } from '../../utils/serveSimMetricsArtifacts';
 import { ServeSimMetricsRecorder } from '../../utils/serveSimMetricsRecorder';
 import { createCollectServeSimMetricsBuildFunction } from '../collectServeSimMetrics';
+import { finishGuestMetricsAsync } from '../../utils/guestMetricsRecorder';
+import { uploadDeviceRunSessionArtifactAsync } from '../../utils/deviceRunSessionArtifacts';
+import { stat } from 'node:fs/promises';
 
 jest.mock('../../utils/serveSimMetricsRecorder');
 jest.mock('../../utils/serveSimMetricsArtifacts');
 jest.mock('../../utils/remoteDeviceRunSession');
 jest.mock('../../../sentry');
+jest.mock('../../utils/guestMetricsRecorder');
+jest.mock('../../utils/deviceRunSessionArtifacts');
+jest.mock('node:fs/promises', () => ({ stat: jest.fn() }));
 
 function createLoggerMock(): bunyan {
   return { info: jest.fn(), warn: jest.fn() } as unknown as bunyan;
@@ -60,6 +66,31 @@ describe(createCollectServeSimMetricsBuildFunction, () => {
     expect(uploadServeSimMetricsFileAsync).not.toHaveBeenCalled();
   });
 
+  it('uploads guest metrics even without a connected serve-sim metrics stream', async () => {
+    jest.mocked(ServeSimMetricsRecorder.finishAsync).mockResolvedValue([]);
+    jest.mocked(finishGuestMetricsAsync).mockResolvedValue('/tmp/guest-metrics.ndjson');
+    jest.mocked(stat).mockResolvedValue({ size: 123 } as Awaited<ReturnType<typeof stat>>);
+    await runAsync(createLoggerMock());
+    expect(uploadDeviceRunSessionArtifactAsync).toHaveBeenCalledWith(
+      ctx,
+      expect.objectContaining({
+        artifactId: 'guest-metrics',
+        filename: 'guest-metrics.ndjson',
+        size: 123,
+        metadata: { schemaVersion: 1, scope: 'guest' },
+      })
+    );
+  });
+
+  it('still collects app metrics if guest telemetry fails', async () => {
+    jest.mocked(finishGuestMetricsAsync).mockRejectedValue(new Error('disk failure'));
+    jest.mocked(ServeSimMetricsRecorder.finishAsync).mockResolvedValue([]);
+    const logger = createLoggerMock();
+    await runAsync(logger);
+    expect(logger.warn).toHaveBeenCalledWith(expect.anything(), 'Could not collect guest metrics.');
+    expect(ServeSimMetricsRecorder.finishAsync).toHaveBeenCalled();
+  });
+
   it('warns instead of failing the session when the session id is missing', async () => {
     jest
       .mocked(ServeSimMetricsRecorder.finishAsync)

packages/build-tools/src/steps/functions/__tests__/startServeSimMetrics.test.ts

@@ -2,9 +2,11 @@ import { type bunyan } from '@expo/logger';
 import { BuildStepContext, BuildStepEnv } from '@expo/steps';
 
 import { ServeSimMetricsRecorder } from '../../utils/serveSimMetricsRecorder';
+import { startGuestMetricsAsync } from '../../utils/guestMetricsRecorder';
 import { createStartServeSimMetricsBuildFunction } from '../startServeSimMetrics';
 
 jest.mock('../../utils/serveSimMetricsRecorder');
+jest.mock('../../utils/guestMetricsRecorder');
 
 function createLoggerMock(): bunyan {
   return { info: jest.fn(), warn: jest.fn() } as unknown as bunyan;
@@ -20,5 +22,22 @@ describe(createStartServeSimMetricsBuildFunction, () => {
       env: {} as BuildStepEnv,
     });
     expect(ServeSimMetricsRecorder.startAsync).toHaveBeenCalledWith({ logger });
+    expect(startGuestMetricsAsync).not.toHaveBeenCalled();
+  });
+
+  it('starts guest metrics only when requested and tolerates collector failure', async () => {
+    jest.mocked(startGuestMetricsAsync).mockRejectedValueOnce(new Error('unavailable'));
+    const logger = createLoggerMock();
+    await createStartServeSimMetricsBuildFunction().fn?.(
+      { logger } as unknown as BuildStepContext,
+      {
+        inputs: {},
+        outputs: {},
+        env: { EAS_DEVICE_RUN_GUEST_METRICS: '1' } as BuildStepEnv,
+      }
+    );
+    expect(startGuestMetricsAsync).toHaveBeenCalledTimes(1);
+    expect(logger.warn).toHaveBeenCalled();
+    expect(ServeSimMetricsRecorder.startAsync).toHaveBeenCalledWith({ logger });
   });
 });

packages/build-tools/src/steps/functions/collectServeSimMetrics.ts

@@ -1,10 +1,14 @@
 import { BuildFunction, BuildRuntimePlatform } from '@expo/steps';
+import { createReadStream } from 'node:fs';
+import { stat } from 'node:fs/promises';
 
 import { type CustomBuildContext } from '../../customBuildContext';
 import { Sentry } from '../../sentry';
 import { getDeviceRunSessionIdOrThrow } from '../utils/remoteDeviceRunSession';
 import { uploadServeSimMetricsFileAsync } from '../utils/serveSimMetricsArtifacts';
 import { ServeSimMetricsRecorder } from '../utils/serveSimMetricsRecorder';
+import { finishGuestMetricsAsync } from '../utils/guestMetricsRecorder';
+import { uploadDeviceRunSessionArtifactAsync } from '../utils/deviceRunSessionArtifacts';
 
 export function createCollectServeSimMetricsBuildFunction(ctx: CustomBuildContext): BuildFunction {
   return new BuildFunction({
@@ -14,6 +18,23 @@ export function createCollectServeSimMetricsBuildFunction(ctx: CustomBuildContex
     __metricsId: 'eas/collect_serve_sim_metrics',
     supportedRuntimePlatforms: [BuildRuntimePlatform.DARWIN],
     fn: async ({ logger }, { env }) => {
+      try {
+        const filePath = await finishGuestMetricsAsync();
+        if (filePath) {
+          await uploadDeviceRunSessionArtifactAsync(ctx, {
+            deviceRunSessionId: getDeviceRunSessionIdOrThrow(env),
+            artifactId: 'guest-metrics',
+            name: 'Guest resource metrics',
+            filename: 'guest-metrics.ndjson',
+            kind: undefined,
+            metadata: { schemaVersion: 1, scope: 'guest' },
+            size: (await stat(filePath)).size,
+            openStream: () => createReadStream(filePath),
+          });
+        }
+      } catch (err) {
+        logger.warn({ err }, 'Could not collect guest metrics.');
+      }
       const collected = await ServeSimMetricsRecorder.finishAsync({ logger });
       if (collected.length === 0) {
         logger.info('No serve-sim metrics collected; skipping upload.');

packages/build-tools/src/steps/functions/startServeSimMetrics.ts

@@ -1,6 +1,7 @@
 import { BuildFunction, BuildRuntimePlatform } from '@expo/steps';
 
 import { ServeSimMetricsRecorder } from '../utils/serveSimMetricsRecorder';
+import { startGuestMetricsAsync } from '../utils/guestMetricsRecorder';
 
 export function createStartServeSimMetricsBuildFunction(): BuildFunction {
   return new BuildFunction({
@@ -9,7 +10,14 @@ export function createStartServeSimMetricsBuildFunction(): BuildFunction {
     name: 'Start serve-sim metrics',
     __metricsId: 'eas/start_serve_sim_metrics',
     supportedRuntimePlatforms: [BuildRuntimePlatform.DARWIN],
-    fn: async ({ logger }) => {
+    fn: async ({ logger }, { env }) => {
+      if (env.EAS_DEVICE_RUN_GUEST_METRICS === '1') {
+        try {
+          await startGuestMetricsAsync();
+        } catch (err) {
+          logger.warn({ err }, 'Could not start guest metrics.');
+        }
+      }
       await ServeSimMetricsRecorder.startAsync({ logger });
     },
   });

packages/build-tools/src/steps/utils/__tests__/guestMetricsRecorder.test.ts

@@ -0,0 +1,25 @@
+import { parseGuestProcesses } from '../guestMetricsRecorder';
+
+describe(parseGuestProcesses, () => {
+  it('parses cumulative CPU, parentage, start identity and paths containing spaces', () => {
+    expect(
+      parseGuestProcesses(
+        ' 12 1 Thu Oct  1 14:51:02 2026 2-03:04:05.50 200 /Applications/My App.app/App'
+      )[0]
+    ).toEqual({
+      pid: 12,
+      ppid: 1,
+      started: 'Thu Oct  1 14:51:02 2026',
+      cpuSeconds: 183845.5,
+      rssBytes: 204800,
+      executable: '/Applications/My App.app/App',
+    });
+    expect(
+      parseGuestProcesses('12 1 Thu Oct  1 14:51:02 2026 3:05.25 200 /bin/app')[0].cpuSeconds
+    ).toBe(185.25);
+  });
+
+  it('rejects an unrecognized row rather than silently undercounting', () => {
+    expect(() => parseGuestProcesses('12 broken')).toThrow('Unrecognized ps row');
+  });
+});