From 38afc2aac4aeb0f306c051d44b1c3362c74769e6 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 3 Nov 2026 18:22:14 +0100 Subject: [PATCH] perf(tinybench): batch fast walltime invocations into rounds Calibrate approximately 1 ms rounds from warmup and share the iteration count across isolated forks. Normalize latency statistics per invocation while retaining raw total time. Keep per-invocation hooks and concurrent tasks unbatched, and cover shared, forked, synchronous, and asynchronous execution. --- packages/core/src/isolation.ts | 128 +++++++--- packages/core/tests/fixtures/isolation.cjs | 23 +- packages/core/tests/isolation.integ.test.ts | 87 ++++++- packages/tinybench-plugin/src/batching.ts | 107 +++++++++ packages/tinybench-plugin/src/benchOptions.ts | 1 + packages/tinybench-plugin/src/index.ts | 35 ++- packages/tinybench-plugin/src/taskData.ts | 1 + packages/tinybench-plugin/src/walltime.ts | 52 +++-- .../tests/batching.integ.test.ts | 218 ++++++++++++++++++ .../tests/fixtures/batching.mjs | 106 +++++++++ packages/vitest-plugin/src/v5/forkReporter.ts | 2 +- packages/vitest-plugin/src/v5/provider.ts | 2 +- 12 files changed, 705 insertions(+), 57 deletions(-) create mode 100644 packages/tinybench-plugin/src/batching.ts create mode 100644 packages/tinybench-plugin/tests/batching.integ.test.ts create mode 100644 packages/tinybench-plugin/tests/fixtures/batching.mjs diff --git a/packages/core/src/isolation.ts b/packages/core/src/isolation.ts index 6f0f1d36..fd833b5c 100644 --- a/packages/core/src/isolation.ts +++ b/packages/core/src/isolation.ts @@ -23,6 +23,13 @@ export const DEFAULT_WALLTIME_FORKS = 3; export const ISOLATED_BENCHMARK_ENV = "CODSPEED_ISOLATED_BENCHMARK"; /** Set on each fork: the file where it writes its round samples. */ export const FORK_SAMPLES_FILE_ENV = "CODSPEED_FORK_SAMPLES_FILE"; +/** + * Set on the forks after the first one by {@link runForks}: the number of + * iterations that the first fork measured per round. Every fork of a benchmark + * MUST batch the same way, because the samples of all forks are pooled. + */ +export const FORK_ITERATIONS_PER_ROUND_ENV = + "CODSPEED_FORK_ITERATIONS_PER_ROUND"; /** Walltime settings that every integration exposes next to its own options. */ export interface WalltimeOptions { @@ -40,8 +47,16 @@ export type IsolationMode = | { kind: "SharedProcess" } /** Every benchmark is measured in `forks` fresh processes, one after the other. */ | { kind: "Forks"; forks: number } - /** A fork: measures only `uri` and writes its samples to `samplesFile`. */ - | { kind: "Fork"; uri: string; samplesFile: string }; + /** + * A fork: measures only `uri` and writes its samples to `samplesFile`. The + * first fork picks `iterationsPerRound` itself; the ones after it inherit it. + */ + | { + kind: "Fork"; + uri: string; + samplesFile: string; + iterationsPerRound?: number; + }; /** * The isolation of a walltime run, from the environment. `forks` is the @@ -56,7 +71,16 @@ export function walltimeIsolation(forks?: number): IsolationMode { `[CodSpeed] ${ISOLATED_BENCHMARK_ENV} is set without ${FORK_SAMPLES_FILE_ENV}`, ); } - return { kind: "Fork", uri, samplesFile }; + const iterationsPerRound = process.env[FORK_ITERATIONS_PER_ROUND_ENV]; + return { + kind: "Fork", + uri, + samplesFile, + iterationsPerRound: + iterationsPerRound === undefined || iterationsPerRound === "" + ? undefined + : validPositiveInt(iterationsPerRound, FORK_ITERATIONS_PER_ROUND_ENV), + }; } const isolation = process.env[PROCESS_ISOLATION_ENV]; if (isolation === "0" || isolation === "false") { @@ -77,22 +101,22 @@ export function walltimeIsolation(forks?: number): IsolationMode { export function resolveWalltimeForks(option?: number): number { const env = process.env[WALLTIME_FORKS_ENV]; if (env !== undefined && env !== "") { - return validForks(env, WALLTIME_FORKS_ENV); + return validPositiveInt(env, WALLTIME_FORKS_ENV); } if (option !== undefined) { - return validForks(option, "forks"); + return validPositiveInt(option, "forks"); } return DEFAULT_WALLTIME_FORKS; } -function validForks(raw: string | number, source: string): number { - const forks = Number(raw); - if (!Number.isInteger(forks) || forks < 1) { +function validPositiveInt(raw: string | number, source: string): number { + const value = Number(raw); + if (!Number.isInteger(value) || value < 1) { throw new Error( `[CodSpeed] ${source} must be a positive integer, got \`${raw}\``, ); } - return forks; + return value; } /** The command of this process, which each fork re-runs. */ @@ -101,31 +125,62 @@ function currentCommand(): string[] { return [process.execPath, ...process.execArgv, ...process.argv.slice(1)]; } +/** The pooled samples of the forks of one benchmark. */ +export interface ForkSamples { + /** Round samples (milliseconds), one per measured round. */ + samples: number[]; + /** Benchmark iterations one round measured; every fork MUST agree. */ + iterationsPerRound: number; +} + /** * Runs the benchmark `uri` in `forks` fresh processes, one after the other, * and returns the round samples of all of them, in milliseconds. Throws when * a fork fails, without running the next one. * + * The first fork picks how many iterations one round batches; the ones after + * it inherit that count, so the pooled samples mean the same thing. + * * Each fork re-runs this process's command, not fork(): Node can't fork, and a * fork would only copy the calling thread. Call it from the process whose * command runs the benchmarks: Vitest's main process, not a worker. */ -export async function runForks(uri: string, forks: number): Promise { - let samples: number[] = []; +export async function runForks( + uri: string, + forks: number, +): Promise { + let result: ForkSamples | undefined; for (let fork = 0; fork < forks; fork++) { - // Not push(...): spreading a time-based run's samples overflows the stack. - samples = samples.concat(await runFork(uri)); + const next = await runFork(uri, result?.iterationsPerRound); + result = poolForkSamples(uri, result, next); } - return samples; + return result ?? { samples: [], iterationsPerRound: 1 }; } /** {@link runForks} for synchronous runs, which block anyway. */ -export function runForksSync(uri: string, forks: number): number[] { - let samples: number[] = []; +export function runForksSync(uri: string, forks: number): ForkSamples { + let result: ForkSamples | undefined; for (let fork = 0; fork < forks; fork++) { - samples = samples.concat(runForkSync(uri)); + const next = runForkSync(uri, result?.iterationsPerRound); + result = poolForkSamples(uri, result, next); + } + return result ?? { samples: [], iterationsPerRound: 1 }; +} + +function poolForkSamples( + uri: string, + pooled: ForkSamples | undefined, + next: ForkSamples, +): ForkSamples { + if (!pooled) return next; + if (pooled.iterationsPerRound !== next.iterationsPerRound) { + throw new Error( + `[CodSpeed] the forks of ${uri} measured a different number of iterations per round (${pooled.iterationsPerRound} and ${next.iterationsPerRound})`, + ); } - return samples; + // Not push(...): spreading a time-based run's samples overflows the stack. + for (const sample of next.samples) pooled.samples.push(sample); + return pooled; } /** A fresh file per fork: overlapping runForks() calls must not share one. */ @@ -137,12 +192,15 @@ function newSamplesFile(): string { } /** Runs one fork of `uri` and returns the samples it wrote. */ -async function runFork(uri: string): Promise { +async function runFork( + uri: string, + iterationsPerRound?: number, +): Promise { const samplesFile = newSamplesFile(); const [executable, ...args] = currentCommand(); const child = spawn(executable, args, { stdio: "inherit", - env: forkEnv(uri, samplesFile), + env: forkEnv(uri, samplesFile, iterationsPerRound), }); const { promise, resolve, reject } = Promise.withResolvers<[number | null, NodeJS.Signals | null]>(); @@ -152,12 +210,12 @@ async function runFork(uri: string): Promise { return forkSamples(uri, samplesFile, status, signal); } -function runForkSync(uri: string): number[] { +function runForkSync(uri: string, iterationsPerRound?: number): ForkSamples { const samplesFile = newSamplesFile(); const [executable, ...args] = currentCommand(); const result = spawnSync(executable, args, { stdio: "inherit", - env: forkEnv(uri, samplesFile), + env: forkEnv(uri, samplesFile, iterationsPerRound), }); if (result.error !== undefined) { throw startError(uri, result.error); @@ -165,11 +223,16 @@ function runForkSync(uri: string): number[] { return forkSamples(uri, samplesFile, result.status, result.signal); } -function forkEnv(uri: string, samplesFile: string): NodeJS.ProcessEnv { +function forkEnv( + uri: string, + samplesFile: string, + iterationsPerRound?: number, +): NodeJS.ProcessEnv { return { ...process.env, [ISOLATED_BENCHMARK_ENV]: uri, [FORK_SAMPLES_FILE_ENV]: samplesFile, + [FORK_ITERATIONS_PER_ROUND_ENV]: iterationsPerRound?.toString(), }; } @@ -185,7 +248,7 @@ function forkSamples( samplesFile: string, status: number | null, signal: NodeJS.Signals | null, -): number[] { +): ForkSamples { try { if (status !== 0) { throw new Error( @@ -193,7 +256,7 @@ function forkSamples( ); } try { - return deserialize(gunzipSync(readFileSync(samplesFile))) as number[]; + return deserialize(gunzipSync(readFileSync(samplesFile))) as ForkSamples; } catch (error) { throw new Error( `[CodSpeed] the process for ${uri} did not report its samples: ${(error as Error).message}`, @@ -204,10 +267,19 @@ function forkSamples( } } -/** Hands round samples (milliseconds) to the parent as gzip-compressed V8 data. */ -export function writeForkSamples(samplesFile: string, samples: number[]) { +/** + * Hands round samples (milliseconds) to the parent as gzip-compressed V8 data, + * along with the number of benchmark iterations one round measured. + */ +export function writeForkSamples( + samplesFile: string, + samples: number[], + iterationsPerRound = 1, +) { writeFileSync( samplesFile, - gzipSync(serialize(samples), { level: constants.Z_BEST_SPEED }), + gzipSync(serialize({ samples, iterationsPerRound }), { + level: constants.Z_BEST_SPEED, + }), ); } diff --git a/packages/core/tests/fixtures/isolation.cjs b/packages/core/tests/fixtures/isolation.cjs index bd2d6d2e..f6bf8500 100644 --- a/packages/core/tests/fixtures/isolation.cjs +++ b/packages/core/tests/fixtures/isolation.cjs @@ -7,6 +7,8 @@ const { } = require("../../dist/index.cjs.js"); const mode = walltimeIsolation(); +/** The count a first fork picks; the forks after it inherit it from the parent. */ +const CALIBRATED = Number(process.env.ISOLATION_FIXTURE_ITERATIONS ?? 1); function measure() { console.log(`ran ${process.pid} ${JSON.stringify(process.execArgv)}`); @@ -16,7 +18,21 @@ function measure() { case "no-samples": process.exit(0); } - writeForkSamples(mode.samplesFile, [Number(process.pid), 1]); + const inherited = mode.iterationsPerRound; + // Tells the test whether this fork picked the count or reused the first one. + console.log( + `${inherited === undefined ? "calibrated" : "reused"} ${inherited ?? CALIBRATED}`, + ); + const iterationsPerRound = + inherited !== undefined && + process.env.ISOLATION_FIXTURE_ACTION === "mismatch" + ? inherited + 1 // batches differently than the first fork + : (inherited ?? CALIBRATED); + writeForkSamples( + mode.samplesFile, + [Number(process.pid), 1], + iterationsPerRound, + ); } async function main() { @@ -25,11 +41,12 @@ async function main() { return; } const forks = Number(process.env.ISOLATION_FIXTURE_FORKS); - const samples = + const result = process.env.ISOLATION_FIXTURE_SYNC !== undefined ? runForksSync("a", forks) : await runForks("a", forks); - console.log(`samples ${samples.length}`); + console.log(`samples ${result.samples.length}`); + console.log(`iterations ${result.iterationsPerRound}`); } // A rejection ends the process with exit code 1. diff --git a/packages/core/tests/isolation.integ.test.ts b/packages/core/tests/isolation.integ.test.ts index c61d3f45..fc5866c1 100644 --- a/packages/core/tests/isolation.integ.test.ts +++ b/packages/core/tests/isolation.integ.test.ts @@ -1,6 +1,7 @@ import { spawnSync } from "child_process"; import { join } from "path"; import { + FORK_ITERATIONS_PER_ROUND_ENV, FORK_SAMPLES_FILE_ENV, ISOLATED_BENCHMARK_ENV, PROCESS_ISOLATION_ENV, @@ -15,6 +16,10 @@ const RUN_TIMEOUT_MS = 60_000; const CHILD_EXIT_CODE = 3; const RAN_LINE = /^ran (\d+) (.+)$/; const SAMPLES_LINE = /^samples (\d+)$/; +/** A fork tells whether it picked the iterations per round or reused them. */ +const CALIBRATED_LINE = /^calibrated (\d+)$/; +const REUSED_LINE = /^reused (\d+)$/; +const ITERATIONS_LINE = /^iterations (\d+)$/; /** Passed to Node to check that forks keep the flags of their parent. */ const NODE_FLAG = "--no-warnings"; /** Each fork of the fixture reports two samples. */ @@ -24,6 +29,7 @@ const ENV_KEYS = [ PROCESS_ISOLATION_ENV, ISOLATED_BENCHMARK_ENV, FORK_SAMPLES_FILE_ENV, + FORK_ITERATIONS_PER_ROUND_ENV, WALLTIME_FORKS_ENV, ]; const savedEnv = Object.fromEntries( @@ -104,19 +110,48 @@ describe("walltimeIsolation", () => { samplesFile: "samples.json", }); }); + + it("tells a fork the iterations per round of the first fork", () => { + setEnv({ + [ISOLATED_BENCHMARK_ENV]: "file::bench", + [FORK_SAMPLES_FILE_ENV]: "samples.json", + [FORK_ITERATIONS_PER_ROUND_ENV]: "4", + }); + expect(walltimeIsolation()).toEqual({ + kind: "Fork", + uri: "file::bench", + samplesFile: "samples.json", + iterationsPerRound: 4, + }); + }); + + it.each(["0", "-1", "1.5", "abc"])( + "rejects the iterations per round %p", + (value) => { + setEnv({ + [ISOLATED_BENCHMARK_ENV]: "file::bench", + [FORK_SAMPLES_FILE_ENV]: "samples.json", + [FORK_ITERATIONS_PER_ROUND_ENV]: value, + }); + expect(() => walltimeIsolation()).toThrow(FORK_ITERATIONS_PER_ROUND_ENV); + }, + ); }); interface FixtureOptions { forks: number; - action?: "exit" | "no-samples"; + /** The iterations per round the first fork reports. Defaults to 1. */ + iterations?: number; + action?: "exit" | "no-samples" | "mismatch"; sync?: boolean; } -function runFixture({ forks, action, sync }: FixtureOptions) { +function runFixture({ forks, iterations, action, sync }: FixtureOptions) { const env: NodeJS.ProcessEnv = { ...process.env, ISOLATION_FIXTURE_ACTION: action, ISOLATION_FIXTURE_FORKS: String(forks), + ISOLATION_FIXTURE_ITERATIONS: iterations?.toString(), }; for (const key of ENV_KEYS) { delete env[key]; @@ -141,9 +176,19 @@ function runFixture({ forks, action, sync }: FixtureOptions) { stderr: result.stderr, ran, samples: samples ? Number(samples[1]) : undefined, + calibrated: counts(CALIBRATED_LINE, lines), + reused: counts(REUSED_LINE, lines), + iterations: counts(ITERATIONS_LINE, lines)[0], }; } +function counts(pattern: RegExp, lines: string[]): number[] { + return lines + .map((line) => pattern.exec(line)) + .filter((match): match is RegExpExecArray => match !== null) + .map((match) => Number(match[1])); +} + describe.each([ { name: "async", sync: false }, { name: "sync", sync: true }, @@ -161,6 +206,44 @@ describe.each([ expect(samples).toBe(forks * SAMPLES_PER_FORK); }); + it("reports one iteration per round when nothing batches", () => { + const { status, calibrated, reused, iterations } = runFixture({ + forks: 2, + sync, + }); + + expect(status).toBe(0); + expect(calibrated).toEqual([1]); + expect(reused).toEqual([1]); + expect(iterations).toBe(1); + }); + + it("runs the forks after the first one with its iterations per round", () => { + const forks = 3; + const { status, calibrated, reused, iterations } = runFixture({ + forks, + sync, + iterations: 4, + }); + + expect(status).toBe(0); + expect(calibrated).toEqual([4]); + expect(reused).toEqual(Array(forks - 1).fill(4)); + expect(iterations).toBe(4); + }); + + it("fails when a fork measures a different number of iterations per round", () => { + const { status, stderr } = runFixture({ + forks: 2, + sync, + iterations: 2, + action: "mismatch", + }); + + expect(status).not.toBe(0); + expect(stderr).toContain("a different number of iterations per round"); + }); + it("fails on a failing fork and runs no further fork", () => { const { status, stderr, ran, samples } = runFixture({ forks: 3, diff --git a/packages/tinybench-plugin/src/batching.ts b/packages/tinybench-plugin/src/batching.ts new file mode 100644 index 00000000..b58804a2 --- /dev/null +++ b/packages/tinybench-plugin/src/batching.ts @@ -0,0 +1,107 @@ +import { msToNs } from "@codspeed/core"; +import { Bench, Fn, FnOptions, Task } from "tinybench"; +import { getBenchOptions } from "./benchOptions"; + +const MIN_ROUND_TIME_MS = 1; +export const MIN_ROUND_TIME_NS = msToNs(MIN_ROUND_TIME_MS); + +/** Round samples are batch averages, not individual-call latency distributions. */ +export function batchTask( + bench: Bench, + fn: Fn, + options: FnOptions | undefined, + inheritedIterations?: number, +) { + const state = { iterationsPerRound: inheritedIterations ?? 1 }; + if ( + options?.beforeEach || + options?.afterEach || + bench.concurrency === "task" + ) { + return { iterationsPerRound: 1, fn, options }; + } + + const { now } = getBenchOptions(bench); + let active = false; + let warmingUp = false; + let calibrated = inheritedIterations !== undefined; + let elapsed = 0; + let calls = 0; + + function finishCalibration() { + if (elapsed > 0) { + state.iterationsPerRound = Math.max( + 1, + Math.ceil((MIN_ROUND_TIME_MS * calls) / elapsed), + ); + } + calibrated = true; + } + function runBatch() { + const result = fn(); + if (result instanceof Promise) { + return (async () => { + await result; + for (let i = 1; i < state.iterationsPerRound; i++) await fn(); + })(); + } + for (let i = 1; i < state.iterationsPerRound; i++) fn(); + return result; + } + + function measure(): void | Promise { + const iterations = state.iterationsPerRound; + const start = now(); + const result = runBatch(); + function record() { + elapsed += now() - start; + calls += iterations; + if (!calibrated && elapsed >= MIN_ROUND_TIME_MS) finishCalibration(); + } + if (result instanceof Promise) return result.then(record); + record(); + } + + function prepare(mode?: "warmup" | "run"): void | Promise { + active = true; + warmingUp = mode === "warmup"; + if (warmingUp || inheritedIterations !== undefined) return; + if (calls > 0) { + finishCalibration(); + return; + } + // With warmup disabled, calibrate after setup/beforeAll, outside timed rounds. + // Wall-clock bounds also terminate calibration with a coarse/custom timer. + const deadline = performance.now() + MIN_ROUND_TIME_MS; + function calibrate(): void | Promise { + do { + const result = measure(); + if (result instanceof Promise) { + return result.then(() => { + if (performance.now() < deadline) return calibrate(); + finishCalibration(); + }); + } + } while (performance.now() < deadline); + finishCalibration(); + } + return calibrate(); + } + + return Object.assign(state, { + fn: () => { + // Tinybench may call the function at registration to detect promises. + if (!active) return fn(); + if (warmingUp) return measure(); + return runBatch(); + }, + options: { + ...options, + beforeAll(this: Task, mode?: "warmup" | "run") { + const result = options?.beforeAll?.call(this, mode); + if (result instanceof Promise) return result.then(() => prepare(mode)); + return prepare(mode); + }, + }, + }); +} diff --git a/packages/tinybench-plugin/src/benchOptions.ts b/packages/tinybench-plugin/src/benchOptions.ts index a1ffef42..a95b6ae0 100644 --- a/packages/tinybench-plugin/src/benchOptions.ts +++ b/packages/tinybench-plugin/src/benchOptions.ts @@ -4,6 +4,7 @@ import { Bench, Hook } from "tinybench"; // setup/teardown hooks. tinybench guarantees `setup`/`teardown` are populated // (at least with no-op defaults), so they are non-optional here. export interface ResolvedBenchOptions { + now: () => number; setup: Hook; teardown: Hook; warmup?: boolean; diff --git a/packages/tinybench-plugin/src/index.ts b/packages/tinybench-plugin/src/index.ts index bb903ae0..fbba5cda 100644 --- a/packages/tinybench-plugin/src/index.ts +++ b/packages/tinybench-plugin/src/index.ts @@ -7,17 +7,25 @@ import { SetupInstrumentsRequestBody, SetupInstrumentsResponse, tryIntrospect, + walltimeIsolation, wrapWithRootFrameSync, type WalltimeOptions, } from "@codspeed/core"; import { Bench } from "tinybench"; import { setupCodspeedAnalysisBench } from "./analysis"; +import { batchTask } from "./batching"; import { getOrCreateTaskDataMap } from "./taskData"; import { getOrCreateUriMap } from "./uri"; import { setupCodspeedWalltimeBench } from "./walltime"; tryIntrospect(); +/** + * Walltime rounds batch sequential invocations toward a 1 ms target, estimated + * from warmup (or a short calibration when disabled). Forks share one batch size. + * Per-invocation hooks and task concurrency stay unbatched. CodSpeed reports + * per-invocation batch averages; total time is the sum of measured round durations. + */ export function withCodSpeed( bench: Bench, options: WalltimeOptions = {}, @@ -42,15 +50,24 @@ export function withCodSpeed( } uri += `::${name}`; uriMap.set(name, uri); - taskDataMap.set(name, { fn, fnOpts: opts }); - // In walltime mode the task is driven by tinybench's own measured loop, so - // the root frame must be baked into the function tinybench stores rather - // than injected later (the function is an inaccessible private field on the - // task from tinybench v6 onwards). The sync wrapper is transparent to the - // function's return value, so it preserves tinybench's sync/async handling. - const registeredFn = - instrumentMode === "walltime" ? wrapWithRootFrameSync(fn) : fn; - return rawAdd.bind(bench)(name, registeredFn, opts); + if (instrumentMode !== "walltime") { + taskDataMap.set(name, { fn, fnOpts: opts }); + return rawAdd.call(bench, name, fn, opts); + } + const isolation = walltimeIsolation(options.forks); + let inheritedIterations: number | undefined; + if (isolation.kind === "Fork") { + inheritedIterations = isolation.iterationsPerRound; + } + const batching = batchTask(bench, fn, opts, inheritedIterations); + taskDataMap.set(name, { fn, fnOpts: opts, batching }); + // Tinybench v6 stores the function in a private field, so wrap at registration. + return rawAdd.call( + bench, + name, + wrapWithRootFrameSync(batching.fn), + batching.options, + ); }; if (instrumentMode === "analysis") { diff --git a/packages/tinybench-plugin/src/taskData.ts b/packages/tinybench-plugin/src/taskData.ts index a5ecb2ab..58eedd82 100644 --- a/packages/tinybench-plugin/src/taskData.ts +++ b/packages/tinybench-plugin/src/taskData.ts @@ -7,6 +7,7 @@ import { Bench, Fn, FnOptions } from "tinybench"; export interface CapturedTaskData { fn: Fn; fnOpts?: FnOptions; + batching?: { iterationsPerRound: number }; } const taskDataMap = new WeakMap>(); diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index acf8837b..8d0ac694 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -18,6 +18,7 @@ import { type WalltimeOptions, } from "@codspeed/core"; import { Bench, Task } from "tinybench"; +import { MIN_ROUND_TIME_NS } from "./batching"; import { getBenchOptions } from "./benchOptions"; import { BaseBenchRunner } from "./shared"; @@ -102,8 +103,15 @@ class WalltimeBenchRunner extends BaseBenchRunner { protected async runTaskAsync(task: Task, uri: string): Promise { const { isolationMode } = this; if (isolationMode.kind === "Forks") { - const samples = await runForks(uri, isolationMode.forks); - return this.registerCodspeedBenchmark(task, samplesToResult(samples)); + const { samples, iterationsPerRound } = await runForks( + uri, + isolationMode.forks, + ); + return this.registerCodspeedBenchmark( + task, + samplesToResult(samples), + iterationsPerRound, + ); } // run the warmup of the task right before its actual run @@ -121,8 +129,15 @@ class WalltimeBenchRunner extends BaseBenchRunner { protected runTaskSync(task: Task, uri: string): void { const { isolationMode } = this; if (isolationMode.kind === "Forks") { - const samples = runForksSync(uri, isolationMode.forks); - return this.registerCodspeedBenchmark(task, samplesToResult(samples)); + const { samples, iterationsPerRound } = runForksSync( + uri, + isolationMode.forks, + ); + return this.registerCodspeedBenchmark( + task, + samplesToResult(samples), + iterationsPerRound, + ); } if (getBenchOptions(this.bench).warmup) { @@ -149,18 +164,25 @@ class WalltimeBenchRunner extends BaseBenchRunner { ); return; } + const iterationsPerRound = + this.getTaskData(task).batching?.iterationsPerRound ?? 1; // A fork hands its samples to the parent, which reports the benchmark. if (this.isolationMode.kind === "Fork") { writeForkSamples( this.isolationMode.samplesFile, task.result.latency.samples, + iterationsPerRound, ); return; } - this.registerCodspeedBenchmark(task, task.result); + this.registerCodspeedBenchmark(task, task.result, iterationsPerRound); } - private registerCodspeedBenchmark(task: Task, result: RoundsResult): void { + private registerCodspeedBenchmark( + task: Task, + result: RoundsResult, + iterationsPerRound: number, + ): void { const uri = this.getTaskUri(task); const opts = getBenchOptions(this.bench); const warmupIterations = opts.warmup @@ -169,6 +191,7 @@ class WalltimeBenchRunner extends BaseBenchRunner { const stats = convertTinybenchResultToBenchmarkStats( result, warmupIterations, + iterationsPerRound, ); this.codspeedBenchmarks.push({ @@ -177,7 +200,7 @@ class WalltimeBenchRunner extends BaseBenchRunner { config: { max_rounds: opts.iterations ?? null, max_time_ns: opts.time ? msToNs(opts.time) : null, - min_round_time_ns: null, // tinybench does not have an option for this + min_round_time_ns: iterationsPerRound > 1 ? MIN_ROUND_TIME_NS : null, warmup_time_ns: opts.warmup && opts.warmupTime ? msToNs(opts.warmupTime) : null, }, @@ -210,27 +233,30 @@ const TINYBENCH_WARMUP_DEFAULT = 16; function convertTinybenchResultToBenchmarkStats( result: RoundsResult, warmupIterations: number, + iterationsPerRound: number, ): BenchmarkStats { const { min, max, mean, sd, samples } = result.latency; // Get individual sample times in nanoseconds and sort them - const sortedTimesNs = samples.map(msToNs).sort((a, b) => a - b); - const meanNs = msToNs(mean); - const stdevNs = msToNs(sd); + const sortedTimesNs = samples + .map((sample) => msToNs(sample) / iterationsPerRound) + .sort((a, b) => a - b); + const meanNs = msToNs(mean) / iterationsPerRound; + const stdevNs = msToNs(sd) / iterationsPerRound; const { q1_ns, q3_ns, median_ns, iqr_outlier_rounds, stdev_outlier_rounds } = calculateQuantiles({ meanNs, stdevNs, sortedTimesNs }); return { - min_ns: msToNs(min), - max_ns: msToNs(max), + min_ns: msToNs(min) / iterationsPerRound, + max_ns: msToNs(max) / iterationsPerRound, mean_ns: meanNs, stdev_ns: stdevNs, q1_ns, median_ns, q3_ns, total_time: msToS(result.totalTime), - iter_per_round: 1, // as there is only one round in tinybench, we define that there were n rounds of 1 iteration + iter_per_round: iterationsPerRound, rounds: sortedTimesNs.length, iqr_outlier_rounds, stdev_outlier_rounds, diff --git a/packages/tinybench-plugin/tests/batching.integ.test.ts b/packages/tinybench-plugin/tests/batching.integ.test.ts new file mode 100644 index 00000000..e1f4f307 --- /dev/null +++ b/packages/tinybench-plugin/tests/batching.integ.test.ts @@ -0,0 +1,218 @@ +import { getV8Flags } from "@codspeed/core"; +import { spawn } from "child_process"; +import { once } from "events"; +import { mkdtempSync, readdirSync, readFileSync, rmSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { fileURLToPath } from "url"; +import { describe, expect, it } from "vitest"; + +const FIXTURE = fileURLToPath( + new URL("./fixtures/batching.mjs", import.meta.url), +); +const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +const WALLTIME_FORKS_ENV = "CODSPEED_WALLTIME_FORKS"; +const FIXTURE_SYNC_ENV = "BATCHING_FIXTURE_SYNC"; +/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ +const RUN_TIMEOUT_MS = 120_000; +const FORKS = 2; +/** Rounds the fixture's benchmarks measure per process once a round costs 1ms. */ +const ROUNDS_PER_FORK = 4; +/** The fixture's iteration cost: 0.01ms on the fake clock. */ +const BATCHED_ITERATION_NS = 10_000; +/** The lifecycle benchmark halves its iteration cost in the suite setup hook. */ +const LIFECYCLE_ITERATION_NS = 5_000; +/** The 1ms round target in units of the fixture's iteration cost. */ +const BATCHED_ITER_PER_ROUND = 100; +const LIFECYCLE_ITER_PER_ROUND = 200; +/** Allows for the fake clock's rounding, both ways, around the target count. */ +const ITERATIONS_TOLERANCE = 3; + +interface BenchmarkStats { + min_ns: number; + max_ns: number; + mean_ns: number; + rounds: number; + total_time: number; + iter_per_round: number; + warmup_iters: number; +} + +interface FixtureOptions { + /** `CODSPEED_PROCESS_ISOLATION=false`: every benchmark shares our process. */ + shared?: boolean; + /** `CODSPEED_WALLTIME_FORKS`. */ + forks?: number; + sync?: boolean; +} + +async function runFixture({ + shared, + forks, + sync, +}: FixtureOptions = {}): Promise> { + const profileFolder = mkdtempSync(join(tmpdir(), "codspeed-batching-")); + try { + const env: NodeJS.ProcessEnv = { + ...process.env, + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "walltime", + CODSPEED_PROFILE_FOLDER: profileFolder, + }; + delete env[PROCESS_ISOLATION_ENV]; + delete env[WALLTIME_FORKS_ENV]; + if (shared) { + env[PROCESS_ISOLATION_ENV] = "false"; + } + if (forks !== undefined) { + env[WALLTIME_FORKS_ENV] = String(forks); + } + if (sync) { + env[FIXTURE_SYNC_ENV] = "1"; + } + + // The runner starts Node with these flags, and forks must keep them. Not + // spawnSync: blocking the test worker times out its calls to Vitest. + const child = spawn(process.execPath, [...getV8Flags(), FIXTURE], { + env, + timeout: RUN_TIMEOUT_MS, + }); + let stdout = ""; + child.stdout.setEncoding("utf8").on("data", (chunk) => (stdout += chunk)); + const [status] = (await once(child, "close")) as [number | null]; + expect(status, stdout).toBe(0); + + const results = new Map(); + const resultsDir = join(profileFolder, "results"); + for (const file of readdirSync(resultsDir)) { + const data = JSON.parse(readFileSync(join(resultsDir, file), "utf8")) as { + benchmarks: { name: string; stats: BenchmarkStats }[]; + }; + for (const { name, stats } of data.benchmarks) { + results.set(name, stats); + } + } + return results; + } finally { + rmSync(profileFolder, { recursive: true, force: true }); + } +} + +function getStats( + results: Map, + name: string, +): BenchmarkStats { + const stats = results.get(name); + if (!stats) { + throw new Error(`no results for ${name}`); + } + return stats; +} + +/** Every sample is one iteration, not one batch of iterations. */ +function expectPerInvocationStats( + stats: BenchmarkStats, + perInvocationNs: number, +): void { + expect(Math.round(stats.mean_ns)).toBe(perInvocationNs); + expect(Math.round(stats.min_ns)).toBe(perInvocationNs); + expect(Math.round(stats.max_ns)).toBe(perInvocationNs); + // The total stays the raw batched time: rounds x iterations x per-iteration. + expect(stats.total_time).toBeCloseTo( + (stats.rounds * stats.iter_per_round * stats.mean_ns) / 1e9, + 9, + ); +} + +function expectBatchedStats( + stats: BenchmarkStats, + { + rounds, + iterationsPerRound = BATCHED_ITER_PER_ROUND, + perInvocationNs = BATCHED_ITERATION_NS, + }: { + rounds: number; + iterationsPerRound?: number; + perInvocationNs?: number; + }, +): void { + expect(stats.iter_per_round).toBeGreaterThanOrEqual( + iterationsPerRound - ITERATIONS_TOLERANCE, + ); + expect(stats.iter_per_round).toBeLessThanOrEqual( + iterationsPerRound + ITERATIONS_TOLERANCE, + ); + expect(stats.rounds).toBe(rounds); + expectPerInvocationStats(stats, perInvocationNs); +} + +function expectHookedStats( + stats: BenchmarkStats, + { forks }: { forks: number }, +): void { + expect(stats.iter_per_round).toBe(1); + // Each 0.01ms call is timed on its own, so tinybench's 1ms budget is spent. + expect(stats.rounds).toBeGreaterThanOrEqual(98 * forks); + expect(stats.rounds).toBeLessThanOrEqual(102 * forks); + expectPerInvocationStats(stats, BATCHED_ITERATION_NS); +} + +describe("walltime batching", () => { + it.each([ + { name: "async", sync: false }, + { name: "sync", sync: true }, + ])( + "batches fast benchmarks and normalizes their stats in the shared process ($name run)", + async ({ sync }) => { + const stats = await runFixture({ shared: true, sync }); + + expectBatchedStats(getStats(stats, "batched"), { + rounds: ROUNDS_PER_FORK, + }); + // The suite setup hook halves the iteration cost before calibration. + expectBatchedStats(getStats(stats, "lifecycle"), { + rounds: ROUNDS_PER_FORK, + iterationsPerRound: LIFECYCLE_ITER_PER_ROUND, + perInvocationNs: LIFECYCLE_ITERATION_NS, + }); + expect(getStats(stats, "lifecycle").warmup_iters).toBe(0); + expectHookedStats(getStats(stats, "hooked"), { forks: 1 }); + + if (!sync) { + // An overlapping async round would have failed the fixture run. + expectBatchedStats(getStats(stats, "async"), { + rounds: ROUNDS_PER_FORK, + }); + } + }, + RUN_TIMEOUT_MS, + ); + + it.each([ + { name: "async", sync: false }, + { name: "sync", sync: true }, + ])( + "batches and normalizes the pooled samples of several forks ($name run)", + async ({ sync }) => { + const stats = await runFixture({ forks: FORKS, sync }); + + expectBatchedStats(getStats(stats, "batched"), { + rounds: FORKS * ROUNDS_PER_FORK, + }); + expectBatchedStats(getStats(stats, "lifecycle"), { + rounds: FORKS * ROUNDS_PER_FORK, + iterationsPerRound: LIFECYCLE_ITER_PER_ROUND, + perInvocationNs: LIFECYCLE_ITERATION_NS, + }); + // Batched and unbatched benchmarks pool very different sample counts. + expectHookedStats(getStats(stats, "hooked"), { forks: FORKS }); + + if (!sync) { + expectBatchedStats(getStats(stats, "async"), { + rounds: FORKS * ROUNDS_PER_FORK, + }); + } + }, + RUN_TIMEOUT_MS, + ); +}); diff --git a/packages/tinybench-plugin/tests/fixtures/batching.mjs b/packages/tinybench-plugin/tests/fixtures/batching.mjs new file mode 100644 index 00000000..4e149b2a --- /dev/null +++ b/packages/tinybench-plugin/tests/fixtures/batching.mjs @@ -0,0 +1,106 @@ +// Run by `batching.integ.test.ts`. A deterministic clock makes batching +// observable: every benchmark iteration costs exactly one tick, wherever +// tinybench or the plugin reads the clock. +import { Bench } from "tinybench"; +import { withCodSpeed } from "../../dist/index.es5.js"; + +/** Milliseconds one benchmark iteration costs on the fake clock. */ +const ITERATION_MS = 0.01; +/** tinybench stops after this many rounds once a round costs 1ms. */ +const ROUNDS = 4; +const sync = process.env.BATCHING_FIXTURE_SYNC !== undefined; + +// Warmup enabled: the plugin learns the per-iteration cost from the warmup +// rounds and then batches ~1ms worth of iterations per measured round. +let tickA = 0; +const costA = ITERATION_MS; +const benchA = withCodSpeed( + new Bench({ iterations: ROUNDS, time: 1, warmupTime: 0, now: () => tickA }), +); + +benchA.add("batched", () => { + tickA += costA; +}); + +// A batched async benchmark must await every iteration of a round: overlapping +// iterations are a bug, and they fail the run instead of skewing the stats. +let inFlight = 0; +if (!sync) { + benchA.add("async", async () => { + if (inFlight++ > 0) { + throw new Error("async iterations overlapped"); + } + tickA += costA; + await Promise.resolve(); + inFlight--; + }); +} + +// beforeEach/afterEach are only correct when tinybench times each call alone. +let beforeEachCalls = 0; +let afterEachCalls = 0; +let hookedCalls = 0; +benchA.add( + "hooked", + () => { + if (beforeEachCalls > 0 && beforeEachCalls !== hookedCalls + 1) { + throw new Error("beforeEach did not run for this invocation"); + } + hookedCalls++; + tickA += costA; + }, + { + beforeAll: () => { + beforeEachCalls = 0; + afterEachCalls = 0; + hookedCalls = 0; + }, + beforeEach: () => { + beforeEachCalls++; + }, + afterEach: () => { + afterEachCalls++; + }, + afterAll: () => { + if (beforeEachCalls !== hookedCalls || afterEachCalls !== hookedCalls) { + throw new Error("afterEach did not run for every invocation"); + } + }, + }, +); + +// Warmup disabled: the plugin calibrates inside the run lifecycle, after the +// suite setup hook has run (which is what decides the iteration cost here). +let tickB = 0; +let costB = ITERATION_MS; +let lifecycleRuns = 0; +const benchB = withCodSpeed( + new Bench({ + iterations: ROUNDS, + time: 1, + warmup: false, + now: () => tickB, + setup: (_task, mode) => { + if (mode === "run") { + lifecycleRuns++; + costB = ITERATION_MS / 2; + } + }, + }), +); +benchB.add("lifecycle", () => { + tickB += costB; +}); + +if (sync) { + benchA.runSync(); + benchB.runSync(); +} else { + await benchA.run(); + await benchB.run(); +} + +// A fork that owns another benchmark's URI does not run this suite at all. +if (lifecycleRuns > 1) { + throw new Error(`ran the lifecycle ${lifecycleRuns} times`); +} diff --git a/packages/vitest-plugin/src/v5/forkReporter.ts b/packages/vitest-plugin/src/v5/forkReporter.ts index 7055eb7b..89731ab0 100644 --- a/packages/vitest-plugin/src/v5/forkReporter.ts +++ b/packages/vitest-plugin/src/v5/forkReporter.ts @@ -41,7 +41,7 @@ async function measureInForks( for (const test of module.children.allTests()) { for (const queued of forkedBenchmarks(test.meta())) { const forks = resolveWalltimeForks(queued.forks ?? options.forks); - const samples = await runForks(queued.uri, forks); + const { samples } = await runForks(queued.uri, forks); const benchmark = tinybenchTaskToBenchmark( { name: queued.name, result: samplesToResult(samples) }, queued.uri, diff --git a/packages/vitest-plugin/src/v5/provider.ts b/packages/vitest-plugin/src/v5/provider.ts index bd15ae80..d047af70 100644 --- a/packages/vitest-plugin/src/v5/provider.ts +++ b/packages/vitest-plugin/src/v5/provider.ts @@ -2,11 +2,11 @@ import { getGitDir, getInstrumentMode, InstrumentHooks, - walltimeIsolation, optimizeFunction, optimizeFunctionSync, setupCore, teardownCore, + walltimeIsolation, wrapWithRootFrame, wrapWithRootFrameSync, writeForkSamples,