From 5fe8c595d8a6e8947d3433202c89e975f82bf11d Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Wed, 2 Sep 2026 10:39:02 +0100 Subject: [PATCH 1/3] Stabilise walltime benchmarks across runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CodSpeed walltime legs report the same code up to 2x apart from one run to the next: `add_shapes_neon[(128, PerRowPerRow)]` reads ~1.8 µs on most develop pushes and ~2.9-3.7 µs on some, across commits that do not touch the benchmark, so a PR whose base and head land in different runs is reported as a change it did not make. Two separate things are going on, and this fixes both. The first is a fixed per-sample cost that differs by host. In a slow run every benchmark in the neon `binary_ops` binary is slower by the same ~1.2 µs whatever its length (1.8 to 2.9 µs, 8.3 to 9.6, 9.3 to 10.5), with the timer precision unchanged at 29 ns. The runner brackets each sample with its own timestamps and marker hooks, and divan sizes samples by its first, cold run, so everything over a few microseconds was measured one iteration at a time with that overhead landing on it. `#[cpu_features]` now requires `sample_size` in `#[divan::bench(..)]`, chosen so that a sample lasts ~100 µs or more, and injects the 1,000-sample count itself: an env `DIVAN_SAMPLE_COUNT` or `DIVAN_SAMPLE_SIZE` overrides the attribute, so the workflow no longer sets one. The second is layout: `arrow_checked_add_u32_neon[16384]` reads 13 µs or 20 µs depending on where the process's memory lands. On one c7g.metal host with ASLR turned back on, it landed in either about half the time on the main thread, and held 13 µs through every run on a spawned thread; the same across a sweep of the environment block's size, which moved nothing on either thread. `#[vortex_bench_support::main]` runs the benchmark binary's body on a spawned thread with an 8 MiB stack, and the walltime job refuses a tagged binary whose `fn main` lacks it. Sweep: https://github.com/vortex-data/vortex/actions/runs/33615899226 Signed-off-by: Joe Isaacs Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YV1PdJJrd7r1Zcts63RrWa --- .github/workflows/codspeed.yml | 37 ++++--- benchmarks/bench-support/src/lib.rs | 138 ++++++++++++++++++++++-- vortex-array/benches/binary_ops.rs | 41 +++---- vortex-array/benches/compare.rs | 17 +-- vortex-array/benches/row_fn_output.rs | 11 +- vortex-array/benches/scalar_subtract.rs | 3 +- vortex-buffer/benches/collect_bool.rs | 5 +- vortex-compute/benches/lane_kernels.rs | 5 +- 8 files changed, 196 insertions(+), 61 deletions(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 7e98e37f5bc..9138245b95e 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -144,6 +144,10 @@ jobs: # crate that adds an `#[cpu_features]` benchmark is picked up without editing this workflow. # Building the whole workspace would do the same, but links every bench binary in it # — with debuginfo, from the bench profile — to measure only the tagged ones. + # + # A tagged binary must also run under `#[vortex_bench_support::main]`: ASLR is off on + # these runners, so without it the numbers depend on the size of the environment + # block, which differs between a push and a pull request. See that attribute's docs. - name: Select packages with tagged benchmarks id: select run: | @@ -152,16 +156,23 @@ jobs: meta = json.loads(subprocess.check_output( ["cargo", "metadata", "--no-deps", "--format-version", "1"])) - tagged = sorted( - package["name"] - for package in meta["packages"] - for target in package["targets"] - if "bench" in target["kind"] - and "cpu_features]" in pathlib.Path(target["src_path"]).read_text() - ) + tagged = {} + for package in meta["packages"]: + for target in package["targets"]: + if "bench" not in target["kind"]: + continue + source = pathlib.Path(target["src_path"]).read_text() + if "cpu_features]" in source: + tagged[target["src_path"]] = (package["name"], "bench_support::main]" in source) if not tagged: raise SystemExit("no benchmark carries `#[cpu_features]`; this job has nothing to measure") - print("packages=" + " ".join(f"-p {name}" for name in dict.fromkeys(tagged))) + unwrapped = sorted(path for path, (_, wrapped) in tagged.items() if not wrapped) + if unwrapped: + raise SystemExit( + "these carry `#[cpu_features]` but their `fn main` lacks " + "`#[vortex_bench_support::main]`:\n " + "\n ".join(unwrapped)) + packages = sorted({name for name, _ in tagged.values()}) + print("packages=" + " ".join(f"-p {name}" for name in packages)) EOF - name: Build benchmarks env: @@ -185,15 +196,11 @@ jobs: # so this selects paths of the form `::::`. That middle # component only exists because `#[cpu_features]` puts it there: an untagged benchmark has one # component fewer and cannot match, whatever it is called. + # How many samples to take and how many iterations each one runs are set on the + # benchmarks by `#[cpu_features]`, not here: a `DIVAN_SAMPLE_COUNT` or + # `DIVAN_SAMPLE_SIZE` in the environment would override every one of them. - name: Run benchmarks uses: CodSpeedHQ/action@4296e51e7041e24dadb86d1d6e8b9320d223dbe8 # v5 - env: - # divan's default of 100 samples leaves these benchmarks too noisy to compare - # across runs: the same commit measured twice varied by up to 2.2x on the 1,024 - # element cases and ~50-70% on the 65,536 element ones. At 1,000 samples the same - # experiment stays within ~6-10%, and the suite still runs in seconds, so the extra - # sampling is close to free next to the minute-plus spent building it. - DIVAN_SAMPLE_COUNT: "1000" with: run: bash scripts/bench-taskset.sh cargo codspeed run -- '.*::${{ matrix.features }}::' token: ${{ secrets.CODSPEED_TOKEN }} diff --git a/benchmarks/bench-support/src/lib.rs b/benchmarks/bench-support/src/lib.rs index 912605c781e..dd2461d7248 100644 --- a/benchmarks/bench-support/src/lib.rs +++ b/benchmarks/bench-support/src/lib.rs @@ -3,15 +3,21 @@ //! Support for benchmarks measured once per CPU feature set. //! -//! One attribute, [`cpu_features`]. Which feature sets exist, what each is built with, and -//! where it runs are all in `.github/workflows/codspeed.yml`. +//! Two attributes. [`cpu_features`] marks a benchmark as measured on every walltime leg, and +//! [`main`] goes on the `fn main` of a binary carrying such benchmarks. Which feature sets +//! exist, what each is built with, and where it runs are all in `.github/workflows/codspeed.yml`. use proc_macro::TokenStream; use proc_macro2::TokenStream as TokenStream2; use quote::quote; use syn::ItemFn; +use syn::ReturnType; use syn::parse_macro_input; +/// Stack size of the thread [`main`] runs the benchmarks on, matching the 8 MiB Linux gives the +/// main thread so nothing that fit before overflows now. +const BENCH_THREAD_STACK_SIZE: usize = 8 << 20; + /// Measure this benchmark on every walltime CPU-feature leg instead of in simulation. /// /// Takes no argument: the benchmark runs on all of them. Write it *above* `#[divan::bench]`, @@ -23,12 +29,24 @@ use syn::parse_macro_input; /// /// ```ignore /// #[vortex_bench_support::cpu_features] -/// #[divan::bench(args = INPUT_SIZE)] +/// #[divan::bench(args = INPUT_SIZE, sample_size = 256)] /// fn words_gather_dispatch(bencher: Bencher, len: usize) { /* ... */ } /// ``` /// +/// `sample_size` is required: it is how many times one sample runs the benchmark, and the +/// number reported is the sample's time divided by it. The runner brackets every sample with +/// its own hooks and timestamps, and what that costs the sample that follows is a fixed +/// amount that differs from one host to the next by about a microsecond. Left to itself, +/// divan sizes samples by its first, cold run and settles on one iteration for anything over +/// a few microseconds, so on a bad host every case in a binary read the same ~1.2 µs slower +/// regardless of its length. Pick the size so that a sample lasts at least ~100 µs on the +/// smallest argument: 64 for a 2 µs case, 16 for a 10 µs one, 4 for 50 µs. When the arguments +/// span a wide range, add `sample_count` below the default of 1,000 so the largest one does +/// not take seconds. +/// /// Spell it out in full rather than importing it: benchmark files are read a function at a -/// time, and the path says where the behaviour comes from. +/// time, and the path says where the behaviour comes from. The binary's `fn main` carries +/// [`main`], which the walltime legs need for a repeatable layout. /// /// This is for code that is written once and *compiled* differently per feature set — a /// shipped entry point that selects its kernel through `cfg(target_feature)`, or a scalar @@ -76,12 +94,15 @@ pub fn cpu_features(attr: TokenStream, item: TokenStream) -> TokenStream { } }; - for reserved in ["name", "ignore"] { - if existing + let has_option = |option: &str| { + existing .clone() .into_iter() - .any(|token| matches!(&token, proc_macro2::TokenTree::Ident(i) if i == reserved)) - { + .any(|token| matches!(&token, proc_macro2::TokenTree::Ident(i) if i == option)) + }; + + for reserved in ["name", "ignore"] { + if has_option(reserved) { return syn::Error::new_spanned( &existing, format!( @@ -93,6 +114,27 @@ pub fn cpu_features(attr: TokenStream, item: TokenStream) -> TokenStream { } } + if !has_option("sample_size") { + return syn::Error::new_spanned( + bench, + "`#[cpu_features]` needs `sample_size` in `#[divan::bench(..)]`: enough iterations \ + per sample for one sample to last ~100 µs or more, so the runner's per-sample \ + overhead, which varies by host, is amortised (see the attribute's docs)", + ) + .to_compile_error() + .into(); + } + + // The default used to be `DIVAN_SAMPLE_COUNT` in the workflow, but a value set there beats + // one set on the benchmark, and the cases that need thousands of iterations per sample + // need fewer samples in exchange. divan's own default of 100 samples left the same commit + // measured twice up to 2.2x apart on the 1,024 element cases; 1,000 holds them to ~6-10%. + let sample_count = if has_option("sample_count") { + quote!() + } else { + quote!(sample_count = 1000,) + }; + // Skipping simulation, rather than naming the legs to run on, is what keeps the leg list // in the workflow alone. `env!` rather than reading the environment during expansion: // rustc records it as a dependency of the crate, so changing legs rebuilds the benchmarks. @@ -112,6 +154,7 @@ pub fn cpu_features(attr: TokenStream, item: TokenStream) -> TokenStream { bench.meta = syn::parse_quote! { #bench_path( #existing #separator + #sample_count name = concat!( env!("VORTEX_BENCH_PREFIX"), #name, @@ -123,3 +166,82 @@ pub fn cpu_features(attr: TokenStream, item: TokenStream) -> TokenStream { quote!(#function).into() } + +/// Run the benchmarks on a thread of their own, so where their stack and heap land is set by +/// the binary rather than by the process. +/// +/// Write it on `fn main`, which must take no arguments and return `()`: +/// +/// ```ignore +/// #[vortex_bench_support::main] +/// fn main() { +/// divan::main(); +/// } +/// ``` +/// +/// Every binary with a [`cpu_features`] benchmark needs it; the workflow refuses to measure +/// one without it. The body runs unchanged on a spawned thread with a fixed-size stack, and a +/// panic there is re-raised on the main thread so the exit status is what it was. +/// +/// Some µs-scale cases have two stable timings up to 1.5× apart, and which one a run lands in +/// depends on the layout of the process rather than on the code: `arrow_checked_add_u32` in +/// `vortex-compute` read 13 µs or 20 µs on the neon leg from one run to the next. On the main +/// thread, with the address space randomised, it landed in either about half the time; on a +/// spawned thread it held one timing through every one of those runs, and through a sweep of +/// the environment block's size (which, contrary to the obvious guess, did not move it on +/// either thread). The spawned thread's stack is a fresh mapping and the allocators in use +/// serve it from mappings of their own, so its layout is set by the binary alone. +#[proc_macro_attribute] +pub fn main(attr: TokenStream, item: TokenStream) -> TokenStream { + if !attr.is_empty() { + let attr = TokenStream2::from(attr); + return syn::Error::new_spanned(attr, "`#[vortex_bench_support::main]` takes no argument") + .to_compile_error() + .into(); + } + + let function = parse_macro_input!(item as ItemFn); + let signature = &function.sig; + + if signature.ident != "main" { + return syn::Error::new_spanned( + &signature.ident, + "`#[vortex_bench_support::main]` goes on `fn main`", + ) + .to_compile_error() + .into(); + } + if !signature.inputs.is_empty() + || !matches!(signature.output, ReturnType::Default) + || signature.asyncness.is_some() + { + return syn::Error::new_spanned( + signature, + "`#[vortex_bench_support::main]` expects `fn main()` with no arguments and no return type", + ) + .to_compile_error() + .into(); + } + + let attrs = &function.attrs; + let vis = &function.vis; + let body = &function.block; + + // The body becomes the closure, so a `return` in it leaves the benchmarks as before. An + // explicit stack size rather than the default keeps `RUST_MIN_STACK` from putting the + // environment back into the layout. + quote! { + #(#attrs)* + #vis fn main() -> ::std::io::Result<()> { + let benchmarks = ::std::thread::Builder::new() + .name("vortex-bench".to_owned()) + .stack_size(#BENCH_THREAD_STACK_SIZE) + .spawn(|| #body)?; + if let Err(panic) = benchmarks.join() { + ::std::panic::resume_unwind(panic); + } + Ok(()) + } + } + .into() +} diff --git a/vortex-array/benches/binary_ops.rs b/vortex-array/benches/binary_ops.rs index a1f14ad9bb1..c0b7ebe991e 100644 --- a/vortex-array/benches/binary_ops.rs +++ b/vortex-array/benches/binary_ops.rs @@ -41,6 +41,7 @@ use vortex_session::VortexSession; #[global_allocator] static GLOBAL: MiMalloc = MiMalloc; +#[vortex_bench_support::main] fn main() { LazyLock::force(&SESSION); divan::main(); @@ -92,25 +93,25 @@ enum BinaryShape { const DECIMAL_MUL_DIV_LEN: usize = 1_024; #[vortex_bench_support::cpu_features] -#[divan::bench(args = BINARY_SHAPE_CASES)] +#[divan::bench(args = BINARY_SHAPE_CASES, sample_size = 64)] fn add_shapes(bencher: Bencher, &(len, shape): &(usize, BinaryShape)) { bench_binary_shape(bencher, len, shape, Operator::Add); } #[vortex_bench_support::cpu_features] -#[divan::bench(args = CONSTANT_SHAPE_CASES)] +#[divan::bench(args = CONSTANT_SHAPE_CASES, sample_size = 16)] fn add_constant_shapes(bencher: Bencher, &(len, shape): &(usize, BinaryShape)) { bench_binary_shape(bencher, len, shape, Operator::Add); } #[vortex_bench_support::cpu_features] -#[divan::bench(args = BINARY_SHAPE_CASES)] +#[divan::bench(args = BINARY_SHAPE_CASES, sample_size = 64)] fn subtract_shapes(bencher: Bencher, &(len, shape): &(usize, BinaryShape)) { bench_binary_shape(bencher, len, shape, Operator::Sub); } #[vortex_bench_support::cpu_features] -#[divan::bench(args = BINARY_SHAPE_CASES)] +#[divan::bench(args = BINARY_SHAPE_CASES, sample_size = 64)] fn multiply_shapes(bencher: Bencher, &(len, shape): &(usize, BinaryShape)) { bench_binary_shape(bencher, len, shape, Operator::Mul); } @@ -132,7 +133,7 @@ fn bench_binary_shape(bencher: Bencher, len: usize, shape: BinaryShape, operator } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 16)] fn add_i64_nonnull(bencher: Bencher) { let lhs = primitive_nonnull(0, I64_LEN).into_array(); let rhs = primitive_nonnull(1_000_000, I64_LEN).into_array(); @@ -141,7 +142,7 @@ fn add_i64_nonnull(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 16)] fn add_i64_nullable(bencher: Bencher) { let lhs = primitive_nullable(0, 7, I64_LEN).into_array(); let rhs = primitive_nullable(1_000_000, 5, I64_LEN).into_array(); @@ -150,7 +151,7 @@ fn add_i64_nullable(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 16)] fn add_i32_nonnull(bencher: Bencher) { let lhs = primitive_i32_small_nonnull(1, I32_LEN).into_array(); let rhs = primitive_i32_small_nonnull(17, I32_LEN).into_array(); @@ -159,7 +160,7 @@ fn add_i32_nonnull(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 16)] fn add_u32_nonnull(bencher: Bencher) { let lhs = primitive_u32_small_nonnull(1, I32_LEN).into_array(); let rhs = primitive_u32_small_nonnull(17, I32_LEN).into_array(); @@ -168,7 +169,7 @@ fn add_u32_nonnull(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 8)] fn mul_i64_nonnull(bencher: Bencher) { let lhs = primitive_small_nonnull(1, I64_LEN).into_array(); let rhs = primitive_small_nonnull(17, I64_LEN).into_array(); @@ -177,7 +178,7 @@ fn mul_i64_nonnull(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 16)] fn mul_i8_nonnull(bencher: Bencher) { let lhs = primitive_i8_small_nonnull(1, I8_LEN).into_array(); let rhs = primitive_i8_small_nonnull(7, I8_LEN).into_array(); @@ -186,7 +187,7 @@ fn mul_i8_nonnull(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 16)] fn mul_u8_nonnull(bencher: Bencher) { let lhs = primitive_u8_small_nonnull(1, I8_LEN).into_array(); let rhs = primitive_u8_small_nonnull(7, I8_LEN).into_array(); @@ -195,7 +196,7 @@ fn mul_u8_nonnull(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 16)] fn mul_i16_nonnull(bencher: Bencher) { let lhs = primitive_i16_small_nonnull(1, I16_LEN).into_array(); let rhs = primitive_i16_small_nonnull(17, I16_LEN).into_array(); @@ -204,7 +205,7 @@ fn mul_i16_nonnull(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 16)] fn mul_u16_nonnull(bencher: Bencher) { let lhs = primitive_u16_small_nonnull(1, I16_LEN).into_array(); let rhs = primitive_u16_small_nonnull(17, I16_LEN).into_array(); @@ -213,7 +214,7 @@ fn mul_u16_nonnull(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 16)] fn mul_i32_nonnull(bencher: Bencher) { let lhs = primitive_i32_small_nonnull(1, I32_LEN).into_array(); let rhs = primitive_i32_small_nonnull(17, I32_LEN).into_array(); @@ -222,7 +223,7 @@ fn mul_i32_nonnull(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 16)] fn mul_u32_nonnull(bencher: Bencher) { let lhs = primitive_u32_small_nonnull(1, I32_LEN).into_array(); let rhs = primitive_u32_small_nonnull(17, I32_LEN).into_array(); @@ -231,7 +232,7 @@ fn mul_u32_nonnull(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 8)] fn mul_u64_nonnull(bencher: Bencher) { let lhs = primitive_u64_small_nonnull(1, I64_LEN).into_array(); let rhs = primitive_u64_small_nonnull(17, I64_LEN).into_array(); @@ -240,7 +241,7 @@ fn mul_u64_nonnull(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 16)] fn mul_i32_nullable(bencher: Bencher) { let lhs = primitive_i32_small_nullable(1, 7, I32_LEN).into_array(); let rhs = primitive_i32_small_nullable(17, 5, I32_LEN).into_array(); @@ -249,7 +250,7 @@ fn mul_i32_nullable(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 4)] fn div_i64_nonnull(bencher: Bencher) { let lhs = primitive_nonnull(1_000_000, I64_LEN).into_array(); let rhs = primitive_nonzero(I64_LEN).into_array(); @@ -258,7 +259,7 @@ fn div_i64_nonnull(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 4)] fn div_i64_nullable(bencher: Bencher) { let lhs = primitive_nullable(1_000_000, 7, I64_LEN).into_array(); let rhs = primitive_nullable(17, 5, I64_LEN).into_array(); @@ -315,7 +316,7 @@ fn div_decimal_i128_nullable(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 32)] fn lt_i64_nullable(bencher: Bencher) { let lhs = primitive_nullable(0, 7, LEN).into_array(); let rhs = primitive_nullable(1_000_000, 5, LEN).into_array(); diff --git a/vortex-array/benches/compare.rs b/vortex-array/benches/compare.rs index 8e040ce1b91..d4a72b5b1f9 100644 --- a/vortex-array/benches/compare.rs +++ b/vortex-array/benches/compare.rs @@ -49,6 +49,7 @@ use vortex_buffer::Buffer; #[global_allocator] static GLOBAL: MiMalloc = MiMalloc; +#[vortex_bench_support::main] fn main() { divan::main(); } @@ -175,7 +176,7 @@ fn compare_bool_constant(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 32)] fn compare_int(bencher: Bencher) { let mut rng = StdRng::seed_from_u64(0); let arr1 = int_array(&mut rng); @@ -184,7 +185,7 @@ fn compare_int(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 32)] fn compare_int_nullable(bencher: Bencher) { let mut rng = StdRng::seed_from_u64(0); let arr1 = int_array_nullable(&mut rng); @@ -193,7 +194,7 @@ fn compare_int_nullable(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 32)] fn compare_int_constant(bencher: Bencher) { let mut rng = StdRng::seed_from_u64(0); let arr = int_array(&mut rng); @@ -202,7 +203,7 @@ fn compare_int_constant(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 32)] fn compare_u8(bencher: Bencher) { let mut rng = StdRng::seed_from_u64(0); let arr1 = u8_array(&mut rng); @@ -211,7 +212,7 @@ fn compare_u8(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 32)] fn compare_u64(bencher: Bencher) { let mut rng = StdRng::seed_from_u64(0); let arr1 = u64_array(&mut rng); @@ -220,7 +221,7 @@ fn compare_u64(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 32)] fn compare_f32(bencher: Bencher) { let mut rng = StdRng::seed_from_u64(0); let arr1 = f32_array(&mut rng); @@ -229,7 +230,7 @@ fn compare_f32(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 32)] fn compare_int_eq(bencher: Bencher) { let mut rng = StdRng::seed_from_u64(0); let arr1 = int_array(&mut rng); @@ -238,7 +239,7 @@ fn compare_int_eq(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 32)] fn compare_float(bencher: Bencher) { let mut rng = StdRng::seed_from_u64(0); let arr1 = float_array(&mut rng); diff --git a/vortex-array/benches/row_fn_output.rs b/vortex-array/benches/row_fn_output.rs index 31c97ba77a2..673ff2eca5f 100644 --- a/vortex-array/benches/row_fn_output.rs +++ b/vortex-array/benches/row_fn_output.rs @@ -35,6 +35,7 @@ use vortex_session::registry::CachedId; #[global_allocator] static GLOBAL: MiMalloc = MiMalloc; +#[vortex_bench_support::main] fn main() { LazyLock::force(&SESSION); divan::main(); @@ -182,35 +183,35 @@ impl RowFn for DeferredI64 { } #[vortex_bench_support::cpu_features] -#[divan::bench(types = [i32, i64], args = INPUT_SHAPES)] +#[divan::bench(types = [i32, i64], args = INPUT_SHAPES, sample_size = 32)] fn infallible_bool(bencher: Bencher, &shape: &InputShape) { let function = InfallibleBool::(PhantomData); bench_row_fn(bencher, &function, make_args::(shape)); } #[vortex_bench_support::cpu_features] -#[divan::bench(args = CONSTANT_SHAPES)] +#[divan::bench(args = CONSTANT_SHAPES, sample_size = 32)] fn infallible_bool_constant(bencher: Bencher, &shape: &InputShape) { let function = InfallibleBool::(PhantomData); bench_row_fn(bencher, &function, make_args::(shape)); } #[vortex_bench_support::cpu_features] -#[divan::bench(types = [i32, i64], args = INPUT_SHAPES)] +#[divan::bench(types = [i32, i64], args = INPUT_SHAPES, sample_size = 32)] fn deferred_bool(bencher: Bencher, &shape: &InputShape) { let function = DeferredBool::(PhantomData); bench_row_fn(bencher, &function, make_args::(shape)); } #[vortex_bench_support::cpu_features] -#[divan::bench(args = CONSTANT_SHAPES)] +#[divan::bench(args = CONSTANT_SHAPES, sample_size = 32)] fn deferred_bool_constant(bencher: Bencher, &shape: &InputShape) { let function = DeferredBool::(PhantomData); bench_row_fn(bencher, &function, make_args::(shape)); } #[vortex_bench_support::cpu_features] -#[divan::bench(args = INPUT_SHAPES)] +#[divan::bench(args = INPUT_SHAPES, sample_size = 8)] fn deferred_i64(bencher: Bencher, &shape: &InputShape) { bench_row_fn(bencher, &DeferredI64, make_args::(shape)); } diff --git a/vortex-array/benches/scalar_subtract.rs b/vortex-array/benches/scalar_subtract.rs index 50c0d903f20..74fa9e824db 100644 --- a/vortex-array/benches/scalar_subtract.rs +++ b/vortex-array/benches/scalar_subtract.rs @@ -22,6 +22,7 @@ use vortex_array::scalar_fn::fns::operators::Operator; use vortex_buffer::Buffer; use vortex_session::VortexSession; +#[vortex_bench_support::main] fn main() { LazyLock::force(&SESSION); divan::main(); @@ -30,7 +31,7 @@ fn main() { static SESSION: LazyLock = LazyLock::new(array_session); #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 16)] fn scalar_subtract(bencher: Bencher) { let mut rng = StdRng::seed_from_u64(0); let range = Uniform::new(0i64, 100_000_000).unwrap(); diff --git a/vortex-buffer/benches/collect_bool.rs b/vortex-buffer/benches/collect_bool.rs index 864179471f8..e035609d77e 100644 --- a/vortex-buffer/benches/collect_bool.rs +++ b/vortex-buffer/benches/collect_bool.rs @@ -31,6 +31,7 @@ use vortex_buffer::collect_bool_word_scalar; #[cfg(not(codspeed))] use vortex_buffer::pack_bool_word_swar; +#[vortex_bench_support::main] fn main() { // Pre-warm CPUID feature detection so the one-time probe cost is never // included in any benchmark iteration. @@ -107,7 +108,7 @@ fn bench_words_gather( } #[vortex_bench_support::cpu_features] -#[divan::bench(args = INPUT_SIZE)] +#[divan::bench(args = INPUT_SIZE, sample_size = 2048, sample_count = 200)] fn words_gather_dispatch(bencher: Bencher, len: usize) { bench_words_gather(bencher, len, |words, len, bools| { // SAFETY: `collect_bool_words` invokes the predicate with indices `0..len` only. @@ -116,7 +117,7 @@ fn words_gather_dispatch(bencher: Bencher, len: usize) { } #[vortex_bench_support::cpu_features] -#[divan::bench(args = INPUT_SIZE)] +#[divan::bench(args = INPUT_SIZE, sample_size = 256, sample_count = 200)] fn words_gather_scalar(bencher: Bencher, len: usize) { bench_words_gather(bencher, len, |words, len, bools| { // SAFETY: `collect_bool_words_old` invokes the predicate with indices `0..len` only. diff --git a/vortex-compute/benches/lane_kernels.rs b/vortex-compute/benches/lane_kernels.rs index 2e9e145add7..6025310a125 100644 --- a/vortex-compute/benches/lane_kernels.rs +++ b/vortex-compute/benches/lane_kernels.rs @@ -48,6 +48,7 @@ use vortex_compute::lane_kernels::IndexedSourceExt; use vortex_compute::lane_kernels::LaneZip; use vortex_compute::lane_kernels::ReinterpretSink; +#[vortex_bench_support::main] fn main() { assert_overflow_parity(); assert_null_overflow_suppressed(); @@ -330,7 +331,7 @@ fn add_fixture(n: usize) -> AddFixture { } #[vortex_bench_support::cpu_features] -#[divan::bench(args = SIZES)] +#[divan::bench(args = SIZES, sample_size = 16)] fn lanezip_checked_add_u32(bencher: Bencher, n: usize) { let f = add_fixture(n); bencher @@ -353,7 +354,7 @@ fn lanezip_checked_add_u32(bencher: Bencher, n: usize) { } #[vortex_bench_support::cpu_features] -#[divan::bench(args = SIZES)] +#[divan::bench(args = SIZES, sample_size = 8)] fn arrow_checked_add_u32(bencher: Bencher, n: usize) { let f = add_fixture(n); let lhs_arr: ArrowArrayRef = Arc::new(UInt32Array::new( From 77aa54fcef704083e8c275aed6e18675b459ab6f Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Wed, 2 Sep 2026 11:32:36 +0100 Subject: [PATCH 2/3] Keep a walltime sample's live memory small With `sample_size` iterations per sample, divan builds that many `with_inputs` up front and holds that many outputs until the sample ends, so the cases with a large output ran on cold memory instead of a block the allocator had just recycled: `lanezip_checked_add_u32` read 3x slower and `scalar_subtract` 2.3x on the first run of this branch, and the 16384-row `binary_ops` cases 15-25%. Drops a large output inside the closure with `divan::black_box_drop`, and gives `collect_bool` one output buffer for the whole run rather than one per iteration. The spawned thread's stack is 16 MiB, matching the runners' main-thread limit rather than the 8 MiB assumed before. Signed-off-by: Joe Isaacs Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YV1PdJJrd7r1Zcts63RrWa --- benchmarks/bench-support/Cargo.toml | 2 +- benchmarks/bench-support/src/lib.rs | 13 ++++++++++--- vortex-array/benches/binary_ops.rs | 14 +++++++++----- vortex-array/benches/scalar_subtract.rs | 20 ++++++++++++-------- vortex-buffer/benches/collect_bool.rs | 9 ++++++--- vortex-compute/benches/lane_kernels.rs | 6 ++++-- 6 files changed, 42 insertions(+), 22 deletions(-) diff --git a/benchmarks/bench-support/Cargo.toml b/benchmarks/bench-support/Cargo.toml index 06b364c3085..2408d453e01 100644 --- a/benchmarks/bench-support/Cargo.toml +++ b/benchmarks/bench-support/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vortex-bench-support" -description = "Proc macros for gating Vortex benchmarks by CPU architecture and instruction set" +description = "Proc macros for Vortex benchmarks measured once per CPU feature set" authors = { workspace = true } categories = { workspace = true } edition = { workspace = true } diff --git a/benchmarks/bench-support/src/lib.rs b/benchmarks/bench-support/src/lib.rs index dd2461d7248..edb7d61ab42 100644 --- a/benchmarks/bench-support/src/lib.rs +++ b/benchmarks/bench-support/src/lib.rs @@ -14,9 +14,10 @@ use syn::ItemFn; use syn::ReturnType; use syn::parse_macro_input; -/// Stack size of the thread [`main`] runs the benchmarks on, matching the 8 MiB Linux gives the -/// main thread so nothing that fit before overflows now. -const BENCH_THREAD_STACK_SIZE: usize = 8 << 20; +/// Stack size of the thread [`main`] runs the benchmarks on: the 16 MiB the walltime runners +/// give the main thread (`ulimit -s`), so nothing that fit before overflows now. It is only +/// reserved, not committed, so there is no cost to matching the larger figure. +const BENCH_THREAD_STACK_SIZE: usize = 16 << 20; /// Measure this benchmark on every walltime CPU-feature leg instead of in simulation. /// @@ -44,6 +45,12 @@ const BENCH_THREAD_STACK_SIZE: usize = 8 << 20; /// span a wide range, add `sample_count` below the default of 1,000 so the largest one does /// not take seconds. /// +/// A larger sample changes what divan keeps alive: it builds a sample's `with_inputs` up +/// front and holds its outputs until the sample ends, so with `sample_size` of each in +/// flight, an input built per iteration or an output of any size is written to cold memory +/// instead of a block the allocator just recycled. Build inputs once, and drop a large output +/// inside the closure with `divan::black_box_drop` rather than returning it. +/// /// Spell it out in full rather than importing it: benchmark files are read a function at a /// time, and the path says where the behaviour comes from. The binary's `fn main` carries /// [`main`], which the walltime legs need for a repeatable layout. diff --git a/vortex-array/benches/binary_ops.rs b/vortex-array/benches/binary_ops.rs index c0b7ebe991e..dde9c0d7bb5 100644 --- a/vortex-array/benches/binary_ops.rs +++ b/vortex-array/benches/binary_ops.rs @@ -353,12 +353,16 @@ fn bench_binary( let mut ctx = SESSION.create_execution_ctx(); let len = lhs.len(); + // Dropped in the loop rather than returned: divan holds a sample's outputs until it ends, + // and with `sample_size` of them alive the next one is written to cold memory. bencher.counter(ItemsCount::new(len)).bench_local(|| { - lhs.clone() - .binary(rhs.clone(), operator) - .unwrap() - .execute::(&mut ctx) - .unwrap() + divan::black_box_drop( + lhs.clone() + .binary(rhs.clone(), operator) + .unwrap() + .execute::(&mut ctx) + .unwrap(), + ) }); } diff --git a/vortex-array/benches/scalar_subtract.rs b/vortex-array/benches/scalar_subtract.rs index 74fa9e824db..4d6cd838d54 100644 --- a/vortex-array/benches/scalar_subtract.rs +++ b/vortex-array/benches/scalar_subtract.rs @@ -53,13 +53,17 @@ fn scalar_subtract(bencher: Bencher) { bencher .with_inputs(|| (&chunked, SESSION.create_execution_ctx())) .bench_refs(|(chunked, ctx)| { - chunked - .clone() - .binary( - ConstantArray::new(Scalar::from(to_subtract), chunked.len()).into_array(), - Operator::Sub, - ) - .unwrap() - .execute::(ctx) + // Dropped here rather than returned, so the output is recycled every iteration + // instead of `sample_size` of them piling up cold (see `cpu_features`). + divan::black_box_drop( + chunked + .clone() + .binary( + ConstantArray::new(Scalar::from(to_subtract), chunked.len()).into_array(), + Operator::Sub, + ) + .unwrap() + .execute::(ctx), + ); }); } diff --git a/vortex-buffer/benches/collect_bool.rs b/vortex-buffer/benches/collect_bool.rs index e035609d77e..4a45ebd7428 100644 --- a/vortex-buffer/benches/collect_bool.rs +++ b/vortex-buffer/benches/collect_bool.rs @@ -102,9 +102,12 @@ fn bench_words_gather( collect: impl Fn(&mut [u64], usize, &[bool]) + Sync, ) { let bools = make_bools(len); - bencher - .with_inputs(|| vec![0u64; len.div_ceil(64)]) - .bench_refs(|words| collect(words, len, &bools)); + // One output buffer for every iteration rather than one per iteration through + // `with_inputs`: divan builds a sample's inputs up front, and `sample_size` of them would + // leave the loop writing to cold memory (see `cpu_features`). Every word is assigned, so + // nothing carries over. + let mut words = vec![0u64; len.div_ceil(64)]; + bencher.bench_local(|| collect(&mut words, len, &bools)); } #[vortex_bench_support::cpu_features] diff --git a/vortex-compute/benches/lane_kernels.rs b/vortex-compute/benches/lane_kernels.rs index 6025310a125..6bc4cd85d5b 100644 --- a/vortex-compute/benches/lane_kernels.rs +++ b/vortex-compute/benches/lane_kernels.rs @@ -349,7 +349,9 @@ fn lanezip_checked_add_u32(bencher: Bencher, n: usize) { LaneZip::new(lhs.as_slice(), rhs.as_slice()) .try_map_masked_into(&combined, out.as_mut_slice(), |(a, b)| a.checked_add(b)) .unwrap(); - (combined, out) + // Dropped here rather than returned, so the output block is recycled every + // iteration instead of `sample_size` of them piling up cold (see `cpu_features`). + divan::black_box_drop((combined, out)); }); } @@ -368,7 +370,7 @@ fn arrow_checked_add_u32(bencher: Bencher, n: usize) { bencher .with_inputs(|| (lhs_arr.clone(), rhs_arr.clone())) - .bench_values(|(lhs, rhs)| add(&lhs, &rhs).unwrap()); + .bench_values(|(lhs, rhs)| divan::black_box_drop(add(&lhs, &rhs).unwrap())); } // ----------------------------------------------------------------------------- From e5bb971e99e49f576f9f399d1a9fd5e3cd27d420 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Wed, 2 Sep 2026 11:56:40 +0100 Subject: [PATCH 3/3] Keep collect_bool on divan's input-slot loop With one output buffer and no per-iteration input, the benchmark took divan's zero-sized-input path, which black-boxes every iteration: on the second and third runs of this branch the 1,024-bit x86 cases read 3-4 ns above their 9-17 ns base. The bools stay the per-iteration input, so the loop is the one the fresh-buffer version used, and a `black_box` on the buffer keeps the packing stores observable, since nothing reads them before the buffer is freed. Signed-off-by: Joe Isaacs Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YV1PdJJrd7r1Zcts63RrWa --- vortex-buffer/benches/collect_bool.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/vortex-buffer/benches/collect_bool.rs b/vortex-buffer/benches/collect_bool.rs index 4a45ebd7428..fc8435ade88 100644 --- a/vortex-buffer/benches/collect_bool.rs +++ b/vortex-buffer/benches/collect_bool.rs @@ -105,9 +105,18 @@ fn bench_words_gather( // One output buffer for every iteration rather than one per iteration through // `with_inputs`: divan builds a sample's inputs up front, and `sample_size` of them would // leave the loop writing to cold memory (see `cpu_features`). Every word is assigned, so - // nothing carries over. + // nothing carries over. The bools stay the per-iteration input so the loop takes divan's + // input-slot path, as it did with a buffer per iteration; the zero-sized-input path + // black-boxes every iteration, which on a 10 ns case is a measurable share. `black_box` + // keeps the stores observable: nothing reads the buffer before it is freed, so they would + // otherwise be fair game for the optimizer. let mut words = vec![0u64; len.div_ceil(64)]; - bencher.bench_local(|| collect(&mut words, len, &bools)); + bencher + .with_inputs(|| bools.as_slice()) + .bench_local_refs(|bools| { + collect(&mut words, len, bools); + divan::black_box(&words); + }); } #[vortex_bench_support::cpu_features]