diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 7e98e37f5bc..9138245b95e 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -144,6 +144,10 @@ jobs: # crate that adds an `#[cpu_features]` benchmark is picked up without editing this workflow. # Building the whole workspace would do the same, but links every bench binary in it # — with debuginfo, from the bench profile — to measure only the tagged ones. + # + # A tagged binary must also run under `#[vortex_bench_support::main]`: ASLR is off on + # these runners, so without it the numbers depend on the size of the environment + # block, which differs between a push and a pull request. See that attribute's docs. - name: Select packages with tagged benchmarks id: select run: | @@ -152,16 +156,23 @@ jobs: meta = json.loads(subprocess.check_output( ["cargo", "metadata", "--no-deps", "--format-version", "1"])) - tagged = sorted( - package["name"] - for package in meta["packages"] - for target in package["targets"] - if "bench" in target["kind"] - and "cpu_features]" in pathlib.Path(target["src_path"]).read_text() - ) + tagged = {} + for package in meta["packages"]: + for target in package["targets"]: + if "bench" not in target["kind"]: + continue + source = pathlib.Path(target["src_path"]).read_text() + if "cpu_features]" in source: + tagged[target["src_path"]] = (package["name"], "bench_support::main]" in source) if not tagged: raise SystemExit("no benchmark carries `#[cpu_features]`; this job has nothing to measure") - print("packages=" + " ".join(f"-p {name}" for name in dict.fromkeys(tagged))) + unwrapped = sorted(path for path, (_, wrapped) in tagged.items() if not wrapped) + if unwrapped: + raise SystemExit( + "these carry `#[cpu_features]` but their `fn main` lacks " + "`#[vortex_bench_support::main]`:\n " + "\n ".join(unwrapped)) + packages = sorted({name for name, _ in tagged.values()}) + print("packages=" + " ".join(f"-p {name}" for name in packages)) EOF - name: Build benchmarks env: @@ -185,15 +196,11 @@ jobs: # so this selects paths of the form `::::`. That middle # component only exists because `#[cpu_features]` puts it there: an untagged benchmark has one # component fewer and cannot match, whatever it is called. + # How many samples to take and how many iterations each one runs are set on the + # benchmarks by `#[cpu_features]`, not here: a `DIVAN_SAMPLE_COUNT` or + # `DIVAN_SAMPLE_SIZE` in the environment would override every one of them. - name: Run benchmarks uses: CodSpeedHQ/action@4296e51e7041e24dadb86d1d6e8b9320d223dbe8 # v5 - env: - # divan's default of 100 samples leaves these benchmarks too noisy to compare - # across runs: the same commit measured twice varied by up to 2.2x on the 1,024 - # element cases and ~50-70% on the 65,536 element ones. At 1,000 samples the same - # experiment stays within ~6-10%, and the suite still runs in seconds, so the extra - # sampling is close to free next to the minute-plus spent building it. - DIVAN_SAMPLE_COUNT: "1000" with: run: bash scripts/bench-taskset.sh cargo codspeed run -- '.*::${{ matrix.features }}::' token: ${{ secrets.CODSPEED_TOKEN }} diff --git a/benchmarks/bench-support/Cargo.toml b/benchmarks/bench-support/Cargo.toml index 06b364c3085..2408d453e01 100644 --- a/benchmarks/bench-support/Cargo.toml +++ b/benchmarks/bench-support/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vortex-bench-support" -description = "Proc macros for gating Vortex benchmarks by CPU architecture and instruction set" +description = "Proc macros for Vortex benchmarks measured once per CPU feature set" authors = { workspace = true } categories = { workspace = true } edition = { workspace = true } diff --git a/benchmarks/bench-support/src/lib.rs b/benchmarks/bench-support/src/lib.rs index 912605c781e..edb7d61ab42 100644 --- a/benchmarks/bench-support/src/lib.rs +++ b/benchmarks/bench-support/src/lib.rs @@ -3,15 +3,22 @@ //! Support for benchmarks measured once per CPU feature set. //! -//! One attribute, [`cpu_features`]. Which feature sets exist, what each is built with, and -//! where it runs are all in `.github/workflows/codspeed.yml`. +//! Two attributes. [`cpu_features`] marks a benchmark as measured on every walltime leg, and +//! [`main`] goes on the `fn main` of a binary carrying such benchmarks. Which feature sets +//! exist, what each is built with, and where it runs are all in `.github/workflows/codspeed.yml`. use proc_macro::TokenStream; use proc_macro2::TokenStream as TokenStream2; use quote::quote; use syn::ItemFn; +use syn::ReturnType; use syn::parse_macro_input; +/// Stack size of the thread [`main`] runs the benchmarks on: the 16 MiB the walltime runners +/// give the main thread (`ulimit -s`), so nothing that fit before overflows now. It is only +/// reserved, not committed, so there is no cost to matching the larger figure. +const BENCH_THREAD_STACK_SIZE: usize = 16 << 20; + /// Measure this benchmark on every walltime CPU-feature leg instead of in simulation. /// /// Takes no argument: the benchmark runs on all of them. Write it *above* `#[divan::bench]`, @@ -23,12 +30,30 @@ use syn::parse_macro_input; /// /// ```ignore /// #[vortex_bench_support::cpu_features] -/// #[divan::bench(args = INPUT_SIZE)] +/// #[divan::bench(args = INPUT_SIZE, sample_size = 256)] /// fn words_gather_dispatch(bencher: Bencher, len: usize) { /* ... */ } /// ``` /// +/// `sample_size` is required: it is how many times one sample runs the benchmark, and the +/// number reported is the sample's time divided by it. The runner brackets every sample with +/// its own hooks and timestamps, and what that costs the sample that follows is a fixed +/// amount that differs from one host to the next by about a microsecond. Left to itself, +/// divan sizes samples by its first, cold run and settles on one iteration for anything over +/// a few microseconds, so on a bad host every case in a binary read the same ~1.2 µs slower +/// regardless of its length. Pick the size so that a sample lasts at least ~100 µs on the +/// smallest argument: 64 for a 2 µs case, 16 for a 10 µs one, 4 for 50 µs. When the arguments +/// span a wide range, add `sample_count` below the default of 1,000 so the largest one does +/// not take seconds. +/// +/// A larger sample changes what divan keeps alive: it builds a sample's `with_inputs` up +/// front and holds its outputs until the sample ends, so with `sample_size` of each in +/// flight, an input built per iteration or an output of any size is written to cold memory +/// instead of a block the allocator just recycled. Build inputs once, and drop a large output +/// inside the closure with `divan::black_box_drop` rather than returning it. +/// /// Spell it out in full rather than importing it: benchmark files are read a function at a -/// time, and the path says where the behaviour comes from. +/// time, and the path says where the behaviour comes from. The binary's `fn main` carries +/// [`main`], which the walltime legs need for a repeatable layout. /// /// This is for code that is written once and *compiled* differently per feature set — a /// shipped entry point that selects its kernel through `cfg(target_feature)`, or a scalar @@ -76,12 +101,15 @@ pub fn cpu_features(attr: TokenStream, item: TokenStream) -> TokenStream { } }; - for reserved in ["name", "ignore"] { - if existing + let has_option = |option: &str| { + existing .clone() .into_iter() - .any(|token| matches!(&token, proc_macro2::TokenTree::Ident(i) if i == reserved)) - { + .any(|token| matches!(&token, proc_macro2::TokenTree::Ident(i) if i == option)) + }; + + for reserved in ["name", "ignore"] { + if has_option(reserved) { return syn::Error::new_spanned( &existing, format!( @@ -93,6 +121,27 @@ pub fn cpu_features(attr: TokenStream, item: TokenStream) -> TokenStream { } } + if !has_option("sample_size") { + return syn::Error::new_spanned( + bench, + "`#[cpu_features]` needs `sample_size` in `#[divan::bench(..)]`: enough iterations \ + per sample for one sample to last ~100 µs or more, so the runner's per-sample \ + overhead, which varies by host, is amortised (see the attribute's docs)", + ) + .to_compile_error() + .into(); + } + + // The default used to be `DIVAN_SAMPLE_COUNT` in the workflow, but a value set there beats + // one set on the benchmark, and the cases that need thousands of iterations per sample + // need fewer samples in exchange. divan's own default of 100 samples left the same commit + // measured twice up to 2.2x apart on the 1,024 element cases; 1,000 holds them to ~6-10%. + let sample_count = if has_option("sample_count") { + quote!() + } else { + quote!(sample_count = 1000,) + }; + // Skipping simulation, rather than naming the legs to run on, is what keeps the leg list // in the workflow alone. `env!` rather than reading the environment during expansion: // rustc records it as a dependency of the crate, so changing legs rebuilds the benchmarks. @@ -112,6 +161,7 @@ pub fn cpu_features(attr: TokenStream, item: TokenStream) -> TokenStream { bench.meta = syn::parse_quote! { #bench_path( #existing #separator + #sample_count name = concat!( env!("VORTEX_BENCH_PREFIX"), #name, @@ -123,3 +173,82 @@ pub fn cpu_features(attr: TokenStream, item: TokenStream) -> TokenStream { quote!(#function).into() } + +/// Run the benchmarks on a thread of their own, so where their stack and heap land is set by +/// the binary rather than by the process. +/// +/// Write it on `fn main`, which must take no arguments and return `()`: +/// +/// ```ignore +/// #[vortex_bench_support::main] +/// fn main() { +/// divan::main(); +/// } +/// ``` +/// +/// Every binary with a [`cpu_features`] benchmark needs it; the workflow refuses to measure +/// one without it. The body runs unchanged on a spawned thread with a fixed-size stack, and a +/// panic there is re-raised on the main thread so the exit status is what it was. +/// +/// Some µs-scale cases have two stable timings up to 1.5× apart, and which one a run lands in +/// depends on the layout of the process rather than on the code: `arrow_checked_add_u32` in +/// `vortex-compute` read 13 µs or 20 µs on the neon leg from one run to the next. On the main +/// thread, with the address space randomised, it landed in either about half the time; on a +/// spawned thread it held one timing through every one of those runs, and through a sweep of +/// the environment block's size (which, contrary to the obvious guess, did not move it on +/// either thread). The spawned thread's stack is a fresh mapping and the allocators in use +/// serve it from mappings of their own, so its layout is set by the binary alone. +#[proc_macro_attribute] +pub fn main(attr: TokenStream, item: TokenStream) -> TokenStream { + if !attr.is_empty() { + let attr = TokenStream2::from(attr); + return syn::Error::new_spanned(attr, "`#[vortex_bench_support::main]` takes no argument") + .to_compile_error() + .into(); + } + + let function = parse_macro_input!(item as ItemFn); + let signature = &function.sig; + + if signature.ident != "main" { + return syn::Error::new_spanned( + &signature.ident, + "`#[vortex_bench_support::main]` goes on `fn main`", + ) + .to_compile_error() + .into(); + } + if !signature.inputs.is_empty() + || !matches!(signature.output, ReturnType::Default) + || signature.asyncness.is_some() + { + return syn::Error::new_spanned( + signature, + "`#[vortex_bench_support::main]` expects `fn main()` with no arguments and no return type", + ) + .to_compile_error() + .into(); + } + + let attrs = &function.attrs; + let vis = &function.vis; + let body = &function.block; + + // The body becomes the closure, so a `return` in it leaves the benchmarks as before. An + // explicit stack size rather than the default keeps `RUST_MIN_STACK` from putting the + // environment back into the layout. + quote! { + #(#attrs)* + #vis fn main() -> ::std::io::Result<()> { + let benchmarks = ::std::thread::Builder::new() + .name("vortex-bench".to_owned()) + .stack_size(#BENCH_THREAD_STACK_SIZE) + .spawn(|| #body)?; + if let Err(panic) = benchmarks.join() { + ::std::panic::resume_unwind(panic); + } + Ok(()) + } + } + .into() +} diff --git a/vortex-array/benches/binary_ops.rs b/vortex-array/benches/binary_ops.rs index a1f14ad9bb1..dde9c0d7bb5 100644 --- a/vortex-array/benches/binary_ops.rs +++ b/vortex-array/benches/binary_ops.rs @@ -41,6 +41,7 @@ use vortex_session::VortexSession; #[global_allocator] static GLOBAL: MiMalloc = MiMalloc; +#[vortex_bench_support::main] fn main() { LazyLock::force(&SESSION); divan::main(); @@ -92,25 +93,25 @@ enum BinaryShape { const DECIMAL_MUL_DIV_LEN: usize = 1_024; #[vortex_bench_support::cpu_features] -#[divan::bench(args = BINARY_SHAPE_CASES)] +#[divan::bench(args = BINARY_SHAPE_CASES, sample_size = 64)] fn add_shapes(bencher: Bencher, &(len, shape): &(usize, BinaryShape)) { bench_binary_shape(bencher, len, shape, Operator::Add); } #[vortex_bench_support::cpu_features] -#[divan::bench(args = CONSTANT_SHAPE_CASES)] +#[divan::bench(args = CONSTANT_SHAPE_CASES, sample_size = 16)] fn add_constant_shapes(bencher: Bencher, &(len, shape): &(usize, BinaryShape)) { bench_binary_shape(bencher, len, shape, Operator::Add); } #[vortex_bench_support::cpu_features] -#[divan::bench(args = BINARY_SHAPE_CASES)] +#[divan::bench(args = BINARY_SHAPE_CASES, sample_size = 64)] fn subtract_shapes(bencher: Bencher, &(len, shape): &(usize, BinaryShape)) { bench_binary_shape(bencher, len, shape, Operator::Sub); } #[vortex_bench_support::cpu_features] -#[divan::bench(args = BINARY_SHAPE_CASES)] +#[divan::bench(args = BINARY_SHAPE_CASES, sample_size = 64)] fn multiply_shapes(bencher: Bencher, &(len, shape): &(usize, BinaryShape)) { bench_binary_shape(bencher, len, shape, Operator::Mul); } @@ -132,7 +133,7 @@ fn bench_binary_shape(bencher: Bencher, len: usize, shape: BinaryShape, operator } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 16)] fn add_i64_nonnull(bencher: Bencher) { let lhs = primitive_nonnull(0, I64_LEN).into_array(); let rhs = primitive_nonnull(1_000_000, I64_LEN).into_array(); @@ -141,7 +142,7 @@ fn add_i64_nonnull(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 16)] fn add_i64_nullable(bencher: Bencher) { let lhs = primitive_nullable(0, 7, I64_LEN).into_array(); let rhs = primitive_nullable(1_000_000, 5, I64_LEN).into_array(); @@ -150,7 +151,7 @@ fn add_i64_nullable(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 16)] fn add_i32_nonnull(bencher: Bencher) { let lhs = primitive_i32_small_nonnull(1, I32_LEN).into_array(); let rhs = primitive_i32_small_nonnull(17, I32_LEN).into_array(); @@ -159,7 +160,7 @@ fn add_i32_nonnull(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 16)] fn add_u32_nonnull(bencher: Bencher) { let lhs = primitive_u32_small_nonnull(1, I32_LEN).into_array(); let rhs = primitive_u32_small_nonnull(17, I32_LEN).into_array(); @@ -168,7 +169,7 @@ fn add_u32_nonnull(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 8)] fn mul_i64_nonnull(bencher: Bencher) { let lhs = primitive_small_nonnull(1, I64_LEN).into_array(); let rhs = primitive_small_nonnull(17, I64_LEN).into_array(); @@ -177,7 +178,7 @@ fn mul_i64_nonnull(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 16)] fn mul_i8_nonnull(bencher: Bencher) { let lhs = primitive_i8_small_nonnull(1, I8_LEN).into_array(); let rhs = primitive_i8_small_nonnull(7, I8_LEN).into_array(); @@ -186,7 +187,7 @@ fn mul_i8_nonnull(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 16)] fn mul_u8_nonnull(bencher: Bencher) { let lhs = primitive_u8_small_nonnull(1, I8_LEN).into_array(); let rhs = primitive_u8_small_nonnull(7, I8_LEN).into_array(); @@ -195,7 +196,7 @@ fn mul_u8_nonnull(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 16)] fn mul_i16_nonnull(bencher: Bencher) { let lhs = primitive_i16_small_nonnull(1, I16_LEN).into_array(); let rhs = primitive_i16_small_nonnull(17, I16_LEN).into_array(); @@ -204,7 +205,7 @@ fn mul_i16_nonnull(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 16)] fn mul_u16_nonnull(bencher: Bencher) { let lhs = primitive_u16_small_nonnull(1, I16_LEN).into_array(); let rhs = primitive_u16_small_nonnull(17, I16_LEN).into_array(); @@ -213,7 +214,7 @@ fn mul_u16_nonnull(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 16)] fn mul_i32_nonnull(bencher: Bencher) { let lhs = primitive_i32_small_nonnull(1, I32_LEN).into_array(); let rhs = primitive_i32_small_nonnull(17, I32_LEN).into_array(); @@ -222,7 +223,7 @@ fn mul_i32_nonnull(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 16)] fn mul_u32_nonnull(bencher: Bencher) { let lhs = primitive_u32_small_nonnull(1, I32_LEN).into_array(); let rhs = primitive_u32_small_nonnull(17, I32_LEN).into_array(); @@ -231,7 +232,7 @@ fn mul_u32_nonnull(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 8)] fn mul_u64_nonnull(bencher: Bencher) { let lhs = primitive_u64_small_nonnull(1, I64_LEN).into_array(); let rhs = primitive_u64_small_nonnull(17, I64_LEN).into_array(); @@ -240,7 +241,7 @@ fn mul_u64_nonnull(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 16)] fn mul_i32_nullable(bencher: Bencher) { let lhs = primitive_i32_small_nullable(1, 7, I32_LEN).into_array(); let rhs = primitive_i32_small_nullable(17, 5, I32_LEN).into_array(); @@ -249,7 +250,7 @@ fn mul_i32_nullable(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 4)] fn div_i64_nonnull(bencher: Bencher) { let lhs = primitive_nonnull(1_000_000, I64_LEN).into_array(); let rhs = primitive_nonzero(I64_LEN).into_array(); @@ -258,7 +259,7 @@ fn div_i64_nonnull(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 4)] fn div_i64_nullable(bencher: Bencher) { let lhs = primitive_nullable(1_000_000, 7, I64_LEN).into_array(); let rhs = primitive_nullable(17, 5, I64_LEN).into_array(); @@ -315,7 +316,7 @@ fn div_decimal_i128_nullable(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 32)] fn lt_i64_nullable(bencher: Bencher) { let lhs = primitive_nullable(0, 7, LEN).into_array(); let rhs = primitive_nullable(1_000_000, 5, LEN).into_array(); @@ -352,12 +353,16 @@ fn bench_binary( let mut ctx = SESSION.create_execution_ctx(); let len = lhs.len(); + // Dropped in the loop rather than returned: divan holds a sample's outputs until it ends, + // and with `sample_size` of them alive the next one is written to cold memory. bencher.counter(ItemsCount::new(len)).bench_local(|| { - lhs.clone() - .binary(rhs.clone(), operator) - .unwrap() - .execute::(&mut ctx) - .unwrap() + divan::black_box_drop( + lhs.clone() + .binary(rhs.clone(), operator) + .unwrap() + .execute::(&mut ctx) + .unwrap(), + ) }); } diff --git a/vortex-array/benches/compare.rs b/vortex-array/benches/compare.rs index 8e040ce1b91..d4a72b5b1f9 100644 --- a/vortex-array/benches/compare.rs +++ b/vortex-array/benches/compare.rs @@ -49,6 +49,7 @@ use vortex_buffer::Buffer; #[global_allocator] static GLOBAL: MiMalloc = MiMalloc; +#[vortex_bench_support::main] fn main() { divan::main(); } @@ -175,7 +176,7 @@ fn compare_bool_constant(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 32)] fn compare_int(bencher: Bencher) { let mut rng = StdRng::seed_from_u64(0); let arr1 = int_array(&mut rng); @@ -184,7 +185,7 @@ fn compare_int(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 32)] fn compare_int_nullable(bencher: Bencher) { let mut rng = StdRng::seed_from_u64(0); let arr1 = int_array_nullable(&mut rng); @@ -193,7 +194,7 @@ fn compare_int_nullable(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 32)] fn compare_int_constant(bencher: Bencher) { let mut rng = StdRng::seed_from_u64(0); let arr = int_array(&mut rng); @@ -202,7 +203,7 @@ fn compare_int_constant(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 32)] fn compare_u8(bencher: Bencher) { let mut rng = StdRng::seed_from_u64(0); let arr1 = u8_array(&mut rng); @@ -211,7 +212,7 @@ fn compare_u8(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 32)] fn compare_u64(bencher: Bencher) { let mut rng = StdRng::seed_from_u64(0); let arr1 = u64_array(&mut rng); @@ -220,7 +221,7 @@ fn compare_u64(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 32)] fn compare_f32(bencher: Bencher) { let mut rng = StdRng::seed_from_u64(0); let arr1 = f32_array(&mut rng); @@ -229,7 +230,7 @@ fn compare_f32(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 32)] fn compare_int_eq(bencher: Bencher) { let mut rng = StdRng::seed_from_u64(0); let arr1 = int_array(&mut rng); @@ -238,7 +239,7 @@ fn compare_int_eq(bencher: Bencher) { } #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 32)] fn compare_float(bencher: Bencher) { let mut rng = StdRng::seed_from_u64(0); let arr1 = float_array(&mut rng); diff --git a/vortex-array/benches/row_fn_output.rs b/vortex-array/benches/row_fn_output.rs index 31c97ba77a2..673ff2eca5f 100644 --- a/vortex-array/benches/row_fn_output.rs +++ b/vortex-array/benches/row_fn_output.rs @@ -35,6 +35,7 @@ use vortex_session::registry::CachedId; #[global_allocator] static GLOBAL: MiMalloc = MiMalloc; +#[vortex_bench_support::main] fn main() { LazyLock::force(&SESSION); divan::main(); @@ -182,35 +183,35 @@ impl RowFn for DeferredI64 { } #[vortex_bench_support::cpu_features] -#[divan::bench(types = [i32, i64], args = INPUT_SHAPES)] +#[divan::bench(types = [i32, i64], args = INPUT_SHAPES, sample_size = 32)] fn infallible_bool(bencher: Bencher, &shape: &InputShape) { let function = InfallibleBool::(PhantomData); bench_row_fn(bencher, &function, make_args::(shape)); } #[vortex_bench_support::cpu_features] -#[divan::bench(args = CONSTANT_SHAPES)] +#[divan::bench(args = CONSTANT_SHAPES, sample_size = 32)] fn infallible_bool_constant(bencher: Bencher, &shape: &InputShape) { let function = InfallibleBool::(PhantomData); bench_row_fn(bencher, &function, make_args::(shape)); } #[vortex_bench_support::cpu_features] -#[divan::bench(types = [i32, i64], args = INPUT_SHAPES)] +#[divan::bench(types = [i32, i64], args = INPUT_SHAPES, sample_size = 32)] fn deferred_bool(bencher: Bencher, &shape: &InputShape) { let function = DeferredBool::(PhantomData); bench_row_fn(bencher, &function, make_args::(shape)); } #[vortex_bench_support::cpu_features] -#[divan::bench(args = CONSTANT_SHAPES)] +#[divan::bench(args = CONSTANT_SHAPES, sample_size = 32)] fn deferred_bool_constant(bencher: Bencher, &shape: &InputShape) { let function = DeferredBool::(PhantomData); bench_row_fn(bencher, &function, make_args::(shape)); } #[vortex_bench_support::cpu_features] -#[divan::bench(args = INPUT_SHAPES)] +#[divan::bench(args = INPUT_SHAPES, sample_size = 8)] fn deferred_i64(bencher: Bencher, &shape: &InputShape) { bench_row_fn(bencher, &DeferredI64, make_args::(shape)); } diff --git a/vortex-array/benches/scalar_subtract.rs b/vortex-array/benches/scalar_subtract.rs index 50c0d903f20..4d6cd838d54 100644 --- a/vortex-array/benches/scalar_subtract.rs +++ b/vortex-array/benches/scalar_subtract.rs @@ -22,6 +22,7 @@ use vortex_array::scalar_fn::fns::operators::Operator; use vortex_buffer::Buffer; use vortex_session::VortexSession; +#[vortex_bench_support::main] fn main() { LazyLock::force(&SESSION); divan::main(); @@ -30,7 +31,7 @@ fn main() { static SESSION: LazyLock = LazyLock::new(array_session); #[vortex_bench_support::cpu_features] -#[divan::bench] +#[divan::bench(sample_size = 16)] fn scalar_subtract(bencher: Bencher) { let mut rng = StdRng::seed_from_u64(0); let range = Uniform::new(0i64, 100_000_000).unwrap(); @@ -52,13 +53,17 @@ fn scalar_subtract(bencher: Bencher) { bencher .with_inputs(|| (&chunked, SESSION.create_execution_ctx())) .bench_refs(|(chunked, ctx)| { - chunked - .clone() - .binary( - ConstantArray::new(Scalar::from(to_subtract), chunked.len()).into_array(), - Operator::Sub, - ) - .unwrap() - .execute::(ctx) + // Dropped here rather than returned, so the output is recycled every iteration + // instead of `sample_size` of them piling up cold (see `cpu_features`). + divan::black_box_drop( + chunked + .clone() + .binary( + ConstantArray::new(Scalar::from(to_subtract), chunked.len()).into_array(), + Operator::Sub, + ) + .unwrap() + .execute::(ctx), + ); }); } diff --git a/vortex-buffer/benches/collect_bool.rs b/vortex-buffer/benches/collect_bool.rs index 864179471f8..fc8435ade88 100644 --- a/vortex-buffer/benches/collect_bool.rs +++ b/vortex-buffer/benches/collect_bool.rs @@ -31,6 +31,7 @@ use vortex_buffer::collect_bool_word_scalar; #[cfg(not(codspeed))] use vortex_buffer::pack_bool_word_swar; +#[vortex_bench_support::main] fn main() { // Pre-warm CPUID feature detection so the one-time probe cost is never // included in any benchmark iteration. @@ -101,13 +102,25 @@ fn bench_words_gather( collect: impl Fn(&mut [u64], usize, &[bool]) + Sync, ) { let bools = make_bools(len); + // One output buffer for every iteration rather than one per iteration through + // `with_inputs`: divan builds a sample's inputs up front, and `sample_size` of them would + // leave the loop writing to cold memory (see `cpu_features`). Every word is assigned, so + // nothing carries over. The bools stay the per-iteration input so the loop takes divan's + // input-slot path, as it did with a buffer per iteration; the zero-sized-input path + // black-boxes every iteration, which on a 10 ns case is a measurable share. `black_box` + // keeps the stores observable: nothing reads the buffer before it is freed, so they would + // otherwise be fair game for the optimizer. + let mut words = vec![0u64; len.div_ceil(64)]; bencher - .with_inputs(|| vec![0u64; len.div_ceil(64)]) - .bench_refs(|words| collect(words, len, &bools)); + .with_inputs(|| bools.as_slice()) + .bench_local_refs(|bools| { + collect(&mut words, len, bools); + divan::black_box(&words); + }); } #[vortex_bench_support::cpu_features] -#[divan::bench(args = INPUT_SIZE)] +#[divan::bench(args = INPUT_SIZE, sample_size = 2048, sample_count = 200)] fn words_gather_dispatch(bencher: Bencher, len: usize) { bench_words_gather(bencher, len, |words, len, bools| { // SAFETY: `collect_bool_words` invokes the predicate with indices `0..len` only. @@ -116,7 +129,7 @@ fn words_gather_dispatch(bencher: Bencher, len: usize) { } #[vortex_bench_support::cpu_features] -#[divan::bench(args = INPUT_SIZE)] +#[divan::bench(args = INPUT_SIZE, sample_size = 256, sample_count = 200)] fn words_gather_scalar(bencher: Bencher, len: usize) { bench_words_gather(bencher, len, |words, len, bools| { // SAFETY: `collect_bool_words_old` invokes the predicate with indices `0..len` only. diff --git a/vortex-compute/benches/lane_kernels.rs b/vortex-compute/benches/lane_kernels.rs index 2e9e145add7..6bc4cd85d5b 100644 --- a/vortex-compute/benches/lane_kernels.rs +++ b/vortex-compute/benches/lane_kernels.rs @@ -48,6 +48,7 @@ use vortex_compute::lane_kernels::IndexedSourceExt; use vortex_compute::lane_kernels::LaneZip; use vortex_compute::lane_kernels::ReinterpretSink; +#[vortex_bench_support::main] fn main() { assert_overflow_parity(); assert_null_overflow_suppressed(); @@ -330,7 +331,7 @@ fn add_fixture(n: usize) -> AddFixture { } #[vortex_bench_support::cpu_features] -#[divan::bench(args = SIZES)] +#[divan::bench(args = SIZES, sample_size = 16)] fn lanezip_checked_add_u32(bencher: Bencher, n: usize) { let f = add_fixture(n); bencher @@ -348,12 +349,14 @@ fn lanezip_checked_add_u32(bencher: Bencher, n: usize) { LaneZip::new(lhs.as_slice(), rhs.as_slice()) .try_map_masked_into(&combined, out.as_mut_slice(), |(a, b)| a.checked_add(b)) .unwrap(); - (combined, out) + // Dropped here rather than returned, so the output block is recycled every + // iteration instead of `sample_size` of them piling up cold (see `cpu_features`). + divan::black_box_drop((combined, out)); }); } #[vortex_bench_support::cpu_features] -#[divan::bench(args = SIZES)] +#[divan::bench(args = SIZES, sample_size = 8)] fn arrow_checked_add_u32(bencher: Bencher, n: usize) { let f = add_fixture(n); let lhs_arr: ArrowArrayRef = Arc::new(UInt32Array::new( @@ -367,7 +370,7 @@ fn arrow_checked_add_u32(bencher: Bencher, n: usize) { bencher .with_inputs(|| (lhs_arr.clone(), rhs_arr.clone())) - .bench_values(|(lhs, rhs)| add(&lhs, &rhs).unwrap()); + .bench_values(|(lhs, rhs)| divan::black_box_drop(add(&lhs, &rhs).unwrap())); } // -----------------------------------------------------------------------------