From 1ba05c3f8516992bb01f1475fd119335f21d2b54 Mon Sep 17 00:00:00 2001 From: vltbaudbot <264001112+vltbaudbot@users.noreply.github.com> Date: Wed, 30 Sep 2026 14:24:22 +0000 Subject: [PATCH 1/3] Add ci+setup benchmark variation Add a new benchmark variation that measures the end-to-end time of installing the package manager binary AND running its CI/frozen-lockfile install command. This simulates the full CI pipeline experience where the tool must be set up from scratch before dependencies can be installed. The variation works like the existing 'ci' variation but: - The measured command includes PM binary installation (npm install -g, corepack download, etc.) before the ci/frozen-lockfile command - The prepare step generates the lockfile (via regular install), then uninstalls the PM binary / purges corepack cache so the next measured run includes re-downloading it For corepack-managed PMs (yarn, berry, zpm, pnpm), the corepack cache is purged so the measured command includes the corepack download step. For npm-installed PMs (vlt, bun, deno, aube, upm, pacquet), the global package is uninstalled between runs. Changes: - New variation script: scripts/variations/ci+setup.sh - Registered in bench, benchmark.sh, process-results.sh, generate-chart.js - Added to CI workflow matrix (benchmark.yaml) - Added to app types (Variation union) and UI category lists Co-authored-by: Evert Pot --- .github/workflows/benchmark.yaml | 6 +- app/src/lib/utils.ts | 3 + app/src/types/chart-data.ts | 2 + bench | 1 + scripts/benchmark.sh | 3 + scripts/generate-chart.js | 1 + scripts/process-results.sh | 2 +- scripts/variations/ci+setup.sh | 220 +++++++++++++++++++++++++++++++ 8 files changed, 234 insertions(+), 4 deletions(-) create mode 100644 scripts/variations/ci+setup.sh diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 07f4320f99..d545261767 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -8,7 +8,7 @@ on: default: '["next", "astro", "vue", "svelte", "large", "babylon"]' variations: description: "The benchmark variations to run (build/build-cache/run are added via matrix include pairs)" - default: '["clean", "node_modules", "cache", "cache+node_modules", "cache+lockfile", "cache+lockfile+node_modules", "ci", "lockfile", "lockfile+node_modules", "registry-clean", "registry-lockfile"]' + default: '["clean", "node_modules", "cache", "cache+node_modules", "cache+lockfile", "cache+lockfile+node_modules", "ci", "ci+setup", "lockfile", "lockfile+node_modules", "registry-clean", "registry-lockfile"]' binaries: description: "The binaries to run the benchmarks on" default: '"npm,yarn,berry,zpm,pnpm,pacquet,vlt,bun,deno,aube,upm,nx,turbo,vp,node"' @@ -93,7 +93,7 @@ jobs: partial=false if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then [[ "$IN_FIXTURES" == '["next", "astro", "vue", "svelte", "large", "babylon"]' ]] || partial=true - [[ "$IN_VARIATIONS" == '["clean", "node_modules", "cache", "cache+node_modules", "cache+lockfile", "cache+lockfile+node_modules", "ci", "lockfile", "lockfile+node_modules", "registry-clean", "registry-lockfile"]' ]] || partial=true + [[ "$IN_VARIATIONS" == '["clean", "node_modules", "cache", "cache+node_modules", "cache+lockfile", "cache+lockfile+node_modules", "ci", "ci+setup", "lockfile", "lockfile+node_modules", "registry-clean", "registry-lockfile"]' ]] || partial=true [[ "$IN_BINARIES" == '"npm,yarn,berry,zpm,pnpm,pacquet,vlt,bun,deno,aube,upm,nx,turbo,vp,node"' ]] || partial=true [[ "$IN_WARMUP" == "2" ]] || partial=true [[ "$IN_RUNS" == "5" ]] || partial=true @@ -114,7 +114,7 @@ jobs: fail-fast: false matrix: fixture: ${{ fromJson((github.event_name == 'push' && github.ref != 'refs/heads/main') && '["next", "astro", "vue", "svelte"]' || (inputs.fixtures || '["next", "astro", "vue", "svelte", "large", "babylon"]')) }} - variation: ${{ fromJson(inputs.variations || '["clean", "node_modules", "cache", "cache+node_modules", "cache+lockfile", "cache+lockfile+node_modules", "ci", "lockfile", "lockfile+node_modules", "registry-clean", "registry-lockfile"]') }} + variation: ${{ fromJson(inputs.variations || '["clean", "node_modules", "cache", "cache+node_modules", "cache+lockfile", "cache+lockfile+node_modules", "ci", "ci+setup", "lockfile", "lockfile+node_modules", "registry-clean", "registry-lockfile"]') }} include: - variation: "build" fixture: "next" diff --git a/app/src/lib/utils.ts b/app/src/lib/utils.ts index ae9ab102de..eb1fcd9664 100644 --- a/app/src/lib/utils.ts +++ b/app/src/lib/utils.ts @@ -66,6 +66,7 @@ export const calculateAverageVariationData = ( "cache+lockfile", "cache+lockfile+node_modules", "ci", + "ci+setup", "lockfile", "lockfile+node_modules", ] @@ -165,6 +166,7 @@ export const getVariationCategories = ( "cache+lockfile", "cache+lockfile+node_modules", "ci", + "ci+setup", "lockfile", "lockfile+node_modules", ].filter((v) => variations.includes(v as Variation)) as Variation[]; @@ -412,6 +414,7 @@ export function sortVariations(variations: Variation[]): Variation[] { "cache+lockfile", "cache+lockfile+node_modules", "ci", + "ci+setup", "lockfile", "lockfile+node_modules", "build", diff --git a/app/src/types/chart-data.ts b/app/src/types/chart-data.ts index 4a42e321c9..7a749cf98d 100644 --- a/app/src/types/chart-data.ts +++ b/app/src/types/chart-data.ts @@ -35,6 +35,7 @@ export type Variation = | "cache+lockfile+node_modules" | "cache+node_modules" | "ci" + | "ci+setup" | "clean" | "lockfile" | "lockfile+node_modules" @@ -283,6 +284,7 @@ export function isValidVariation(variation: string): variation is Variation { "cache+lockfile+node_modules", "cache+node_modules", "ci", + "ci+setup", "clean", "lockfile", "lockfile+node_modules", diff --git a/bench b/bench index 561e9b82cd..47d7ab0539 100755 --- a/bench +++ b/bench @@ -14,6 +14,7 @@ AVAILABLE_VARIATIONS=( cache+node_modules cache+lockfile+node_modules ci + ci+setup lockfile lockfile+node_modules node_modules diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh index 8568a227a0..72ca380144 100644 --- a/scripts/benchmark.sh +++ b/scripts/benchmark.sh @@ -35,6 +35,9 @@ case "$2" in ci) bash ../../scripts/variations/ci.sh "../../scripts" "../../results" "$1" "$2" ;; + ci+setup) + bash ../../scripts/variations/ci+setup.sh "../../scripts" "../../results" "$1" "$2" + ;; lockfile) bash ../../scripts/variations/lockfile.sh "../../scripts" "../../results" "$1" "$2" ;; diff --git a/scripts/generate-chart.js b/scripts/generate-chart.js index 49b2b55a8f..d6d4d050dc 100644 --- a/scripts/generate-chart.js +++ b/scripts/generate-chart.js @@ -117,6 +117,7 @@ function generateChartData(option = {}) { "cache+lockfile+node_modules", "cache+node_modules", "ci", + "ci+setup", "clean", "lockfile", "lockfile+node_modules", diff --git a/scripts/process-results.sh b/scripts/process-results.sh index 5949f5dfd3..6a374c04f4 100644 --- a/scripts/process-results.sh +++ b/scripts/process-results.sh @@ -102,7 +102,7 @@ echo "Processing results..." # Process variations results for fixture in next astro svelte vue large babylon; do - for variation in cache cache+lockfile cache+lockfile+node_modules cache+node_modules ci clean lockfile lockfile+node_modules node_modules build build-cache run; do + for variation in cache cache+lockfile cache+lockfile+node_modules cache+node_modules ci ci+setup clean lockfile lockfile+node_modules node_modules build build-cache run; do if benchmark_file=$(resolve_result_path "$fixture" "$variation" "benchmarks.json"); then print_summary "$benchmark_file" "$fixture" "$variation" cp "$benchmark_file" "results/$DATE/$fixture-$variation.json" diff --git a/scripts/variations/ci+setup.sh b/scripts/variations/ci+setup.sh new file mode 100644 index 0000000000..1d4e3751e2 --- /dev/null +++ b/scripts/variations/ci+setup.sh @@ -0,0 +1,220 @@ +# Exit on error +set -Eeuxo pipefail + +# Load common variables +source "$1/variations/common.sh" + +# CI + Setup variation: measures PM installation AND frozen-lockfile install. +# Identical to the `ci` variation but wraps the PM binary installation into the +# measured command, so the benchmark captures the end-to-end time of setting up +# the tool from scratch and running a CI install. +# +# Prepare step (not measured): +# 1. Clean node_modules, lockfiles, PM files +# 2. Install the PM binary (needed to generate the lockfile) +# 3. Run a regular install to generate the lockfile + populate cache +# 4. Clean node_modules (ci starts from scratch) +# 5. Uninstall the PM binary (so the measured step re-installs it) +# +# Measured command: +# 1. Install the PM binary +# 2. Run the PM's ci/frozen-lockfile command + +# --- PM install commands --- +# These mirror what scripts/setup.sh does for each PM. +# npm ships with Node so we reinstall the latest globally. +SETUP_INSTALL_NPM="npm install -g npm@latest >/dev/null 2>&1" +SETUP_INSTALL_YARN="" # corepack auto-downloads on first invocation +SETUP_INSTALL_BERRY="" # corepack auto-downloads on first invocation +SETUP_INSTALL_ZPM="" # corepack auto-downloads on first invocation +SETUP_INSTALL_PNPM="" # corepack auto-downloads on first invocation +SETUP_INSTALL_PACQUET="npm install --global pnpm@next-12 --prefix /tmp/pnpm12 --allow-scripts=pnpm >/dev/null 2>&1" +SETUP_INSTALL_VLT="npm install -g vlt@latest >/dev/null 2>&1" +SETUP_INSTALL_BUN="npm install -g bun@latest >/dev/null 2>&1" +SETUP_INSTALL_DENO="npm install -g deno@latest >/dev/null 2>&1" +SETUP_INSTALL_AUBE="npm install -g --allow-scripts=@endevco/aube @endevco/aube@latest >/dev/null 2>&1" +SETUP_INSTALL_UPM="npm install -g upm@latest >/dev/null 2>&1" + +# --- PM uninstall commands (run at end of prepare to force re-install) --- +SETUP_UNINSTALL_NPM="" # npm cannot uninstall itself; we wipe its global prefix instead +SETUP_UNINSTALL_YARN="" # corepack manages; we purge its cache below +SETUP_UNINSTALL_BERRY="" +SETUP_UNINSTALL_ZPM="" +SETUP_UNINSTALL_PNPM="" +SETUP_UNINSTALL_PACQUET="rm -rf /tmp/pnpm12" +SETUP_UNINSTALL_VLT="npm uninstall -g vlt >/dev/null 2>&1 || true" +SETUP_UNINSTALL_BUN="npm uninstall -g bun >/dev/null 2>&1 || true" +SETUP_UNINSTALL_DENO="npm uninstall -g deno >/dev/null 2>&1 || true" +SETUP_UNINSTALL_AUBE="npm uninstall -g @endevco/aube >/dev/null 2>&1 || true" +SETUP_UNINSTALL_UPM="npm uninstall -g upm >/dev/null 2>&1 || true" + +# For corepack-managed PMs (yarn, berry, zpm, pnpm), purge the corepack cache +# so the measured command includes downloading the PM binary. +COREPACK_PURGE="rm -rf \$(corepack cache path 2>/dev/null || echo /tmp/corepack-noop) 2>/dev/null || true" + +# --- CI commands (same as ci variation) --- +BENCH_CI_NPM="npm ci --no-audit --no-fund --ignore-scripts --silent" +if [ "$BENCH_FIXTURE" = "large" ]; then + BENCH_CI_NPM="$BENCH_CI_NPM --legacy-peer-deps" +fi +BENCH_CI_YARN="corepack yarn@1 install --frozen-lockfile --ignore-scripts --silent" +BENCH_CI_BERRY="corepack yarn@latest install --immutable" +BENCH_CI_ZPM="yarn install --immutable --silent" +BENCH_CI_PNPM="corepack pnpm@latest install --frozen-lockfile --ignore-scripts --silent" +BENCH_CI_PACQUET="/tmp/pnpm12/bin/pnpm install --frozen-lockfile --ignore-scripts --silent" +BENCH_CI_VLT="vlt ci --view=silent" +BENCH_CI_BUN="bun install --frozen-lockfile --ignore-scripts --silent" +BENCH_CI_DENO="deno install --frozen --quiet" +BENCH_CI_AUBE="aube ci --ignore-scripts --silent" +BENCH_CI_UPM="upm install --frozen-lockfile --silent" + +# --- Build measured commands: install PM + ci command --- +# Helper to combine PM install + ci command +setup_and_ci() { + local install_pm="$1" + local ci_cmd="$2" + if [ -n "$install_pm" ]; then + echo "$install_pm; $ci_cmd" + else + echo "$ci_cmd" + fi +} + +BENCH_COMBINED_NPM="$(setup_and_ci "$SETUP_INSTALL_NPM" "$BENCH_CI_NPM")" +BENCH_COMBINED_YARN="$(setup_and_ci "" "$BENCH_CI_YARN")" # corepack download is part of the yarn invocation +BENCH_COMBINED_BERRY="$(setup_and_ci "" "$BENCH_CI_BERRY")" +BENCH_COMBINED_ZPM="$(setup_and_ci "" "$BENCH_CI_ZPM")" +BENCH_COMBINED_PNPM="$(setup_and_ci "" "$BENCH_CI_PNPM")" +BENCH_COMBINED_PACQUET="$(setup_and_ci "$SETUP_INSTALL_PACQUET" "$BENCH_CI_PACQUET")" +BENCH_COMBINED_VLT="$(setup_and_ci "$SETUP_INSTALL_VLT" "$BENCH_CI_VLT")" +BENCH_COMBINED_BUN="$(setup_and_ci "$SETUP_INSTALL_BUN" "$BENCH_CI_BUN")" +BENCH_COMBINED_DENO="$(setup_and_ci "$SETUP_INSTALL_DENO" "$BENCH_CI_DENO")" +BENCH_COMBINED_AUBE="$(setup_and_ci "$SETUP_INSTALL_AUBE" "$BENCH_CI_AUBE")" +BENCH_COMBINED_UPM="$(setup_and_ci "$SETUP_INSTALL_UPM" "$BENCH_CI_UPM")" + +# Override BENCH_COMMAND_* with combined commands + log redirection +BENCH_COMMAND_NPM="timeout $BENCH_TIMEOUT bash -c '$BENCH_COMBINED_NPM' >> $BENCH_OUTPUT_FOLDER/npm-output-\${HYPERFINE_ITERATION}.log 2>&1" +BENCH_COMMAND_YARN="timeout $BENCH_TIMEOUT bash -c '$BENCH_COMBINED_YARN' > $BENCH_OUTPUT_FOLDER/yarn-output-\${HYPERFINE_ITERATION}.log 2>&1" +BENCH_COMMAND_BERRY="timeout $BENCH_TIMEOUT bash -c '$BENCH_COMBINED_BERRY' > $BENCH_OUTPUT_FOLDER/berry-output-\${HYPERFINE_ITERATION}.log 2>&1" +BENCH_COMMAND_ZPM="timeout $BENCH_TIMEOUT bash -c '$BENCH_COMBINED_ZPM' > $BENCH_OUTPUT_FOLDER/zpm-output-\${HYPERFINE_ITERATION}.log 2>&1" +BENCH_COMMAND_PNPM="timeout $BENCH_TIMEOUT bash -c '$BENCH_COMBINED_PNPM' > $BENCH_OUTPUT_FOLDER/pnpm-output-\${HYPERFINE_ITERATION}.log 2>&1" +BENCH_COMMAND_PACQUET="timeout $BENCH_TIMEOUT bash -c '$BENCH_COMBINED_PACQUET' > $BENCH_OUTPUT_FOLDER/pacquet-output-\${HYPERFINE_ITERATION}.log 2>&1" +BENCH_COMMAND_VLT="timeout $BENCH_TIMEOUT bash -c '$BENCH_COMBINED_VLT' > $BENCH_OUTPUT_FOLDER/vlt-output-\${HYPERFINE_ITERATION}.log 2>&1" +BENCH_COMMAND_BUN="timeout $BENCH_TIMEOUT bash -c '$BENCH_COMBINED_BUN' > $BENCH_OUTPUT_FOLDER/bun-output-\${HYPERFINE_ITERATION}.log 2>&1" +BENCH_COMMAND_DENO="timeout $BENCH_TIMEOUT bash -c '$BENCH_COMBINED_DENO' > $BENCH_OUTPUT_FOLDER/deno-output-\${HYPERFINE_ITERATION}.log 2>&1" +BENCH_COMMAND_AUBE="timeout $BENCH_TIMEOUT bash -c '$BENCH_COMBINED_AUBE' > $BENCH_OUTPUT_FOLDER/aube-output-\${HYPERFINE_ITERATION}.log 2>&1" +BENCH_COMMAND_UPM="timeout $BENCH_TIMEOUT bash -c '$BENCH_COMBINED_UPM' > $BENCH_OUTPUT_FOLDER/upm-output-\${HYPERFINE_ITERATION}.log 2>&1" + +# Override bare install commands for strace process counting (uses combined commands) +BENCH_INSTALL_NPM="bash -c '$BENCH_COMBINED_NPM'" +BENCH_INSTALL_YARN="bash -c '$BENCH_COMBINED_YARN'" +BENCH_INSTALL_BERRY="bash -c '$BENCH_COMBINED_BERRY'" +BENCH_INSTALL_ZPM="bash -c '$BENCH_COMBINED_ZPM'" +BENCH_INSTALL_PNPM="bash -c '$BENCH_COMBINED_PNPM'" +BENCH_INSTALL_PACQUET="bash -c '$BENCH_COMBINED_PACQUET'" +BENCH_INSTALL_VLT="bash -c '$BENCH_COMBINED_VLT'" +BENCH_INSTALL_BUN="bash -c '$BENCH_COMBINED_BUN'" +BENCH_INSTALL_DENO="bash -c '$BENCH_COMBINED_DENO'" +BENCH_INSTALL_AUBE="bash -c '$BENCH_COMBINED_AUBE'" +BENCH_INSTALL_UPM="bash -c '$BENCH_COMBINED_UPM'" + +# --- Prepare commands --- +# Like the ci variation, prepare generates a lockfile. Additionally, it +# uninstalls the PM binary so the measured command includes re-installing it. +BENCH_CLEAN_PRE="bash $BENCH_SCRIPTS/clean-helpers.sh clean_node_modules clean_lockfiles clean_package_manager_files clean_package_manager_field" +BENCH_CLEAN_POST="bash $BENCH_SCRIPTS/clean-helpers.sh clean_node_modules clean_package_manager_files" + +# Helper: build per-PM prepare that installs PM, generates lockfile, cleans +# node_modules, then uninstalls the PM binary. +ci_setup_prepare() { + local pm_install="$1" # command to install the PM binary + local setup_cmd="$2" # PM-specific setup (e.g. .yarnrc.yml) + local install_cmd="$3" # regular install to generate lockfile + local pm_uninstall="$4" # command to uninstall the PM binary + local corepack_purge="$5" # optional: purge corepack cache + + local result="sleep 1; $BENCH_CLEAN_PRE" + + # Install PM binary if needed (for non-corepack PMs) + if [ -n "$pm_install" ]; then + result="$result; $pm_install" + fi + + # PM-specific setup + if [ -n "$setup_cmd" ]; then + result="$result; $setup_cmd" + fi + + # Generate lockfile via regular install + result="$result; $install_cmd >/dev/null 2>&1 || true" + + # Clean node_modules (ci starts fresh) + result="$result; $BENCH_CLEAN_POST" + + # Uninstall PM binary so the measured command re-installs it + if [ -n "$pm_uninstall" ]; then + result="$result; $pm_uninstall" + fi + + # Purge corepack cache if applicable + if [ -n "$corepack_purge" ]; then + result="$result; $corepack_purge" + fi + + echo "$result" +} + +# Regular (non-ci) install commands to generate lockfiles during prepare +_INSTALL_NPM="npm install --no-audit --no-fund --ignore-scripts --silent" +if [ "$BENCH_FIXTURE" = "large" ]; then + _INSTALL_NPM="$_INSTALL_NPM --legacy-peer-deps" +fi + +BENCH_PREPARE_NPM="$(ci_setup_prepare "$SETUP_INSTALL_NPM" "$BENCH_SETUP_NPM" "$_INSTALL_NPM" "$SETUP_UNINSTALL_NPM" "")" +BENCH_PREPARE_YARN="$(ci_setup_prepare "$SETUP_INSTALL_YARN" "$BENCH_SETUP_YARN" "corepack yarn@1 install --ignore-scripts --silent" "$SETUP_UNINSTALL_YARN" "$COREPACK_PURGE")" +BENCH_PREPARE_BERRY="$(ci_setup_prepare "$SETUP_INSTALL_BERRY" "$BENCH_SETUP_BERRY" "corepack yarn@latest install" "$SETUP_UNINSTALL_BERRY" "$COREPACK_PURGE")" +BENCH_PREPARE_ZPM="$(ci_setup_prepare "$SETUP_INSTALL_ZPM" "$BENCH_SETUP_ZPM" "yarn install --silent" "$SETUP_UNINSTALL_ZPM" "$COREPACK_PURGE")" +BENCH_PREPARE_PNPM="$(ci_setup_prepare "$SETUP_INSTALL_PNPM" "$BENCH_SETUP_PNPM" "corepack pnpm@latest install --ignore-scripts --silent" "$SETUP_UNINSTALL_PNPM" "$COREPACK_PURGE")" +BENCH_PREPARE_PACQUET="$(ci_setup_prepare "$SETUP_INSTALL_PACQUET" "$BENCH_SETUP_PACQUET" "/tmp/pnpm12/bin/pnpm install --ignore-scripts --silent" "$SETUP_UNINSTALL_PACQUET" "")" +BENCH_PREPARE_VLT="$(ci_setup_prepare "$SETUP_INSTALL_VLT" "$BENCH_SETUP_VLT" "vlt install --view=silent" "$SETUP_UNINSTALL_VLT" "")" +BENCH_PREPARE_BUN="$(ci_setup_prepare "$SETUP_INSTALL_BUN" "$BENCH_SETUP_BUN" "bun install --ignore-scripts --silent" "$SETUP_UNINSTALL_BUN" "")" +BENCH_PREPARE_DENO="$(ci_setup_prepare "$SETUP_INSTALL_DENO" "$BENCH_SETUP_DENO" "deno install --quiet" "$SETUP_UNINSTALL_DENO" "")" +BENCH_PREPARE_AUBE="$(ci_setup_prepare "$SETUP_INSTALL_AUBE" "$BENCH_SETUP_AUBE" "aube install --ignore-scripts --silent" "$SETUP_UNINSTALL_AUBE" "")" +BENCH_PREPARE_UPM="$(ci_setup_prepare "$SETUP_INSTALL_UPM" "$BENCH_SETUP_UPM" "upm install --silent" "$SETUP_UNINSTALL_UPM" "")" + +# Run the benchmark suite +hyperfine --ignore-failure \ + --time-unit=millisecond \ + --export-json="$BENCH_OUTPUT_FOLDER/benchmarks.json" \ + --warmup="$BENCH_WARMUP" \ + --runs="$BENCH_RUNS" \ + --conclude="sleep 1; bash $BENCH_SCRIPTS/package-count.sh $BENCH_OUTPUT_FOLDER; bash $BENCH_SCRIPTS/clean-helpers.sh clean_node_modules clean_lockfiles clean_package_manager_files clean_package_manager_field clean_build_files" \ + --cleanup="bash $BENCH_SCRIPTS/clean-helpers.sh clean_all" \ + ${BENCH_INCLUDE_NPM:+--prepare="$BENCH_PREPARE_NPM"} \ + ${BENCH_INCLUDE_NPM:+--command-name="npm" "$BENCH_COMMAND_NPM"} \ + ${BENCH_INCLUDE_YARN:+--prepare="$BENCH_PREPARE_YARN"} \ + ${BENCH_INCLUDE_YARN:+--command-name="yarn" "$BENCH_COMMAND_YARN"} \ + ${BENCH_INCLUDE_BERRY:+--prepare="$BENCH_PREPARE_BERRY"} \ + ${BENCH_INCLUDE_BERRY:+--command-name="berry" "$BENCH_COMMAND_BERRY"} \ + ${BENCH_INCLUDE_ZPM:+--prepare="$BENCH_PREPARE_ZPM"} \ + ${BENCH_INCLUDE_ZPM:+--command-name="zpm" "$BENCH_COMMAND_ZPM"} \ + ${BENCH_INCLUDE_PNPM:+--prepare="$BENCH_PREPARE_PNPM"} \ + ${BENCH_INCLUDE_PNPM:+--command-name="pnpm" "$BENCH_COMMAND_PNPM"} \ + ${BENCH_INCLUDE_PACQUET:+--prepare="$BENCH_PREPARE_PACQUET"} \ + ${BENCH_INCLUDE_PACQUET:+--command-name="pacquet" "$BENCH_COMMAND_PACQUET"} \ + ${BENCH_INCLUDE_VLT:+--prepare="$BENCH_PREPARE_VLT"} \ + ${BENCH_INCLUDE_VLT:+--command-name="vlt" "$BENCH_COMMAND_VLT"} \ + ${BENCH_INCLUDE_BUN:+--prepare="$BENCH_PREPARE_BUN"} \ + ${BENCH_INCLUDE_BUN:+--command-name="bun" "$BENCH_COMMAND_BUN"} \ + ${BENCH_INCLUDE_DENO:+--prepare="$BENCH_PREPARE_DENO"} \ + ${BENCH_INCLUDE_DENO:+--command-name="deno" "$BENCH_COMMAND_DENO"} \ + ${BENCH_INCLUDE_AUBE:+--prepare="$BENCH_PREPARE_AUBE"} \ + ${BENCH_INCLUDE_AUBE:+--command-name="aube" "$BENCH_COMMAND_AUBE"} \ + ${BENCH_INCLUDE_UPM:+--prepare="$BENCH_PREPARE_UPM"} \ + ${BENCH_INCLUDE_UPM:+--command-name="upm" "$BENCH_COMMAND_UPM"} + +collect_package_count + +# For process counting, use a representative prepare base. +BENCH_PREPARE_BASE="$BENCH_PREPARE_NPM" +collect_process_count From d2d99f059163da58850f49b6aef8642ba6fa2cc2 Mon Sep 17 00:00:00 2001 From: vltbaudbot <264001112+vltbaudbot@users.noreply.github.com> Date: Wed, 30 Sep 2026 14:58:57 +0000 Subject: [PATCH 2/3] Address review: isolate corepack/npm/zpm setup, fix average consistency Address CodeRabbit review comments: 1. Remove ci+setup from default average calculation in utils.ts to keep the main chart average consistent with the history chart average. ci+setup remains a standalone viewable variation. 2. Use an isolated COREPACK_HOME (/tmp/bench-corepack-home) instead of relying on the undocumented 'corepack cache path' subcommand. This makes purging deterministic and prevents affecting other benchmarks. 3. Install npm into a dedicated prefix (/tmp/bench-npm) with a separate bootstrap cache (/tmp/bench-npm-cache), and properly uninstall both between runs. The measured command uses the prefix binary, preserving the dependency-level npm cache for npm ci. 4. Add Yarn Switch install/uninstall for zpm (matching setup.sh), and remove its Yarn Switch directory between runs instead of only purging corepack cache. Co-authored-by: Evert Pot --- app/src/lib/utils.ts | 1 - scripts/variations/ci+setup.sh | 38 +++++++++++++++++++++------------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/app/src/lib/utils.ts b/app/src/lib/utils.ts index eb1fcd9664..08b15d9e92 100644 --- a/app/src/lib/utils.ts +++ b/app/src/lib/utils.ts @@ -66,7 +66,6 @@ export const calculateAverageVariationData = ( "cache+lockfile", "cache+lockfile+node_modules", "ci", - "ci+setup", "lockfile", "lockfile+node_modules", ] diff --git a/scripts/variations/ci+setup.sh b/scripts/variations/ci+setup.sh index 1d4e3751e2..c78b3cd2b9 100644 --- a/scripts/variations/ci+setup.sh +++ b/scripts/variations/ci+setup.sh @@ -20,13 +20,23 @@ source "$1/variations/common.sh" # 1. Install the PM binary # 2. Run the PM's ci/frozen-lockfile command +# Use an isolated corepack home so purging it between runs doesn't affect +# other benchmarks, and so the path is deterministic (no reliance on +# `corepack cache path` which is not a documented subcommand). +export COREPACK_HOME="/tmp/bench-corepack-home" + # --- PM install commands --- # These mirror what scripts/setup.sh does for each PM. -# npm ships with Node so we reinstall the latest globally. -SETUP_INSTALL_NPM="npm install -g npm@latest >/dev/null 2>&1" + +# npm: install into a dedicated prefix so the bootstrap npm and its cache are +# separate from the dependency-level npm cache used by `npm ci`. +NPM_SETUP_PREFIX="/tmp/bench-npm" +NPM_SETUP_CACHE="/tmp/bench-npm-cache" +SETUP_INSTALL_NPM="npm install -g npm@latest --prefix $NPM_SETUP_PREFIX --cache $NPM_SETUP_CACHE >/dev/null 2>&1" SETUP_INSTALL_YARN="" # corepack auto-downloads on first invocation SETUP_INSTALL_BERRY="" # corepack auto-downloads on first invocation -SETUP_INSTALL_ZPM="" # corepack auto-downloads on first invocation +# zpm is installed via the Yarn Switch installer (same as setup.sh) +SETUP_INSTALL_ZPM="curl -sS https://repo.yarnpkg.com/install | bash >/dev/null 2>&1" SETUP_INSTALL_PNPM="" # corepack auto-downloads on first invocation SETUP_INSTALL_PACQUET="npm install --global pnpm@next-12 --prefix /tmp/pnpm12 --allow-scripts=pnpm >/dev/null 2>&1" SETUP_INSTALL_VLT="npm install -g vlt@latest >/dev/null 2>&1" @@ -36,10 +46,10 @@ SETUP_INSTALL_AUBE="npm install -g --allow-scripts=@endevco/aube @endevco/aube@l SETUP_INSTALL_UPM="npm install -g upm@latest >/dev/null 2>&1" # --- PM uninstall commands (run at end of prepare to force re-install) --- -SETUP_UNINSTALL_NPM="" # npm cannot uninstall itself; we wipe its global prefix instead -SETUP_UNINSTALL_YARN="" # corepack manages; we purge its cache below +SETUP_UNINSTALL_NPM="rm -rf $NPM_SETUP_PREFIX $NPM_SETUP_CACHE" +SETUP_UNINSTALL_YARN="" # corepack managed; cache purged below SETUP_UNINSTALL_BERRY="" -SETUP_UNINSTALL_ZPM="" +SETUP_UNINSTALL_ZPM="rm -rf \$HOME/.yarn/switch >/dev/null 2>&1 || true" SETUP_UNINSTALL_PNPM="" SETUP_UNINSTALL_PACQUET="rm -rf /tmp/pnpm12" SETUP_UNINSTALL_VLT="npm uninstall -g vlt >/dev/null 2>&1 || true" @@ -48,12 +58,11 @@ SETUP_UNINSTALL_DENO="npm uninstall -g deno >/dev/null 2>&1 || true" SETUP_UNINSTALL_AUBE="npm uninstall -g @endevco/aube >/dev/null 2>&1 || true" SETUP_UNINSTALL_UPM="npm uninstall -g upm >/dev/null 2>&1 || true" -# For corepack-managed PMs (yarn, berry, zpm, pnpm), purge the corepack cache -# so the measured command includes downloading the PM binary. -COREPACK_PURGE="rm -rf \$(corepack cache path 2>/dev/null || echo /tmp/corepack-noop) 2>/dev/null || true" +# Purge the isolated corepack home so corepack-managed PMs must re-download. +COREPACK_PURGE="rm -rf $COREPACK_HOME" # --- CI commands (same as ci variation) --- -BENCH_CI_NPM="npm ci --no-audit --no-fund --ignore-scripts --silent" +BENCH_CI_NPM="$NPM_SETUP_PREFIX/bin/npm ci --no-audit --no-fund --ignore-scripts --silent" if [ "$BENCH_FIXTURE" = "large" ]; then BENCH_CI_NPM="$BENCH_CI_NPM --legacy-peer-deps" fi @@ -83,7 +92,7 @@ setup_and_ci() { BENCH_COMBINED_NPM="$(setup_and_ci "$SETUP_INSTALL_NPM" "$BENCH_CI_NPM")" BENCH_COMBINED_YARN="$(setup_and_ci "" "$BENCH_CI_YARN")" # corepack download is part of the yarn invocation BENCH_COMBINED_BERRY="$(setup_and_ci "" "$BENCH_CI_BERRY")" -BENCH_COMBINED_ZPM="$(setup_and_ci "" "$BENCH_CI_ZPM")" +BENCH_COMBINED_ZPM="$(setup_and_ci "$SETUP_INSTALL_ZPM" "$BENCH_CI_ZPM")" BENCH_COMBINED_PNPM="$(setup_and_ci "" "$BENCH_CI_PNPM")" BENCH_COMBINED_PACQUET="$(setup_and_ci "$SETUP_INSTALL_PACQUET" "$BENCH_CI_PACQUET")" BENCH_COMBINED_VLT="$(setup_and_ci "$SETUP_INSTALL_VLT" "$BENCH_CI_VLT")" @@ -164,8 +173,9 @@ ci_setup_prepare() { echo "$result" } -# Regular (non-ci) install commands to generate lockfiles during prepare -_INSTALL_NPM="npm install --no-audit --no-fund --ignore-scripts --silent" +# Regular (non-ci) install commands to generate lockfiles during prepare. +# npm uses the dedicated prefix binary for lockfile generation too. +_INSTALL_NPM="$NPM_SETUP_PREFIX/bin/npm install --no-audit --no-fund --ignore-scripts --silent" if [ "$BENCH_FIXTURE" = "large" ]; then _INSTALL_NPM="$_INSTALL_NPM --legacy-peer-deps" fi @@ -173,7 +183,7 @@ fi BENCH_PREPARE_NPM="$(ci_setup_prepare "$SETUP_INSTALL_NPM" "$BENCH_SETUP_NPM" "$_INSTALL_NPM" "$SETUP_UNINSTALL_NPM" "")" BENCH_PREPARE_YARN="$(ci_setup_prepare "$SETUP_INSTALL_YARN" "$BENCH_SETUP_YARN" "corepack yarn@1 install --ignore-scripts --silent" "$SETUP_UNINSTALL_YARN" "$COREPACK_PURGE")" BENCH_PREPARE_BERRY="$(ci_setup_prepare "$SETUP_INSTALL_BERRY" "$BENCH_SETUP_BERRY" "corepack yarn@latest install" "$SETUP_UNINSTALL_BERRY" "$COREPACK_PURGE")" -BENCH_PREPARE_ZPM="$(ci_setup_prepare "$SETUP_INSTALL_ZPM" "$BENCH_SETUP_ZPM" "yarn install --silent" "$SETUP_UNINSTALL_ZPM" "$COREPACK_PURGE")" +BENCH_PREPARE_ZPM="$(ci_setup_prepare "$SETUP_INSTALL_ZPM" "$BENCH_SETUP_ZPM" "yarn install --silent" "$SETUP_UNINSTALL_ZPM" "")" BENCH_PREPARE_PNPM="$(ci_setup_prepare "$SETUP_INSTALL_PNPM" "$BENCH_SETUP_PNPM" "corepack pnpm@latest install --ignore-scripts --silent" "$SETUP_UNINSTALL_PNPM" "$COREPACK_PURGE")" BENCH_PREPARE_PACQUET="$(ci_setup_prepare "$SETUP_INSTALL_PACQUET" "$BENCH_SETUP_PACQUET" "/tmp/pnpm12/bin/pnpm install --ignore-scripts --silent" "$SETUP_UNINSTALL_PACQUET" "")" BENCH_PREPARE_VLT="$(ci_setup_prepare "$SETUP_INSTALL_VLT" "$BENCH_SETUP_VLT" "vlt install --view=silent" "$SETUP_UNINSTALL_VLT" "")" From f0c4e717e1075c143c164400496233acff8972be Mon Sep 17 00:00:00 2001 From: vltbaudbot <264001112+vltbaudbot@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:24:10 +0000 Subject: [PATCH 3/3] Address Copilot review: cache isolation, error propagation, per-PM process count 1. Average consistency (utils.ts:168): ci+setup is intentionally excluded from calculateAverageVariationData defaults (already done in prior commit) to match the history average in use-history-data.ts. ci+setup appears in the Package Management category for standalone viewing but doesn't skew the average with setup overhead. 2. Pacquet warm npm cache (ci+setup.sh:54): Give the pacquet bootstrap install a dedicated npm cache directory (PACQUET_SETUP_CACHE) and purge it alongside /tmp/pnpm12 in SETUP_UNINSTALL_PACQUET, so the measured command does a from-scratch npm install without reusing cached tarballs from the prepare step. 3. Use && instead of ; in setup_and_ci (ci+setup.sh:89): Failed PM installations now fail fast via && instead of silently falling through to the ci command with ;. This prevents hyperfine from recording a successful timing when the setup step failed but an older binary was left behind. 4. Per-PM process count (ci+setup.sh:230): Replace the single BENCH_PREPARE_BASE with a custom collect_process_count_per_pm function that passes each PM's own BENCH_PREPARE_ command to process-count.sh. This ensures each PM gets its correct lockfile, PM binary uninstall, and corepack purge before the strace'd run. Co-authored-by: Evert Pot --- scripts/variations/ci+setup.sh | 83 +++++++++++++++++++++++++++++++--- 1 file changed, 76 insertions(+), 7 deletions(-) diff --git a/scripts/variations/ci+setup.sh b/scripts/variations/ci+setup.sh index c78b3cd2b9..106961762d 100644 --- a/scripts/variations/ci+setup.sh +++ b/scripts/variations/ci+setup.sh @@ -38,7 +38,9 @@ SETUP_INSTALL_BERRY="" # corepack auto-downloads on first invocation # zpm is installed via the Yarn Switch installer (same as setup.sh) SETUP_INSTALL_ZPM="curl -sS https://repo.yarnpkg.com/install | bash >/dev/null 2>&1" SETUP_INSTALL_PNPM="" # corepack auto-downloads on first invocation -SETUP_INSTALL_PACQUET="npm install --global pnpm@next-12 --prefix /tmp/pnpm12 --allow-scripts=pnpm >/dev/null 2>&1" +# pacquet: use a dedicated npm cache so the pnpm tarball isn't warm from prepare +PACQUET_SETUP_CACHE="/tmp/bench-pacquet-cache" +SETUP_INSTALL_PACQUET="npm install --global pnpm@next-12 --prefix /tmp/pnpm12 --cache $PACQUET_SETUP_CACHE --allow-scripts=pnpm >/dev/null 2>&1" SETUP_INSTALL_VLT="npm install -g vlt@latest >/dev/null 2>&1" SETUP_INSTALL_BUN="npm install -g bun@latest >/dev/null 2>&1" SETUP_INSTALL_DENO="npm install -g deno@latest >/dev/null 2>&1" @@ -51,7 +53,7 @@ SETUP_UNINSTALL_YARN="" # corepack managed; cache purged below SETUP_UNINSTALL_BERRY="" SETUP_UNINSTALL_ZPM="rm -rf \$HOME/.yarn/switch >/dev/null 2>&1 || true" SETUP_UNINSTALL_PNPM="" -SETUP_UNINSTALL_PACQUET="rm -rf /tmp/pnpm12" +SETUP_UNINSTALL_PACQUET="rm -rf /tmp/pnpm12 $PACQUET_SETUP_CACHE" SETUP_UNINSTALL_VLT="npm uninstall -g vlt >/dev/null 2>&1 || true" SETUP_UNINSTALL_BUN="npm uninstall -g bun >/dev/null 2>&1 || true" SETUP_UNINSTALL_DENO="npm uninstall -g deno >/dev/null 2>&1 || true" @@ -78,12 +80,14 @@ BENCH_CI_AUBE="aube ci --ignore-scripts --silent" BENCH_CI_UPM="upm install --frozen-lockfile --silent" # --- Build measured commands: install PM + ci command --- -# Helper to combine PM install + ci command +# Helper to combine PM install + ci command. Uses && so a failed PM +# installation fails fast instead of silently falling through to the +# ci command (which could succeed against a stale binary left behind). setup_and_ci() { local install_pm="$1" local ci_cmd="$2" if [ -n "$install_pm" ]; then - echo "$install_pm; $ci_cmd" + echo "$install_pm && $ci_cmd" else echo "$ci_cmd" fi @@ -225,6 +229,71 @@ hyperfine --ignore-failure \ collect_package_count -# For process counting, use a representative prepare base. -BENCH_PREPARE_BASE="$BENCH_PREPARE_NPM" -collect_process_count +# --- Process count collection --- +# Unlike other variations that use a single BENCH_PREPARE_BASE for all PMs, +# ci+setup needs per-PM prepare commands because each PM's prepare includes +# PM-specific uninstall/cache-purge steps. Using a single base (e.g. npm's +# prepare) would leave the wrong lockfile and skip corepack purges for +# non-npm PMs, producing inaccurate process counts. +collect_process_count_per_pm() { + if ! command -v strace &>/dev/null; then + echo "Warning: strace not available, skipping process count collection" + return 0 + fi + + echo "=== Collecting spawned process counts (per-PM prepare) ===" + + local -A PM_PREPARE=( + [npm]="$BENCH_PREPARE_NPM" + [yarn]="$BENCH_PREPARE_YARN" + [berry]="$BENCH_PREPARE_BERRY" + [zpm]="$BENCH_PREPARE_ZPM" + [pnpm]="$BENCH_PREPARE_PNPM" + [pacquet]="$BENCH_PREPARE_PACQUET" + [vlt]="$BENCH_PREPARE_VLT" + [bun]="$BENCH_PREPARE_BUN" + [deno]="$BENCH_PREPARE_DENO" + [aube]="$BENCH_PREPARE_AUBE" + [upm]="$BENCH_PREPARE_UPM" + ) + local -A PM_INSTALL=( + [npm]="$BENCH_INSTALL_NPM" + [yarn]="$BENCH_INSTALL_YARN" + [berry]="$BENCH_INSTALL_BERRY" + [zpm]="$BENCH_INSTALL_ZPM" + [pnpm]="$BENCH_INSTALL_PNPM" + [pacquet]="$BENCH_INSTALL_PACQUET" + [vlt]="$BENCH_INSTALL_VLT" + [bun]="$BENCH_INSTALL_BUN" + [deno]="$BENCH_INSTALL_DENO" + [aube]="$BENCH_INSTALL_AUBE" + [upm]="$BENCH_INSTALL_UPM" + ) + local -A PM_INCLUDE=( + [npm]="$BENCH_INCLUDE_NPM" + [yarn]="$BENCH_INCLUDE_YARN" + [berry]="$BENCH_INCLUDE_BERRY" + [zpm]="$BENCH_INCLUDE_ZPM" + [pnpm]="$BENCH_INCLUDE_PNPM" + [pacquet]="$BENCH_INCLUDE_PACQUET" + [vlt]="$BENCH_INCLUDE_VLT" + [bun]="$BENCH_INCLUDE_BUN" + [deno]="$BENCH_INCLUDE_DENO" + [aube]="$BENCH_INCLUDE_AUBE" + [upm]="$BENCH_INCLUDE_UPM" + ) + + for pm in npm yarn berry zpm pnpm pacquet vlt bun deno aube upm; do + if [ -n "${PM_INCLUDE[$pm]:-}" ]; then + bash "$BENCH_SCRIPTS/process-count.sh" \ + "$BENCH_OUTPUT_FOLDER" \ + "$pm" \ + "${PM_INSTALL[$pm]}" \ + "${PM_PREPARE[$pm]}" + fi + done + + node "$BENCH_SCRIPTS/collect-process-count.js" "$BENCH_OUTPUT_FOLDER" +} + +collect_process_count_per_pm