# Exit on error set -Eeuxo pipefail # Load common variables source "$1/variations/common.sh" # CI + Setup variation: measures PM installation AND frozen-lockfile install. # Identical to the `ci` variation but wraps the PM binary installation into the # measured command, so the benchmark captures the end-to-end time of setting up # the tool from scratch and running a CI install. # # Prepare step (not measured): # Once per tool, on its first prepare (see ci_setup_prepare): # 1. Clean node_modules, lockfiles, PM files # 2. Install the PM binary (needed to generate the lockfile) # 3. Run a regular install to generate the lockfile + populate cache # Before every timed run, this first one included: # 4. Clean node_modules (ci starts from scratch) # 5. Uninstall the PM binary (so the measured step re-installs it) # The lockfile and the dependency cache stay from one run to the next: only # the timed command's own work (the PM download and install, node_modules) # is undone. hyperfine's --conclude must leave them too. hyperfine 1.19 has # no per-command --setup, hence the marker file. # # Measured command: # 1. Install the PM binary # 2. Run the PM's ci/frozen-lockfile command # Where the PM prefixes, their npm caches and the corepack home live. Tests # point it somewhere private. BENCH_SETUP_TMP="${BENCH_SETUP_TMP:-/tmp}" # Use an isolated corepack home so purging it between runs doesn't affect # other benchmarks, and so the path is deterministic (no reliance on # `corepack cache path` which is not a documented subcommand). export COREPACK_HOME="$BENCH_SETUP_TMP/bench-corepack-home" # --- PM install commands --- # These mirror what scripts/setup.sh does for each PM. # npm, yarn classic, pnpm and berry's corepack: install into a dedicated prefix # with a dedicated npm cache, both removed by the uninstall step, so every # timed install downloads the package manager again. This also keeps the # bootstrap npm and the copies installed by setup.sh untouched, and the npm # cache used by `npm ci` separate. NPM_SETUP_PREFIX="$BENCH_SETUP_TMP/bench-npm" NPM_SETUP_CACHE="$BENCH_SETUP_TMP/bench-npm-cache" YARN_SETUP_PREFIX="$BENCH_SETUP_TMP/bench-yarn" YARN_SETUP_CACHE="$BENCH_SETUP_TMP/bench-yarn-npm-cache" PNPM_SETUP_PREFIX="$BENCH_SETUP_TMP/bench-pnpm" PNPM_SETUP_CACHE="$BENCH_SETUP_TMP/bench-pnpm-npm-cache" COREPACK_SETUP_PREFIX="$BENCH_SETUP_TMP/bench-corepack" COREPACK_SETUP_CACHE="$BENCH_SETUP_TMP/bench-corepack-npm-cache" SETUP_INSTALL_NPM="npm install -g npm@latest --prefix $NPM_SETUP_PREFIX --cache $NPM_SETUP_CACHE >/dev/null 2>&1" SETUP_INSTALL_YARN="npm install -g yarn@1 --prefix $YARN_SETUP_PREFIX --cache $YARN_SETUP_CACHE >/dev/null 2>&1" # berry: install corepack; its yarn shim then downloads the pinned yarn on # first use (COREPACK_HOME is purged too). SETUP_INSTALL_BERRY="npm install -g corepack@latest --prefix $COREPACK_SETUP_PREFIX --cache $COREPACK_SETUP_CACHE >/dev/null 2>&1" # zpm is installed via the Yarn Switch installer (same as setup.sh) SETUP_INSTALL_ZPM="curl -sS https://repo.yarnpkg.com/install | bash >/dev/null 2>&1" # --allow-scripts=pnpm lets pnpm's postinstall put its native binary in place. SETUP_INSTALL_PNPM="npm install -g --allow-scripts=pnpm pnpm@latest --prefix $PNPM_SETUP_PREFIX --cache $PNPM_SETUP_CACHE >/dev/null 2>&1" # The tools installed with npm get their own npm cache, deleted with the tool # below, like npm's. Otherwise the timed install is served from ~/.npm, which # the prepare's own install of the same tool just filled. TOOL_SETUP_CACHE="$BENCH_SETUP_TMP/bench-tool-setup-cache" SETUP_INSTALL_VLT="npm install -g --cache $TOOL_SETUP_CACHE vlt@latest >/dev/null 2>&1" SETUP_INSTALL_BUN="npm install -g --cache $TOOL_SETUP_CACHE --allow-scripts=bun bun@latest >/dev/null 2>&1" SETUP_INSTALL_DENO="npm install -g --cache $TOOL_SETUP_CACHE --allow-scripts=deno deno@latest >/dev/null 2>&1" SETUP_INSTALL_AUBE="npm install -g --cache $TOOL_SETUP_CACHE --allow-scripts=@endevco/aube @endevco/aube@latest >/dev/null 2>&1" SETUP_INSTALL_UPM="npm install -g --cache $TOOL_SETUP_CACHE upm@latest >/dev/null 2>&1" # --- PM uninstall commands (run at end of prepare to force re-install) --- SETUP_UNINSTALL_NPM="rm -rf $NPM_SETUP_PREFIX $NPM_SETUP_CACHE" SETUP_UNINSTALL_YARN="rm -rf $YARN_SETUP_PREFIX $YARN_SETUP_CACHE" SETUP_UNINSTALL_BERRY="rm -rf $COREPACK_SETUP_PREFIX $COREPACK_SETUP_CACHE" SETUP_UNINSTALL_ZPM="rm -rf \$HOME/.yarn/switch >/dev/null 2>&1 || true" SETUP_UNINSTALL_PNPM="rm -rf $PNPM_SETUP_PREFIX $PNPM_SETUP_CACHE" SETUP_UNINSTALL_VLT="npm uninstall -g vlt >/dev/null 2>&1 || true; rm -rf $TOOL_SETUP_CACHE" SETUP_UNINSTALL_BUN="npm uninstall -g bun >/dev/null 2>&1 || true; rm -rf $TOOL_SETUP_CACHE" SETUP_UNINSTALL_DENO="npm uninstall -g deno >/dev/null 2>&1 || true; rm -rf $TOOL_SETUP_CACHE" SETUP_UNINSTALL_AUBE="npm uninstall -g @endevco/aube >/dev/null 2>&1 || true; rm -rf $TOOL_SETUP_CACHE" SETUP_UNINSTALL_UPM="npm uninstall -g upm >/dev/null 2>&1 || true; rm -rf $TOOL_SETUP_CACHE" # Purge the isolated corepack home so berry must be downloaded again. COREPACK_PURGE="rm -rf $COREPACK_HOME" # --- CI commands (same as ci variation) --- BENCH_CI_NPM="$NPM_SETUP_PREFIX/bin/npm ci --no-audit --no-fund --ignore-scripts --silent" if [ "$BENCH_FIXTURE" = "large" ]; then BENCH_CI_NPM="$BENCH_CI_NPM --legacy-peer-deps" fi BENCH_CI_YARN="$YARN_SETUP_PREFIX/bin/yarn install --frozen-lockfile --ignore-scripts --silent" BENCH_CI_BERRY="$COREPACK_SETUP_PREFIX/bin/yarn install --immutable" BENCH_CI_ZPM="$BENCH_ZPM_BIN install --immutable --silent" BENCH_CI_PNPM="$PNPM_SETUP_PREFIX/bin/pnpm install --frozen-lockfile --ignore-scripts --silent" BENCH_CI_VLT="vlt ci --view=silent" BENCH_CI_BUN="bun install --frozen-lockfile --ignore-scripts --silent" BENCH_CI_DENO="deno install --frozen --quiet" BENCH_CI_AUBE="aube ci --ignore-scripts --silent" BENCH_CI_UPM="upm install --frozen-lockfile --silent" # --- Build measured commands: install PM + ci command --- # Helper to combine PM install + ci command. Uses && so a failed PM # installation fails fast instead of silently falling through to the # ci command (which could succeed against a stale binary left behind). setup_and_ci() { local install_pm="$1" local ci_cmd="$2" if [ -n "$install_pm" ]; then echo "$install_pm && $ci_cmd" else echo "$ci_cmd" fi } BENCH_COMBINED_NPM="$(setup_and_ci "$SETUP_INSTALL_NPM" "$BENCH_CI_NPM")" BENCH_COMBINED_YARN="$(setup_and_ci "$SETUP_INSTALL_YARN" "$BENCH_CI_YARN")" BENCH_COMBINED_BERRY="$(setup_and_ci "$SETUP_INSTALL_BERRY" "$BENCH_CI_BERRY")" # + corepack's download of yarn BENCH_COMBINED_ZPM="$(setup_and_ci "$SETUP_INSTALL_ZPM" "$BENCH_CI_ZPM")" BENCH_COMBINED_PNPM="$(setup_and_ci "$SETUP_INSTALL_PNPM" "$BENCH_CI_PNPM")" BENCH_COMBINED_VLT="$(setup_and_ci "$SETUP_INSTALL_VLT" "$BENCH_CI_VLT")" BENCH_COMBINED_BUN="$(setup_and_ci "$SETUP_INSTALL_BUN" "$BENCH_CI_BUN")" BENCH_COMBINED_DENO="$(setup_and_ci "$SETUP_INSTALL_DENO" "$BENCH_CI_DENO")" BENCH_COMBINED_AUBE="$(setup_and_ci "$SETUP_INSTALL_AUBE" "$BENCH_CI_AUBE")" BENCH_COMBINED_UPM="$(setup_and_ci "$SETUP_INSTALL_UPM" "$BENCH_CI_UPM")" # Override BENCH_COMMAND_* with combined commands + log redirection BENCH_COMMAND_NPM="timeout $BENCH_TIMEOUT bash -c '$BENCH_COMBINED_NPM' >> $BENCH_OUTPUT_FOLDER/npm-output-\${HYPERFINE_ITERATION}.log 2>&1" BENCH_COMMAND_YARN="timeout $BENCH_TIMEOUT bash -c '$BENCH_COMBINED_YARN' > $BENCH_OUTPUT_FOLDER/yarn-output-\${HYPERFINE_ITERATION}.log 2>&1" BENCH_COMMAND_BERRY="timeout $BENCH_TIMEOUT bash -c '$BENCH_COMBINED_BERRY' > $BENCH_OUTPUT_FOLDER/berry-output-\${HYPERFINE_ITERATION}.log 2>&1" BENCH_COMMAND_ZPM="timeout $BENCH_TIMEOUT bash -c '$BENCH_COMBINED_ZPM' > $BENCH_OUTPUT_FOLDER/zpm-output-\${HYPERFINE_ITERATION}.log 2>&1" BENCH_COMMAND_PNPM="timeout $BENCH_TIMEOUT bash -c '$BENCH_COMBINED_PNPM' > $BENCH_OUTPUT_FOLDER/pnpm-output-\${HYPERFINE_ITERATION}.log 2>&1" BENCH_COMMAND_VLT="timeout $BENCH_TIMEOUT bash -c '$BENCH_COMBINED_VLT' > $BENCH_OUTPUT_FOLDER/vlt-output-\${HYPERFINE_ITERATION}.log 2>&1" BENCH_COMMAND_BUN="timeout $BENCH_TIMEOUT bash -c '$BENCH_COMBINED_BUN' > $BENCH_OUTPUT_FOLDER/bun-output-\${HYPERFINE_ITERATION}.log 2>&1" BENCH_COMMAND_DENO="timeout $BENCH_TIMEOUT bash -c '$BENCH_COMBINED_DENO' > $BENCH_OUTPUT_FOLDER/deno-output-\${HYPERFINE_ITERATION}.log 2>&1" BENCH_COMMAND_AUBE="timeout $BENCH_TIMEOUT bash -c '$BENCH_COMBINED_AUBE' > $BENCH_OUTPUT_FOLDER/aube-output-\${HYPERFINE_ITERATION}.log 2>&1" BENCH_COMMAND_UPM="timeout $BENCH_TIMEOUT bash -c '$BENCH_COMBINED_UPM' > $BENCH_OUTPUT_FOLDER/upm-output-\${HYPERFINE_ITERATION}.log 2>&1" # Override bare install commands for strace process counting (uses combined commands) BENCH_INSTALL_NPM="bash -c '$BENCH_COMBINED_NPM'" BENCH_INSTALL_YARN="bash -c '$BENCH_COMBINED_YARN'" BENCH_INSTALL_BERRY="bash -c '$BENCH_COMBINED_BERRY'" BENCH_INSTALL_ZPM="bash -c '$BENCH_COMBINED_ZPM'" BENCH_INSTALL_PNPM="bash -c '$BENCH_COMBINED_PNPM'" BENCH_INSTALL_VLT="bash -c '$BENCH_COMBINED_VLT'" BENCH_INSTALL_BUN="bash -c '$BENCH_COMBINED_BUN'" BENCH_INSTALL_DENO="bash -c '$BENCH_COMBINED_DENO'" BENCH_INSTALL_AUBE="bash -c '$BENCH_COMBINED_AUBE'" BENCH_INSTALL_UPM="bash -c '$BENCH_COMBINED_UPM'" # --- Prepare commands --- # Like the ci variation, prepare generates a lockfile. Additionally, it # uninstalls the PM binary so the measured command includes re-installing it. # The lockfile install runs once per tool, not before every timed run: it # cost an extra full install and two deletions of node_modules per run # (each about 5 minutes for zpm's 13 GB on babylon). --conclude keeps the # lockfile, the packageManager pin and the PM config it wrote (the state a # CI job checks out), so every later prepare only undoes the timed command. BENCH_CLEAN_PRE="bash $BENCH_SCRIPTS/clean-helpers.sh clean_node_modules clean_lockfiles clean_package_manager_files clean_package_manager_field" BENCH_CLEAN_POST="bash $BENCH_SCRIPTS/clean-helpers.sh clean_node_modules clean_package_manager_files" # The marker that says a tool's lockfile and cache are in place. Lives in the # results folder, which common.sh empties at the start of every run. ci_setup_marker() { echo "$BENCH_OUTPUT_FOLDER/.ci-setup-ready-$1" } # Helper: build per-PM prepare. Its first run (no marker yet) cleans what the # previous tool left, installs the PM, runs the PM-specific setup and a # regular install that writes the lockfile and fills the cache. Every run, # that first one included, then cleans node_modules and uninstalls the PM # binary. A failed lockfile install leaves no marker, so the next prepare # tries again, as it did when every prepare ran the install. ci_setup_prepare() { local pm="$1" # tool name (marker) local pm_install="$2" # command to install the PM binary local setup_cmd="$3" # PM-specific setup (e.g. .yarnrc.yml) local install_cmd="$4" # regular install to generate lockfile local pm_uninstall="$5" # command to uninstall the PM binary local corepack_purge="$6" # optional: purge corepack cache local marker marker="$(ci_setup_marker "$pm")" # Once per tool: lockfile + cache local once="$BENCH_CLEAN_PRE" # Install PM binary if [ -n "$pm_install" ]; then once="$once; $pm_install" fi # PM-specific setup if [ -n "$setup_cmd" ]; then once="$once; $setup_cmd" fi # Generate lockfile via regular install once="$once; timeout $BENCH_TIMEOUT $install_cmd >/dev/null 2>&1 && touch $marker || true" local result="sleep 1; if [ ! -e $marker ]; then $once; fi" # Clean node_modules (ci starts fresh) result="$result; $BENCH_CLEAN_POST" # Uninstall PM binary so the measured command re-installs it if [ -n "$pm_uninstall" ]; then result="$result; $pm_uninstall" fi # Purge corepack cache if applicable if [ -n "$corepack_purge" ]; then result="$result; $corepack_purge" fi echo "$result" } # Regular (non-ci) install commands to generate lockfiles during prepare. # npm uses the dedicated prefix binary for lockfile generation too. _INSTALL_NPM="$NPM_SETUP_PREFIX/bin/npm install --no-audit --no-fund --ignore-scripts --silent" if [ "$BENCH_FIXTURE" = "large" ]; then _INSTALL_NPM="$_INSTALL_NPM --legacy-peer-deps" fi BENCH_PREPARE_NPM="$(ci_setup_prepare npm "$SETUP_INSTALL_NPM" "$BENCH_SETUP_NPM" "$_INSTALL_NPM" "$SETUP_UNINSTALL_NPM" "")" BENCH_PREPARE_YARN="$(ci_setup_prepare yarn "$SETUP_INSTALL_YARN" "$BENCH_SETUP_YARN" "$YARN_SETUP_PREFIX/bin/yarn install --ignore-scripts --silent" "$SETUP_UNINSTALL_YARN" "")" BENCH_PREPARE_BERRY="$(ci_setup_prepare berry "$SETUP_INSTALL_BERRY" "$BENCH_SETUP_BERRY" "$COREPACK_SETUP_PREFIX/bin/yarn install" "$SETUP_UNINSTALL_BERRY" "$COREPACK_PURGE")" BENCH_PREPARE_ZPM="$(ci_setup_prepare zpm "$SETUP_INSTALL_ZPM" "$BENCH_SETUP_ZPM" "$BENCH_ZPM_BIN install --silent" "$SETUP_UNINSTALL_ZPM" "")" BENCH_PREPARE_PNPM="$(ci_setup_prepare pnpm "$SETUP_INSTALL_PNPM" "$BENCH_SETUP_PNPM" "$PNPM_SETUP_PREFIX/bin/pnpm install --ignore-scripts --silent" "$SETUP_UNINSTALL_PNPM" "")" BENCH_PREPARE_VLT="$(ci_setup_prepare vlt "$SETUP_INSTALL_VLT" "$BENCH_SETUP_VLT" "vlt install --view=silent" "$SETUP_UNINSTALL_VLT" "")" BENCH_PREPARE_BUN="$(ci_setup_prepare bun "$SETUP_INSTALL_BUN" "$BENCH_SETUP_BUN" "bun install --ignore-scripts --silent" "$SETUP_UNINSTALL_BUN" "")" BENCH_PREPARE_DENO="$(ci_setup_prepare deno "$SETUP_INSTALL_DENO" "$BENCH_SETUP_DENO" "deno install --quiet" "$SETUP_UNINSTALL_DENO" "")" BENCH_PREPARE_AUBE="$(ci_setup_prepare aube "$SETUP_INSTALL_AUBE" "$BENCH_SETUP_AUBE" "aube install --ignore-scripts --silent" "$SETUP_UNINSTALL_AUBE" "")" BENCH_PREPARE_UPM="$(ci_setup_prepare upm "$SETUP_INSTALL_UPM" "$BENCH_SETUP_UPM" "upm install --silent" "$SETUP_UNINSTALL_UPM" "")" # Run the benchmark suite hyperfine --ignore-failure \ --time-unit=millisecond \ --export-json="$BENCH_OUTPUT_FOLDER/benchmarks.json" \ --warmup="$BENCH_WARMUP" \ --runs="$BENCH_RUNS" \ --conclude="sleep 1; bash $BENCH_SCRIPTS/package-count.sh $BENCH_OUTPUT_FOLDER; bash $BENCH_SCRIPTS/clean-helpers.sh clean_node_modules clean_package_manager_files clean_build_files" \ --cleanup="bash $BENCH_SCRIPTS/clean-helpers.sh clean_all" \ ${BENCH_INCLUDE_NPM:+--prepare="$BENCH_PREPARE_NPM"} \ ${BENCH_INCLUDE_NPM:+--command-name="npm" "$BENCH_COMMAND_NPM"} \ ${BENCH_INCLUDE_YARN:+--prepare="$BENCH_PREPARE_YARN"} \ ${BENCH_INCLUDE_YARN:+--command-name="yarn" "$BENCH_COMMAND_YARN"} \ ${BENCH_INCLUDE_BERRY:+--prepare="$BENCH_PREPARE_BERRY"} \ ${BENCH_INCLUDE_BERRY:+--command-name="berry" "$BENCH_COMMAND_BERRY"} \ ${BENCH_INCLUDE_ZPM:+--prepare="$BENCH_PREPARE_ZPM"} \ ${BENCH_INCLUDE_ZPM:+--command-name="zpm" "$BENCH_COMMAND_ZPM"} \ ${BENCH_INCLUDE_PNPM:+--prepare="$BENCH_PREPARE_PNPM"} \ ${BENCH_INCLUDE_PNPM:+--command-name="pnpm" "$BENCH_COMMAND_PNPM"} \ ${BENCH_INCLUDE_VLT:+--prepare="$BENCH_PREPARE_VLT"} \ ${BENCH_INCLUDE_VLT:+--command-name="vlt" "$BENCH_COMMAND_VLT"} \ ${BENCH_INCLUDE_BUN:+--prepare="$BENCH_PREPARE_BUN"} \ ${BENCH_INCLUDE_BUN:+--command-name="bun" "$BENCH_COMMAND_BUN"} \ ${BENCH_INCLUDE_DENO:+--prepare="$BENCH_PREPARE_DENO"} \ ${BENCH_INCLUDE_DENO:+--command-name="deno" "$BENCH_COMMAND_DENO"} \ ${BENCH_INCLUDE_AUBE:+--prepare="$BENCH_PREPARE_AUBE"} \ ${BENCH_INCLUDE_AUBE:+--command-name="aube" "$BENCH_COMMAND_AUBE"} \ ${BENCH_INCLUDE_UPM:+--prepare="$BENCH_PREPARE_UPM"} \ ${BENCH_INCLUDE_UPM:+--command-name="upm" "$BENCH_COMMAND_UPM"} collect_package_count # --- Process count collection --- # Each PM's prepare includes PM-specific uninstall/cache-purge steps and its # own lockfile, so each gets its own prepare. hyperfine's --cleanup removed # every lockfile and cache, so drop the markers: each counted install then # starts from what its first prepare makes, the state the timed runs saw. for pm in npm yarn berry zpm pnpm vlt bun deno aube upm; do P=$(echo "$pm" | tr '[:lower:]' '[:upper:]') prepare_var="BENCH_PREPARE_$P" printf -v "$prepare_var" '%s' "rm -f $(ci_setup_marker "$pm"); ${!prepare_var}" done collect_process_count_per_pm