name: CodSpeed on: push: branches: - "main" pull_request: workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: true permissions: contents: read id-token: write env: CARGO_INCREMENTAL: "0" CARGO_NET_RETRY: 10 CARGO_TERM_COLOR: always # The bench profile inherits release, which uses thin LTO. Every bench # target is its own binary, so the LTO step runs once per benchmark and # made up most of the build time. CodSpeed compares each PR against main # built the same way, so the comparison does not need LTO. CARGO_PROFILE_BENCH_LTO: "false" # mimalloc returns unused memory to the OS on a 10 ms timer, so whether # that work lands inside a measurement depends on wall-clock time. # -1 turns the timer off. MIMALLOC_PURGE_DELAY: "-1" jobs: memory: name: Memory benchmarks uses: ./.github/workflows/codspeed-memory.yml permissions: contents: read id-token: write discover: name: Discover benchmark shards runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 10 outputs: benchmarks: ${{ steps.benchmarks.outputs.benchmarks }} shards: ${{ steps.benchmarks.outputs.shards }} steps: - uses: actions/checkout@v4 - name: Setup Rust toolchain uses: ./.github/actions/setup-rust-toolchain with: toolchain: "1.88.0" - name: Resolve benchmark shards id: benchmarks shell: bash run: | set -euo pipefail mapfile -t benchmarks < <(make codspeed-benchmarks-exclude-tpc-h) if [ "${#benchmarks[@]}" -eq 0 ]; then echo "No benchmarks found (excluding tpc_h_benchmark)." >&2 exit 1 fi printf 'Benchmark shards:\n' printf ' %s\n' "${benchmarks[@]}" json=$(printf '%s\n' "${benchmarks[@]}" | jq -R . | jq -cs .) echo "benchmarks=$json" >> "$GITHUB_OUTPUT" # The slowest benchmark targets run in several jobs. Each job gets # a filter that `cargo codspeed run` matches against every # benchmark's name, e.g. # benches/fts_benchmark.rs::fts_benches::bench_fts_cold_query::FTS Cold Query::cold_query[1000_rows] # The filters of one target split on a single character using # character sets that together cover every character, so each # benchmark, including ones added later, runs in exactly one job. # fts_benchmark: last letter of the benchmark function's name; # functions ending in "n" are split again on the character # before the final "]" (0-1 / anything else, incl. no "]"). # prepare_benchmark: the character before the final "]". splits='{ "fts_benchmark": [ "^[^:]*::[^:]*::[^:]*n::.*[01]\\]$", "^[^:]*::[^:]*::[^:]*n(::(.*[^01]\\]|.*[^\\]]|\\]?))?$", "^[^:]*::[^:]*::[^:]*e(::|$)", "^[^:]*::[^:]*::[^:]*[^ne:](::|$)" ], "prepare_benchmark": [ "[0-9]\\]$", "([^0-9]\\]|[^\\]])$" ] }' shards=$(jq -c --argjson splits "$splits" ' [.[] as $bench | if $splits[$bench] then $splits[$bench] | to_entries[] | {bench: $bench, name: "\($bench) (\(.key + 1)/\($splits[$bench] | length))", filter: .value} else {bench: $bench, name: $bench, filter: ""} end]' <<< "$json") printf 'Benchmark jobs:\n' jq -r '.[] | " \(.name) \(.filter)"' <<< "$shards" echo "shards=$shards" >> "$GITHUB_OUTPUT" build-stable: name: "Build benchmarks (stable)" runs-on: blacksmith-4vcpu-ubuntu-2404 needs: discover timeout-minutes: 60 # Allow this job to fail without failing the entire CI run continue-on-error: true outputs: built: ${{ steps.build_status.outputs.built }} env: TOOLCHAIN_LABEL: stable TOOLCHAIN_VERSION: "1.88.0" TOOLCHAIN_RUSTFLAGS: "" steps: - uses: actions/checkout@v4 - name: Setup Rust toolchain uses: ./.github/actions/setup-rust-toolchain with: toolchain: ${{ env.TOOLCHAIN_VERSION }} - name: Setup mold linker uses: rui314/setup-mold@v1 - uses: Swatinem/rust-cache@v2 with: prefix-key: "v1-rust-codspeed-${{ env.TOOLCHAIN_LABEL }}" cache-on-failure: true save-if: ${{ github.ref == 'refs/heads/main' }} - name: Install cargo-codspeed uses: taiki-e/install-action@v2 with: tool: cargo-codspeed - name: Generate graph-queries DBs run: | python3 perf/graph-queries/generate_seed.py | sqlite3 perf/graph-queries/graph-queries.db cp perf/graph-queries/graph-queries.db perf/graph-queries/graph-queries-analyzed.db sqlite3 perf/graph-queries/graph-queries-analyzed.db "ANALYZE;" - name: Build benchmarks (excluding TPC-H) id: build_benchmarks env: BENCHMARKS_JSON: ${{ needs.discover.outputs.benchmarks }} RUSTFLAGS: ${{ env.TOOLCHAIN_RUSTFLAGS }} shell: bash run: | set -euo pipefail bench_args=$(jq -r '.[] | "--bench " + .' <<< "$BENCHMARKS_JSON" | tr '\n' ' ') cargo +${{ env.TOOLCHAIN_VERSION }} codspeed build -m simulation $bench_args --features codspeed # Every bench binary contains its own copy of turso_core, so the # binaries are mostly the same bytes. A zip compresses each file # on its own (~2 GB); zstd with long-distance matching stores the # shared parts once. Every benchmark job downloads this archive. tar -cf - -C target codspeed | zstd -q -3 -T0 --long=31 -o codspeed-benchmarks.tar.zst ls -lh codspeed-benchmarks.tar.zst - name: Upload benchmark binaries id: upload_artifact if: steps.build_benchmarks.outcome == 'success' uses: actions/upload-artifact@v4 with: name: codspeed-benchmarks-${{ env.TOOLCHAIN_LABEL }} path: codspeed-benchmarks.tar.zst if-no-files-found: error retention-days: 1 # Already compressed. compression-level: 0 - name: Record build status id: build_status if: always() run: echo "built=${{ steps.build_benchmarks.outcome == 'success' && steps.upload_artifact.outcome == 'success' }}" >> "$GITHUB_OUTPUT" build-nightly: name: "Build benchmarks (nightly)" if: github.event_name != 'pull_request' runs-on: blacksmith-4vcpu-ubuntu-2404 needs: discover timeout-minutes: 60 # Allow this job to fail without failing the entire CI run continue-on-error: true outputs: built: ${{ steps.build_status.outputs.built }} env: TOOLCHAIN_LABEL: nightly TOOLCHAIN_VERSION: "nightly-2025-12-17" TOOLCHAIN_RUSTFLAGS: "--cfg nightly" steps: - uses: actions/checkout@v4 - name: Setup Rust toolchain uses: ./.github/actions/setup-rust-toolchain with: toolchain: ${{ env.TOOLCHAIN_VERSION }} - name: Setup mold linker uses: rui314/setup-mold@v1 - uses: Swatinem/rust-cache@v2 with: prefix-key: "v1-rust-codspeed-${{ env.TOOLCHAIN_LABEL }}" cache-on-failure: true save-if: ${{ github.ref == 'refs/heads/main' }} - name: Install cargo-codspeed uses: taiki-e/install-action@v2 with: tool: cargo-codspeed - name: Generate graph-queries DBs run: | python3 perf/graph-queries/generate_seed.py | sqlite3 perf/graph-queries/graph-queries.db cp perf/graph-queries/graph-queries.db perf/graph-queries/graph-queries-analyzed.db sqlite3 perf/graph-queries/graph-queries-analyzed.db "ANALYZE;" - name: Build benchmarks (excluding TPC-H) id: build_benchmarks env: BENCHMARKS_JSON: ${{ needs.discover.outputs.benchmarks }} RUSTFLAGS: ${{ env.TOOLCHAIN_RUSTFLAGS }} shell: bash run: | set -euo pipefail bench_args=$(jq -r '.[] | "--bench " + .' <<< "$BENCHMARKS_JSON" | tr '\n' ' ') cargo +${{ env.TOOLCHAIN_VERSION }} codspeed build -m simulation $bench_args --features codspeed # Every bench binary contains its own copy of turso_core, so the # binaries are mostly the same bytes. A zip compresses each file # on its own (~2 GB); zstd with long-distance matching stores the # shared parts once. Every benchmark job downloads this archive. tar -cf - -C target codspeed | zstd -q -3 -T0 --long=31 -o codspeed-benchmarks.tar.zst ls -lh codspeed-benchmarks.tar.zst - name: Upload benchmark binaries id: upload_artifact if: steps.build_benchmarks.outcome == 'success' uses: actions/upload-artifact@v4 with: name: codspeed-benchmarks-${{ env.TOOLCHAIN_LABEL }} path: codspeed-benchmarks.tar.zst if-no-files-found: error retention-days: 1 # Already compressed. compression-level: 0 - name: Record build status id: build_status if: always() run: echo "built=${{ steps.build_benchmarks.outcome == 'success' && steps.upload_artifact.outcome == 'success' }}" >> "$GITHUB_OUTPUT" benchmarks-stable: name: "Run benchmark (stable, ${{ matrix.shard.name }})" runs-on: blacksmith-4vcpu-ubuntu-2404 needs: - discover - build-stable if: needs.build-stable.outputs.built == 'true' timeout-minutes: 30 # Allow this job to fail without failing the entire CI run continue-on-error: true env: TOOLCHAIN_VERSION: "1.88.0" TOOLCHAIN_LABEL: stable TOOLCHAIN_RUSTFLAGS: "" strategy: fail-fast: false matrix: shard: ${{ fromJson(needs.discover.outputs.shards) }} steps: - uses: actions/checkout@v4 - name: Setup Rust toolchain uses: ./.github/actions/setup-rust-toolchain with: toolchain: ${{ env.TOOLCHAIN_VERSION }} - name: Install cargo-codspeed uses: taiki-e/install-action@v2 with: tool: cargo-codspeed - name: Download benchmark binaries uses: actions/download-artifact@v4 with: name: codspeed-benchmarks-${{ env.TOOLCHAIN_LABEL }} # tar keeps the executable bits, so no chmod is needed. - name: Unpack benchmark binaries run: | mkdir -p target zstd -q -d --long=31 -c codspeed-benchmarks.tar.zst | tar -xf - -C target rm codspeed-benchmarks.tar.zst - name: Generate graph-queries DBs if: matrix.shard.bench == 'graph_queries_benchmark' run: | python3 perf/graph-queries/generate_seed.py | sqlite3 perf/graph-queries/graph-queries.db cp perf/graph-queries/graph-queries.db perf/graph-queries/graph-queries-analyzed.db sqlite3 perf/graph-queries/graph-queries-analyzed.db "ANALYZE;" - name: Run benchmarks (excluding TPC-H) uses: CodSpeedHQ/action@v5.6.3 env: RUSTFLAGS: ${{ env.TOOLCHAIN_RUSTFLAGS }} with: mode: simulation # The CodSpeed runner runs this line with `bash -c`, so the quotes # keep the filter's regex characters away from the shell. run: cargo +${{ env.TOOLCHAIN_VERSION }} codspeed run ${{ matrix.shard.filter != '' && format('''{0}''', matrix.shard.filter) || '' }} --bench ${{ matrix.shard.bench }} benchmarks-nightly: name: "Run benchmark (nightly, ${{ matrix.shard.name }})" runs-on: blacksmith-4vcpu-ubuntu-2404 needs: - discover - build-nightly if: needs.build-nightly.outputs.built == 'true' timeout-minutes: 30 # Allow this job to fail without failing the entire CI run continue-on-error: true env: TOOLCHAIN_VERSION: "nightly-2025-12-17" TOOLCHAIN_LABEL: nightly TOOLCHAIN_RUSTFLAGS: "--cfg nightly" strategy: fail-fast: false matrix: shard: ${{ fromJson(needs.discover.outputs.shards) }} steps: - uses: actions/checkout@v4 - name: Setup Rust toolchain uses: ./.github/actions/setup-rust-toolchain with: toolchain: ${{ env.TOOLCHAIN_VERSION }} - name: Install cargo-codspeed uses: taiki-e/install-action@v2 with: tool: cargo-codspeed - name: Download benchmark binaries uses: actions/download-artifact@v4 with: name: codspeed-benchmarks-${{ env.TOOLCHAIN_LABEL }} # tar keeps the executable bits, so no chmod is needed. - name: Unpack benchmark binaries run: | mkdir -p target zstd -q -d --long=31 -c codspeed-benchmarks.tar.zst | tar -xf - -C target rm codspeed-benchmarks.tar.zst - name: Generate graph-queries DBs if: matrix.shard.bench == 'graph_queries_benchmark' run: | python3 perf/graph-queries/generate_seed.py | sqlite3 perf/graph-queries/graph-queries.db cp perf/graph-queries/graph-queries.db perf/graph-queries/graph-queries-analyzed.db sqlite3 perf/graph-queries/graph-queries-analyzed.db "ANALYZE;" - name: Run benchmarks (excluding TPC-H) uses: CodSpeedHQ/action@v5.6.3 env: RUSTFLAGS: ${{ env.TOOLCHAIN_RUSTFLAGS }} with: mode: simulation # The CodSpeed runner runs this line with `bash -c`, so the quotes # keep the filter's regex characters away from the shell. run: cargo +${{ env.TOOLCHAIN_VERSION }} codspeed run ${{ matrix.shard.filter != '' && format('''{0}''', matrix.shard.filter) || '' }} --bench ${{ matrix.shard.bench }} # tpc-h: # runs-on: ubuntu-latest # timeout-minutes: 60 # env: # DB_FILE: "perf/tpc-h/TPC-H.db" # steps: # - uses: actions/checkout@v4 # - name: Setup Rust toolchain # uses: dtolnay/rust-toolchain@stable # - name: Setup mold linker # uses: rui314/setup-mold@v1 # - uses: Swatinem/rust-cache@v2 # with: # prefix-key: "v1-rust-codspeed" # cache-on-failure: true # - name: Install cargo-codspeed # uses: taiki-e/install-action@v2 # with: # tool: cargo-codspeed # - name: Cache TPC-H # id: cache-tpch # uses: actions/cache@v4 # with: # path: ${{ env.DB_FILE }} # key: tpc-h # - name: Download TPC-H # if: steps.cache-tpch.outputs.cache-hit != 'true' # env: # DB_URL: "https://github.com/lovasoa/TPCH-sqlite/releases/download/v1.0/TPC-H.db" # run: wget -O $DB_FILE --no-verbose $DB_URL # - name: Build TPC-H benchmark # run: cargo codspeed build --features codspeed --bench tpc_h_benchmark # - name: Run TPC-H benchmark # uses: CodSpeedHQ/action@v5.6.3 # with: # mode: simulation # run: cargo codspeed run --bench tpc_h_benchmark