name: benchmark on: workflow_dispatch: inputs: duration: description: Seconds per scenario type: string default: "8" repeat: description: Rounds per engine type: string default: "2" queues: description: TUN queues type: string default: "4" settle: description: Idle seconds before the second memory reading type: string default: "5" publish: description: Commit the charts to docs/bench type: boolean default: true schedule: - cron: "0 4 * * 0" permissions: contents: write env: ZIG_VERSION: 0.16.0 jobs: compare: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v7 - uses: mlugg/setup-zig@v2 with: version: ${{ env.ZIG_VERSION }} - name: runner specification run: | mkdir -p bench-results { printf '| property | value |\n|---|---|\n' printf '| runner image | %s |\n' "${ImageOS:-unknown} ${ImageVersion:-}" printf '| cpu | %s |\n' "$(awk -F': ' '/model name/ {print $2; exit}' /proc/cpuinfo)" printf '| cpu cores | %s |\n' "$(nproc)" printf '| memory | %s |\n' "$(awk '/MemTotal/ {printf "%.1f GB", $2/1048576}' /proc/meminfo)" printf '| kernel | %s |\n' "$(uname -sr)" printf '| zig | %s |\n' "$(zig version)" printf '| date | %s |\n' "$(date -u +%Y-%m-%d)" } > bench-results/runner.md cat bench-results/runner.md - name: peer engines run: | sudo apt-get update sudo apt-get install -y iperf3 build-essential git python3 unzip git clone --depth 1 --recursive https://github.com/heiher/hev-socks5-tunnel "$RUNNER_TEMP/hev-socks5-tunnel" make -C "$RUNNER_TEMP/hev-socks5-tunnel" -j"$(nproc)" git clone --depth 1 --recursive https://github.com/heiher/hev-socks5-server "$RUNNER_TEMP/hev-socks5-server" make -C "$RUNNER_TEMP/hev-socks5-server" -j"$(nproc)" curl -sSL -o "$RUNNER_TEMP/t2s.zip" https://github.com/xjasonlyu/tun2socks/releases/download/v2.7.0/tun2socks-linux-amd64.zip unzip -o "$RUNNER_TEMP/t2s.zip" -d "$RUNNER_TEMP/tun2socks" chmod +x "$RUNNER_TEMP/tun2socks/tun2socks-linux-amd64" curl -sSL https://github.com/SagerNet/sing-box/releases/download/v1.14.0/sing-box-1.14.0-linux-amd64.tar.gz | tar xz -C "$RUNNER_TEMP" - name: allow unprivileged namespaces run: | sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true sudo modprobe tun || true - name: build run: zig build -Doptimize=ReleaseFast - name: throughput, latency and udp env: HEV: ${{ runner.temp }}/hev-socks5-tunnel/bin/hev-socks5-tunnel TUN2SOCKS: ${{ runner.temp }}/tun2socks/tun2socks-linux-amd64 SINGBOX: ${{ runner.temp }}/sing-box-1.14.0-linux-amd64/sing-box SOCKS_SERVER: ${{ runner.temp }}/hev-socks5-server/bin/hev-socks5-server IPERF3: /usr/bin/iperf3 ENGINES: zeptun-userspace hev zeptun-hybrid singbox-system tun2socks singbox-gvisor SCENARIOS: tcp-up-1 tcp-up-10 tcp-down-1 tcp-down-10 rr rr-8x1k crr udp-100k udp-gso-100k DURATION: ${{ inputs.duration || '8' }} REPEAT: ${{ inputs.repeat || '2' }} QUEUES: ${{ inputs.queues || '4' }} SETTLE: ${{ inputs.settle || '5' }} OUT_DIR: bench-results run: sh scripts/bench_netns.sh - name: footprint env: HEV: ${{ runner.temp }}/hev-socks5-tunnel/bin/hev-socks5-tunnel TUN2SOCKS: ${{ runner.temp }}/tun2socks/tun2socks-linux-amd64 SINGBOX: ${{ runner.temp }}/sing-box-1.14.0-linux-amd64/sing-box SOCKS_SERVER: ${{ runner.temp }}/hev-socks5-server/bin/hev-socks5-server ENGINES: zeptun hev singbox-system singbox-gvisor tun2socks TESTS: tcp:1000 udp:1000 OUT_DIR: footprint-results run: sh scripts/footprint_netns.sh - name: charts run: | python3 scripts/bench_charts.py bench-results docs/bench { printf '# Benchmarks\n\n' printf 'Every number on this page comes from the `benchmark` workflow in this repository, run on a GitHub-hosted runner.\n\n' printf '## Machine\n\n' cat bench-results/runner.md printf '\n## Method\n\n' printf 'Two network namespaces joined by a veth pair. Traffic enters the tunnel device (MTU 8500), leaves through the same SOCKS5 server (hev-socks5-server) for every engine, and reaches the servers in the second namespace. Engines are interleaved: each round starts every engine once and runs every scenario against it, so background noise spreads evenly. CPU comes from `/proc//stat` and memory from `VmRSS`, summed over all processes of an engine.\n\n' printf 'Duration %s s per scenario, %s rounds, %s queues. Memory is read twice: the highest sample while the scenario runs, and again after %s s of idle, which shows whether an engine gives the memory back.\n\n' "${DURATION}" "${REPEAT}" "${QUEUES}" "${SETTLE}" printf '## Throughput\n\n![throughput](throughput.svg)\n\n' printf '## CPU\n\n![cpu](cpu.svg)\n\n' printf '## Request/response\n\n![transactions](transactions.svg)\n\n![latency](latency.svg)\n\n' printf '## UDP\n\n![udp](udp.svg)\n\n' printf '## Memory\n\n![memory](memory.svg)\n\n' printf '## Raw results\n\n' cat bench-results/summary.md printf '\n' cat footprint-results/footprint.txt 2> /dev/null || true } > docs/bench/README.md python3 scripts/wiki_benchmarks.py docs/bench/README.md docs/wiki/Benchmarks.md env: DURATION: ${{ inputs.duration || '8' }} REPEAT: ${{ inputs.repeat || '2' }} QUEUES: ${{ inputs.queues || '4' }} SETTLE: ${{ inputs.settle || '5' }} - name: summary if: always() run: | cat bench-results/runner.md >> "$GITHUB_STEP_SUMMARY" || true cat bench-results/summary.md >> "$GITHUB_STEP_SUMMARY" || true cat footprint-results/footprint.txt >> "$GITHUB_STEP_SUMMARY" || true - uses: actions/upload-artifact@v7 if: always() with: name: benchmark-results path: | bench-results footprint-results docs/bench - name: publish charts if: ${{ github.event_name == 'schedule' || inputs.publish }} run: | git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git add docs/bench docs/wiki/Benchmarks.md git diff --cached --quiet && exit 0 git commit -m "benchmark results from the runner" for attempt in 1 2 3; do git pull --rebase origin main && git push origin HEAD:main && exit 0 sleep 5 done exit 1