name: Benchmark a PR on: issue_comment: types: [created] env: PROJECT_ID: vaxine REGISTRY: europe-docker.pkg.dev/vaxine/electric jobs: build: permissions: pull-requests: write contents: read name: Build image and run benchmarks on PR comment if: ${{ github.event.issue.pull_request && (github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'OWNER') && (startsWith(github.event.comment.body, 'benchmark this') || startsWith(github.event.comment.body, 'Benchmark this'))}} runs-on: ubuntu-latest steps: - uses: actions/github-script@v7 id: get-pr with: script: | const request = { owner: context.repo.owner, repo: context.repo.repo, pull_number: context.issue.number } core.info(`Getting PR #${request.pull_number} from ${request.owner}/${request.repo}`) const result = await github.rest.pulls.get(request) core.info(`GOT ${JSON.stringify(result)}`) return result.data - name: Ensure PR branch is trusted run: | if [ '${{ fromJSON(steps.get-pr.outputs.result).head.repo.full_name }}' != '${{ github.repository }}' ]; then echo 'Refusing to benchmark a fork PR with privileged credentials.' >&2 exit 1 fi - uses: docker/setup-buildx-action@v3 - uses: actions/checkout@v4 with: ref: ${{ fromJSON(steps.get-pr.outputs.result).head.sha }} - uses: google-github-actions/auth@v1 with: credentials_json: ${{ secrets.GCLOUD_REGISTRY_JSON_KEY }} - uses: google-github-actions/setup-gcloud@v1 - name: Use gcloud CLI run: gcloud info - name: Docker auth run: gcloud auth configure-docker europe-docker.pkg.dev --quiet - name: Set outputs run: echo "SHORT_SHA=$(git rev-parse --short=5 HEAD)" >> $GITHUB_ENV - name: Build builder image uses: docker/build-push-action@v6 with: context: packages/sync-service target: builder push: true tags: | ${{ env.REGISTRY }}/electric:pr-${{ github.event.issue.number }}-builder cache-from: | type=registry,ref=${{ env.REGISTRY }}/electric:canary-builder type=registry,ref=${{ env.REGISTRY }}/electric:pr-${{ github.event.issue.number }}-builder cache-to: | type=inline build-contexts: | electric-telemetry=${{ github.workspace }}/packages/electric-telemetry - name: Build runner base image uses: docker/build-push-action@v6 with: context: packages/sync-service target: runner_setup push: true tags: | ${{ env.REGISTRY }}/electric:pr-${{ github.event.issue.number }}-runner-base cache-from: | type=registry,ref=${{ env.REGISTRY }}/electric:canary-runner-base type=registry,ref=${{ env.REGISTRY }}/electric:pr-${{ github.event.issue.number }}-runner-base cache-to: | type=inline - name: Build final image uses: docker/build-push-action@v6 with: context: packages/sync-service push: true tags: | ${{ env.REGISTRY }}/electric:pr-${{ github.event.issue.number }}-${{ env.SHORT_SHA }} cache-from: | type=registry,ref=${{ env.REGISTRY }}/electric:pr-${{ github.event.issue.number }}-builder type=registry,ref=${{ env.REGISTRY }}/electric:pr-${{ github.event.issue.number }}-runner-base cache-to: | type=inline build-contexts: | electric-telemetry=${{ github.workspace }}/packages/electric-telemetry - name: Write fanout benchmark run: | curl -X POST 'https://benchmarking.electric-sql.com/api/benchmarks/write_fanout/runs' \ -u benchmarking:${{ secrets.BENCHMARKING_API_PASSWORD }} \ -H 'Content-Type: application/json' \ --fail-with-body \ -d '{ "benchmark_run": { "spec_values": { "electric_image": ["${{ env.REGISTRY }}/electric:pr-${{ github.event.issue.number }}-${{ env.SHORT_SHA }}"], "postgres_image": ["postgres:16-alpine"], "row_count": [500], "concurrent": [5, 105, 205, 305, 405, 505, 605, 705, 805, 905, 1005], "tx_row_count": [50] }, "machine_request": { "vcpu": 4, "mem_gb": 8 }, "metadata": { "pr": ${{ github.event.issue.number }}, "short_version": "pr-${{ github.event.issue.number }}-${{ env.SHORT_SHA }}", "callback": { "method": "POST", "headers": [ ["Accept","application/vnd.github+json"], ["Authorization","Bearer ${{ secrets.CROSSREPO_PAT }}"], ["X-GitHub-Api-Version","2022-11-28"] ], "url":"https://api.github.com/repos/electric-sql/electric/actions/workflows/leave_benchmark_comment.yml/dispatches", "body": "{\"ref\":\"main\",\"inputs\":{\"pr\":\"${{ github.event.issue.number }}\",\"benchmark_info\":#{benchmark_info},\"original_commit\":\"${{ env.SHORT_SHA }}\"}}" } } } }' - name: unrelated_shapes_one_client_latency benchmark run: | curl -X POST 'https://benchmarking.electric-sql.com/api/benchmarks/unrelated_shapes_one_client_latency/runs' \ -u benchmarking:${{ secrets.BENCHMARKING_API_PASSWORD }} \ -H 'Content-Type: application/json' \ --fail-with-body \ -d '{ "benchmark_run": { "spec_values": { "electric_image": ["${{ env.REGISTRY }}/electric:pr-${{ github.event.issue.number }}-${{ env.SHORT_SHA }}"], "postgres_image": ["postgres:16-alpine"], "row_count": [500], "shape_count": [100,300,500,700,900,1100,1300,1500,1700,1900,2100,2300,2500,2700,2900,3100], "tx_row_count": [50], "where_clause": ["name = '"'#{name}'"'"] }, "machine_request": { "vcpu": 4, "mem_gb": 8 }, "metadata": { "pr": ${{ github.event.issue.number }}, "short_version": "pr-${{ github.event.issue.number }}-${{ env.SHORT_SHA }}", "callback": { "method": "POST", "headers": [ ["Accept","application/vnd.github+json"], ["Authorization","Bearer ${{ secrets.CROSSREPO_PAT }}"], ["X-GitHub-Api-Version","2022-11-28"] ], "url":"https://api.github.com/repos/electric-sql/electric/actions/workflows/leave_benchmark_comment.yml/dispatches", "body": "{\"ref\":\"main\",\"inputs\":{\"pr\":\"${{ github.event.issue.number }}\",\"benchmark_info\":#{benchmark_info},\"original_commit\":\"${{ env.SHORT_SHA }}\"}}" } } } }' - name: many_shapes_one_client_latency benchmark run: | curl -X POST 'https://benchmarking.electric-sql.com/api/benchmarks/many_shapes_one_client_latency/runs' \ -u benchmarking:${{ secrets.BENCHMARKING_API_PASSWORD }} \ -H 'Content-Type: application/json' \ --fail-with-body \ -d '{ "benchmark_run": { "spec_values": { "electric_image": ["${{ env.REGISTRY }}/electric:pr-${{ github.event.issue.number }}-${{ env.SHORT_SHA }}"], "postgres_image": ["postgres:16-alpine"], "row_count": [500], "shape_count": [100,300,500,700,900,1100,1300,1500,1700,1900,2100,2300,2500,2700,2900,3100], "tx_row_count": [50] }, "machine_request": { "vcpu": 4, "mem_gb": 8 }, "metadata": { "pr": ${{ github.event.issue.number }}, "short_version": "pr-${{ github.event.issue.number }}-${{ env.SHORT_SHA }}", "callback": { "method": "POST", "headers": [ ["Accept","application/vnd.github+json"], ["Authorization","Bearer ${{ secrets.CROSSREPO_PAT }}"], ["X-GitHub-Api-Version","2022-11-28"] ], "url":"https://api.github.com/repos/electric-sql/electric/actions/workflows/leave_benchmark_comment.yml/dispatches", "body": "{\"ref\":\"main\",\"inputs\":{\"pr\":\"${{ github.event.issue.number }}\",\"benchmark_info\":#{benchmark_info},\"original_commit\":\"${{ env.SHORT_SHA }}\"}}" } } } }' - name: concurrent_shape_creation benchmark run: | curl -X POST 'https://benchmarking.electric-sql.com/api/benchmarks/concurrent_shape_creation/runs' \ -u benchmarking:${{ secrets.BENCHMARKING_API_PASSWORD }} \ -H 'Content-Type: application/json' \ --fail-with-body \ -d '{ "benchmark_run": { "spec_values": { "electric_image": ["${{ env.REGISTRY }}/electric:pr-${{ github.event.issue.number }}-${{ env.SHORT_SHA }}"], "postgres_image": ["postgres:16-alpine"], "row_count": [500], "concurrent": [50, 450, 850, 1250, 1650, 2050, 2450, 2850, 3250, 3650] }, "machine_request": { "vcpu": 4, "mem_gb": 8 }, "metadata": { "pr": ${{ github.event.issue.number }}, "short_version": "pr-${{ github.event.issue.number }}-${{ env.SHORT_SHA }}", "callback": { "method": "POST", "headers": [ ["Accept","application/vnd.github+json"], ["Authorization","Bearer ${{ secrets.CROSSREPO_PAT }}"], ["X-GitHub-Api-Version","2022-11-28"] ], "url":"https://api.github.com/repos/electric-sql/electric/actions/workflows/leave_benchmark_comment.yml/dispatches", "body": "{\"ref\":\"main\",\"inputs\":{\"pr\":\"${{ github.event.issue.number }}\",\"benchmark_info\":#{benchmark_info},\"original_commit\":\"${{ env.SHORT_SHA }}\"}}" } } } }' - name: diverse_shape_fanout benchmark run: | curl -X POST 'https://benchmarking.electric-sql.com/api/benchmarks/diverse_shape_fanout/runs' \ -u benchmarking:${{ secrets.BENCHMARKING_API_PASSWORD }} \ -H 'Content-Type: application/json' \ --fail-with-body \ -d '{ "benchmark_run": { "spec_values": { "electric_image": ["${{ env.REGISTRY }}/electric:pr-${{ github.event.issue.number }}-${{ env.SHORT_SHA }}"], "postgres_image": ["postgres:16-alpine"], "row_count": [500], "concurrent": [50, 450, 850, 1250, 1650, 2050, 2450, 2850, 3250, 3650], "tx_row_count": [50] }, "machine_request": { "vcpu": 4, "mem_gb": 8 }, "metadata": { "pr": ${{ github.event.issue.number }}, "short_version": "pr-${{ github.event.issue.number }}-${{ env.SHORT_SHA }}", "callback": { "method": "POST", "headers": [ ["Accept","application/vnd.github+json"], ["Authorization","Bearer ${{ secrets.CROSSREPO_PAT }}"], ["X-GitHub-Api-Version","2022-11-28"] ], "url":"https://api.github.com/repos/electric-sql/electric/actions/workflows/leave_benchmark_comment.yml/dispatches", "body": "{\"ref\":\"main\",\"inputs\":{\"pr\":\"${{ github.event.issue.number }}\",\"benchmark_info\":#{benchmark_info},\"original_commit\":\"${{ env.SHORT_SHA }}\"}}" } } } }'