name: Performance Tests on: pull_request: release: types: [published] push: branches: [trunk] workflow_dispatch: inputs: branches: description: 'branches or commits to compare (comma separated)' required: true wpversion: description: 'The base WP version to use for the tests (latest or 6.0, 6.1...)' required: false default: 'latest' # Cancels all previous workflow runs for pull requests that have not completed. concurrency: # The concurrency group contains the workflow name and the branch name for pull requests # or the commit hash for any other events. group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }} cancel-in-progress: true # Disable permissions for all available scopes by default. # Any needed permissions should be configured at the job level. permissions: {} jobs: # Analyzes the list of files modified in a pull request to detect any changes that could # potentially impact performance metrics. A series of ignore patterns are used in place of # an include list to ensure tests run even if the list of filters happens to become stale. detect-relevant-changes: name: Detect changes relevant to performance testing runs-on: 'ubuntu-slim' timeout-minutes: 5 if: ${{ github.repository == 'WordPress/gutenberg' }} permissions: pull-requests: read outputs: relevant: ${{ steps.filter.outputs.relevant }} steps: - name: Check for changes that may impact performance metrics id: filter if: ${{ github.event_name == 'pull_request' }} uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 with: predicate-quantifier: 'every' filters: | relevant: # Start from every file, then exclude what cannot change a performance result. - '**' # Documentation and schemas. - '!**/*.md' - '!{backport-changelog,docs,phpunit,schemas,storybook}/**' # Other test suites, which cannot affect the measurements. - '!test/{integration,php,storybook-playwright,unit}/**' - '!test/e2e/{*,!(assets)/**}' - '!{packages,tools}/**/{test,tests,stories}/**' # Development tooling and its configuration. - '!packages/{create-block,docgen,eslint-plugin,jest-*,npm-package-json-lint-config,prettier-config,stylelint-config}/**' # The mu-plugins and plugins below it load at run time, so they still count. - '!packages/e2e-tests/{*,!(mu-plugins|plugins)/**}' - '!tools/{docs,eslint,monorepo,project-management-automation,stylelint,validation}/**' - '!{eslint,lint-staged,prettier,syncpack}.config*' - '!{phpcs,phpunit}*' # Repository metadata. - '!{changelog,readme}.txt' - '!.{editorconfig,gitignore,prettierrc*,stylelintrc*}' # Other workflows, and everything under .github that this one does not use. - '!.github/{*,!(workflows|setup-node)/**,workflows/!(performance.yml)}' prepare: timeout-minutes: 5 name: Resolve branches to compare runs-on: ${{ vars.RUNNERS_NAME || 'ubuntu-24.04' }} needs: [detect-relevant-changes] permissions: contents: read actions: read if: ${{ github.repository == 'WordPress/gutenberg' && ( github.event_name != 'pull_request' || needs.detect-relevant-changes.outputs.relevant == 'true' ) }} outputs: # See tools/release/resolve-performance-branches.mjs for the shapes. # Falls back to the unresolved branches when the reuse lookup fails. branches: ${{ steps.reuse.outputs.branches || steps.branches.outputs.branches }} shards: ${{ steps.branches.outputs.shards }} wp-version: ${{ steps.branches.outputs.wp-version }} plugin-files: ${{ steps.branches.outputs.plugin-files }} build-key: ${{ steps.branches.outputs.build-key }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: show-progress: ${{ runner.debug == '1' && 'true' || 'false' }} persist-credentials: false - name: Use desired version of Node.js uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version-file: '.nvmrc' package-manager-cache: false - name: Resolve branches id: branches env: BASE_SHA: ${{ github.event.pull_request.base.sha }} BASE_REF: ${{ github.base_ref }} RELEASE_TAG: ${{ github.event.release.tag_name }} INPUT_BRANCHES: ${{ github.event.inputs.branches }} INPUT_WP_VERSION: ${{ github.event.inputs.wpversion }} run: node tools/release/resolve-performance-branches.mjs - name: Find reusable plugin builds id: reuse continue-on-error: true env: BRANCHES: ${{ steps.branches.outputs.branches }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | node tools/release/find-reusable-plugin-artifacts.mjs \ --branches "$BRANCHES" --repository "$GITHUB_REPOSITORY" build: timeout-minutes: 30 name: Build plugin (${{ matrix.branch.name }}) runs-on: ${{ vars.RUNNERS_NAME || 'ubuntu-24.04' }} needs: prepare permissions: contents: read actions: read strategy: fail-fast: false matrix: branch: ${{ fromJSON( needs.prepare.outputs.branches ) }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ matrix.branch.ref }} show-progress: ${{ runner.debug == '1' && 'true' || 'false' }} persist-credentials: false # Runs before the plugin is reused so the toolchain it would have been built # with is known, and so a rejected download still has everything to build. - name: Setup Node.js and install dependencies uses: ./.github/setup-node # Describes the plugin this job would produce. It travels with the artifact, and # a downloaded plugin is only reused when its own manifest is identical. - name: Describe this build env: MANIFEST: ${{ runner.temp }}/perf-build.json BUILD_KEY: ${{ needs.prepare.outputs.build-key }} BUILD_SHA: ${{ matrix.branch.sha }} run: | # The revision matters, not just the family: GitHub updates the hosted # image regularly. A runner that does not name both is unidentifiable, so # it keys on this attempt instead, which nothing else can match. image="run-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" if [[ -n "${ImageOS:-}" && -n "${ImageVersion:-}" ]]; then image="$ImageOS-$ImageVersion" fi jq -n --arg sha "$BUILD_SHA" --arg buildKey "$BUILD_KEY" \ --arg node "$(node -v)" --arg npm "$(npm -v)" --arg image "$image" \ '$ARGS.named' | tee "$MANIFEST" # Downloads into a staging directory: a partial download must not survive into # the plugin a fresh build packages. - name: Download the plugin an earlier run built id: download if: matrix.branch.reuseRunId continue-on-error: true uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: plugin-${{ matrix.branch.sha }} run-id: ${{ matrix.branch.reuseRunId }} repository: ${{ github.repository }} github-token: ${{ secrets.GITHUB_TOKEN }} path: ${{ runner.temp }}/download - name: Validate the downloaded plugin id: reused if: steps.download.outcome == 'success' env: DOWNLOAD_DIR: ${{ runner.temp }}/download PLUGIN_DIR: ${{ runner.temp }}/plugin MANIFEST: ${{ runner.temp }}/perf-build.json BUILD_SHA: ${{ matrix.branch.sha }} run: | # A missing or malformed manifest has to build, not fail the job. downloaded=$( jq -S . "$DOWNLOAD_DIR/perf-build.json" 2>/dev/null || echo '{}' ) if [[ -f "$DOWNLOAD_DIR/gutenberg.php" ]] && diff <( echo "$downloaded" ) <( jq -S . "$MANIFEST" ); then mv "$DOWNLOAD_DIR" "$PLUGIN_DIR" echo "ok=true" >> "$GITHUB_OUTPUT" else echo "::notice::Building $BUILD_SHA; the downloaded plugin does not match this run." fi - name: Build if: steps.reused.outputs.ok != 'true' run: npm run build -- --skip-types || npm run build # The list comes from tools/release/plugin-files.mts on the tested commit, so older refs # are packaged the same way. They may lack some of the files. - name: Package plugin if: steps.reused.outputs.ok != 'true' env: PLUGIN_DIR: ${{ runner.temp }}/plugin PLUGIN_FILES: ${{ needs.prepare.outputs.plugin-files }} MANIFEST: ${{ runner.temp }}/perf-build.json run: | mkdir -p "$PLUGIN_DIR" # post-content.php is gone from trunk but refs up to WP 6.x still require it from lib/demo.php. # shellcheck disable=SC2086 # The list holds globs that must expand. tar --ignore-failed-read -cf - $PLUGIN_FILES post-content.php \ | tar -xf - -C "$PLUGIN_DIR" test -f "$PLUGIN_DIR/gutenberg.php" cp "$MANIFEST" "$PLUGIN_DIR/perf-build.json" - name: Upload plugin uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: ${{ matrix.branch.artifact }} path: ${{ runner.temp }}/plugin compression-level: 1 # Long enough to serve the pull requests based on this commit. retention-days: 14 # Lets the trace artifacts be resolved with tools/build-scripts/packages/resolve-trace-source-maps.cjs # after extracting both next to each other. Only the first branch records traces. - name: Collect source maps if: matrix.branch.name == fromJSON( needs.prepare.outputs.branches )[0].name run: | mkdir -p "$RUNNER_TEMP/source-maps" if [[ -d build/scripts ]]; then find build/scripts -name '*.map' | tar -cf - -T - | tar -xf - -C "$RUNNER_TEMP/source-maps" fi - name: Upload source maps if: matrix.branch.name == fromJSON( needs.prepare.outputs.branches )[0].name uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: performance-source-maps path: ${{ runner.temp }}/source-maps if-no-files-found: ignore retention-days: 14 performance: timeout-minutes: 60 name: Run perf tests (${{ matrix.shard }}) runs-on: ${{ vars.RUNNERS_NAME || 'ubuntu-24.04' }} needs: [prepare, build] permissions: contents: read strategy: fail-fast: false # Shards run in parallel to cut the wall clock time. Every shard still # measures all branches on the same runner, so results stay comparable. matrix: include: ${{ fromJSON( needs.prepare.outputs.shards ) }} env: WP_ARTIFACTS_PATH: ${{ github.workspace }}/artifacts PERF_SUITES: ${{ matrix.suites }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: show-progress: ${{ runner.debug == '1' && 'true' || 'false' }} persist-credentials: false - name: Setup Node.js and install dependencies uses: ./.github/setup-node - name: Download prebuilt plugins uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: plugin-* path: ${{ runner.temp }}/artifacts # The command expects `/`. - name: Rename plugin directories run: | shopt -s nullglob dirs=("$RUNNER_TEMP"/artifacts/plugin-*) if [[ ${#dirs[@]} -eq 0 ]]; then echo "::error::No plugin artifacts were downloaded." exit 1 fi mkdir -p "$RUNNER_TEMP/plugins" for dir in "${dirs[@]}"; do mv "$dir" "$RUNNER_TEMP/plugins/$(basename "$dir" | sed 's/^plugin-//')" done ls -la "$RUNNER_TEMP/plugins" - name: Compare performance env: BRANCHES: ${{ needs.prepare.outputs.branches }} WP_VERSION: ${{ needs.prepare.outputs.wp-version }} run: | readarray -t BRANCH_ARGS < <(echo "$BRANCHES" | jq -r '.[].name') WP_VERSION_ARGS=() if [[ -n "$WP_VERSION" ]]; then WP_VERSION_ARGS=(--wp-version "$WP_VERSION") fi npm exec --no release-cli -- perf "${BRANCH_ARGS[@]}" --plugins-dir "$RUNNER_TEMP/plugins" --suites "$PERF_SUITES" "${WP_VERSION_ARGS[@]}" - name: Add workflow summary run: cat "${WP_ARTIFACTS_PATH}/summary.md" >> "$GITHUB_STEP_SUMMARY" - name: Archive performance summary if: success() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: performance-summary-${{ matrix.shard }} path: ${{ env.WP_ARTIFACTS_PATH }}/summary.md if-no-files-found: ignore - name: Archive performance results if: success() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: performance-results-${{ matrix.shard }} path: ${{ env.WP_ARTIFACTS_PATH }}/*.performance-results*.json - name: Archive performance traces if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: performance-traces-${{ matrix.shard }} path: ${{ env.WP_ARTIFACTS_PATH }}/traces if-no-files-found: ignore retention-days: 14 - name: Archive debug artifacts (screenshots, HTML snapshots) uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 if: failure() with: name: failures-artifacts-${{ matrix.shard }} path: ${{ env.WP_ARTIFACTS_PATH }} if-no-files-found: ignore retention-days: 14 results: timeout-minutes: 10 # Keeps the name of the job that existed before sharding, so it stays # the required status check. It fails whenever a shard fails. name: Run performance tests runs-on: ${{ vars.RUNNERS_NAME || 'ubuntu-24.04' }} needs: [prepare, performance] # Runs unless the whole workflow was skipped (forks), so a failure anywhere fails the check. if: ${{ !cancelled() && needs.prepare.result != 'skipped' }} permissions: contents: read env: WP_ARTIFACTS_PATH: ${{ github.workspace }}/artifacts steps: - name: Check job results if: needs.prepare.result != 'success' || needs.performance.result != 'success' run: | echo "::error::Resolving the branches, building a plugin or a performance tests shard failed." exit 1 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: show-progress: ${{ runner.debug == '1' && 'true' || 'false' }} persist-credentials: false - name: Use desired version of Node.js uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version-file: '.nvmrc' package-manager-cache: false - name: Download shard results uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: performance-results-* path: ${{ env.WP_ARTIFACTS_PATH }} merge-multiple: true # Keep the single artifact that existed before the tests were sharded. - name: Archive merged performance results uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: performance-results path: ${{ env.WP_ARTIFACTS_PATH }}/*.performance-results*.json - name: Download shard summaries id: download-summaries if: ${{ github.event_name == 'pull_request' }} continue-on-error: true uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: performance-summary-* path: ${{ env.WP_ARTIFACTS_PATH }}/summaries - name: Merge shard summaries if: ${{ steps.download-summaries.outcome == 'success' }} run: | mkdir -p pr-meta # Every shard renders the same heading and preamble above its # own suites, so keep them once and take only the tables from # each. One directory per shard, ordered by name. printf 'Client side metrics **exclude** the server response time.\n\n' > pr-meta/body.md find "${WP_ARTIFACTS_PATH}/summaries" -name summary.md -print0 \ | sort -z \ | xargs -0 sed -e '/^## Performance Test Results$/d' \ -e '/^Please note that client side metrics \*\*exclude\*\* the server response time\.$/d' \ >> pr-meta/body.md - name: Upload the performance report if: ${{ steps.download-summaries.outcome == 'success' }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: pr-meta--performance path: pr-meta if-no-files-found: ignore - name: Publish performance results if: github.event_name == 'push' env: CODEHEALTH_PROJECT_TOKEN: ${{ secrets.CODEHEALTH_PROJECT_TOKEN }} BRANCHES: ${{ needs.prepare.outputs.branches }} run: | COMMITTED_AT="$(git show -s "$GITHUB_SHA" --format="%cI")" REFERENCE_COMMIT="$(echo "$BRANCHES" | jq -r '.[1].name')" node tools/release/log-performance-results.js "$CODEHEALTH_PROJECT_TOKEN" trunk "$GITHUB_SHA" "$REFERENCE_COMMIT" "$COMMITTED_AT" pr-meta: name: Update PR meta comment needs: results # Runs even when the tests failed, so a stale report is cleared. # Skipped on a fork, whose token cannot write a comment. if: ${{ !cancelled() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} runs-on: 'ubuntu-slim' permissions: contents: read pull-requests: write timeout-minutes: 5 # Shared with every other writer of this comment. See bundle-size.yml. concurrency: group: pr-meta-${{ github.event.number }} cancel-in-progress: false queue: max steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: sparse-checkout: tools/pr-meta show-progress: ${{ runner.debug == '1' && 'true' || 'false' }} persist-credentials: false - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 continue-on-error: true with: name: pr-meta--performance path: pr-meta - uses: ./tools/pr-meta with: repo-token: ${{ secrets.GITHUB_TOKEN }} section: performance body-path: pr-meta/body.md commit-sha: ${{ github.event.pull_request.head.sha }} run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} # Runs on every event so it can serve as the workflow's only required status check while the # preceding jobs are path-filtered away. # # GitHub Actions can only be configured as required status checks on a per job basis using the # job's `name`. When the `name` changes PRs are temporarily blocked from being merged until # a privileged user can update the status checks, a process which is also error prone. # # The script reads every job in the run, so `needs` is only about waiting: it must cover # every job that would otherwise still be running when the script looks. `--ignore` then # names the jobs whose results do not count. Here that is this job itself, and the # housekeeping that runs alongside it. performance-status-check: name: Performance Tests - Status Check runs-on: 'ubuntu-slim' timeout-minutes: 5 permissions: contents: read actions: read needs: [detect-relevant-changes, results] if: ${{ ! cancelled() && github.repository == 'WordPress/gutenberg' }} steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: sparse-checkout: tools/monorepo/scripts show-progress: ${{ runner.debug == '1' && 'true' || 'false' }} persist-credentials: false - name: Use desired version of Node.js uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version-file: .nvmrc package-manager-cache: false - name: Check for any failures of required jobs env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: node tools/monorepo/scripts/ci-status-check.mts --ignore 'Performance Tests - Status Check,Update PR meta comment'