# Description: End-to-end testing with Playwright across multiple browsers, deploys test reports to Cloudflare Pages name: 'CI: Tests E2E' on: push: branches: [main, master, core/*, desktop/*] pull_request: branches-ignore: [wip/*, draft/*, temp/*] merge_group: workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: lint-pr: if: github.event_name == 'pull_request' uses: ./.github/workflows/ci-lint-format.yaml permissions: contents: write pull-requests: write secrets: PR_GH_TOKEN: ${{ secrets.PR_GH_TOKEN }} lint-queue: if: github.event_name == 'merge_group' uses: ./.github/workflows/ci-lint-format-queue.yaml permissions: contents: read fallow: if: github.event_name == 'pull_request' || github.event_name == 'merge_group' uses: ./.github/workflows/ci-fallow.yaml permissions: contents: read pull-requests: write lint-and-format: needs: [lint-pr, lint-queue] if: ${{ always() && (github.event_name == 'pull_request' || github.event_name == 'merge_group') }} runs-on: ubuntu-latest permissions: {} steps: - name: Check candidate lint result env: RESULT: ${{ github.event_name == 'pull_request' && needs.lint-pr.result || needs.lint-queue.result }} run: test "$RESULT" = success changes: runs-on: ubuntu-latest permissions: contents: read outputs: should-run: ${{ steps.changes.outputs.should-run-e2e }} should-build: ${{ steps.changes.outputs.should-run-e2e == 'true' || ((github.event_name == 'pull_request' || github.event_name == 'merge_group') && steps.changes.outputs.should-run-unit == 'true') }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - id: changes uses: ./.github/actions/changes-filter setup: needs: changes if: ${{ needs.changes.outputs.should-build == 'true' }} runs-on: ubuntu-latest env: EXPECTED_FRONTEND_COMMIT: ${{ github.sha }} steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup frontend uses: ./.github/actions/setup-frontend with: include_build_step: true env: COLLECT_COVERAGE: 'true' - name: Verify localhost Assets default run: pnpm test:assets-flag-artifact env: EXPECTED_DISTRIBUTION: localhost # Upload only verified dist/ (containerized test jobs will pnpm install without cache) - name: Upload built frontend uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: frontend-dist path: dist/ retention-days: 1 - name: Build and verify desktop Assets default run: | pnpm build:desktop-dist pnpm test:assets-flag-artifact env: EXPECTED_DISTRIBUTION: desktop VITE_USE_LEGACY_DEFAULT_GRAPH: 'true' - name: Upload desktop frontend uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: frontend-dist-desktop path: dist/ retention-days: 1 - name: Build cloud frontend uses: ./.github/actions/build-cloud-frontend with: expected_commit: ${{ github.sha }} preflight: needs: [changes, setup, lint-and-format, fallow] if: ${{ !cancelled() }} runs-on: ubuntu-latest permissions: {} steps: - name: Require successful prerequisites env: EVENT: ${{ github.event_name }} CHANGES: ${{ needs.changes.result }} SHOULD_BUILD: ${{ needs.changes.outputs.should-build }} BUILD: ${{ needs.setup.result }} LINT: ${{ needs.lint-and-format.result }} FALLOW: ${{ needs.fallow.result }} run: | test "$CHANGES" = success case "$SHOULD_BUILD:$BUILD" in true:success | false:skipped) ;; *) echo "::error::Build prerequisite did not pass: $SHOULD_BUILD:$BUILD"; exit 1 ;; esac case "$EVENT" in pull_request | merge_group) test "$LINT" = success && test "$FALLOW" = success ;; esac unit: needs: preflight if: ${{ !cancelled() && needs.preflight.result == 'success' && (github.event_name == 'pull_request' || github.event_name == 'merge_group') }} uses: ./.github/workflows/ci-tests-unit.yaml permissions: contents: read secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} test: needs: [preflight, unit] if: ${{ always() && (github.event_name == 'pull_request' || github.event_name == 'merge_group') }} runs-on: ubuntu-latest permissions: {} steps: - name: Check unit result env: PREFLIGHT: ${{ needs.preflight.result }} UNIT: ${{ needs.unit.result }} run: test "$PREFLIGHT" = success && test "$UNIT" = success ecosystem: needs: preflight if: ${{ !cancelled() && needs.preflight.result == 'success' && (github.event_name == 'pull_request' || github.event_name == 'merge_group') }} uses: ./.github/workflows/ci-ecosystem-matrix.yaml permissions: contents: read pull-requests: write actions: write # Sharded chromium tests playwright-tests-chromium-sharded: needs: [changes, setup, preflight] if: ${{ !cancelled() && needs.preflight.result == 'success' && needs.changes.outputs.should-run == 'true' }} runs-on: ubuntu-latest timeout-minutes: 60 container: image: ghcr.io/comfy-org/comfyui-ci-container:0.0.28 credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} permissions: contents: read packages: read strategy: fail-fast: false matrix: shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] shardTotal: [16] steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Download built frontend uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: frontend-dist path: dist/ - name: Start ComfyUI server uses: ./.github/actions/start-comfyui-server - name: Setup frontend uses: ./.github/actions/setup-frontend with: node_cache: 'false' - name: Run Playwright tests (Shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}) id: playwright run: pnpm exec playwright test --project=chromium --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --add-reporter=./tools/snapshot-updates/src/snapshotUpdatesReporter.ts env: PLAYWRIGHT_BLOB_OUTPUT_DIR: ./blob-report COLLECT_COVERAGE: 'true' - name: Upload blob report uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 if: ${{ !cancelled() }} with: name: blob-report-chromium-${{ matrix.shardIndex }} path: blob-report/ retention-days: 1 - name: Upload snapshot updates uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 if: ${{ !cancelled() && github.event_name == 'pull_request' }} continue-on-error: true with: name: snapshot-updates-chromium-${{ matrix.shardIndex }}-${{ job.check_run_id }} path: snapshot-updates/ retention-days: 7 if-no-files-found: ignore overwrite: true - name: Upload shard coverage data if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: e2e-coverage-shard-${{ matrix.shardIndex }} path: coverage/playwright/ retention-days: 1 # A shard that produced no coverage fails here rather than going # missing from the merge, so the matrix result is the only # completeness signal the packager needs. if-no-files-found: error # Sharded cloud tests: the cloud project alone ran longer than any chromium # shard, so it was the critical path of every merge-group entry. playwright-tests-cloud-sharded: needs: [changes, setup, preflight] if: ${{ !cancelled() && needs.preflight.result == 'success' && needs.changes.outputs.should-run == 'true' }} runs-on: ubuntu-latest timeout-minutes: 30 container: image: ghcr.io/comfy-org/comfyui-ci-container:0.0.28 credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} permissions: contents: read packages: read strategy: fail-fast: false matrix: shardIndex: [1, 2, 3, 4] shardTotal: [4] steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Download built frontend uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: frontend-dist-cloud path: dist/ - name: Start ComfyUI server uses: ./.github/actions/start-comfyui-server - name: Setup frontend uses: ./.github/actions/setup-frontend with: node_cache: 'false' - name: Run Playwright tests (cloud, Shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}) id: playwright run: pnpm exec playwright test --project=cloud --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --add-reporter=./tools/snapshot-updates/src/snapshotUpdatesReporter.ts env: PLAYWRIGHT_BLOB_OUTPUT_DIR: ./blob-report - name: Upload blob report uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 if: ${{ !cancelled() }} with: name: blob-report-cloud-${{ matrix.shardIndex }} path: blob-report/ retention-days: 1 - name: Upload snapshot updates uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 if: ${{ !cancelled() && github.event_name == 'pull_request' }} continue-on-error: true with: name: snapshot-updates-cloud-${{ matrix.shardIndex }}-${{ job.check_run_id }} path: snapshot-updates/ retention-days: 7 if-no-files-found: ignore overwrite: true playwright-tests: # Ideally, each shard runs test in 6 minutes, but allow up to 30 minutes timeout-minutes: 30 needs: [changes, setup, preflight] if: ${{ !cancelled() && needs.preflight.result == 'success' && needs.changes.outputs.should-run == 'true' }} runs-on: ubuntu-latest container: image: ghcr.io/comfy-org/comfyui-ci-container:0.0.28 credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} permissions: contents: read packages: read strategy: fail-fast: false matrix: include: - browser: chromium-2x artifact: frontend-dist - browser: chromium-0.5x artifact: frontend-dist - browser: mobile-chrome artifact: frontend-dist - browser: mobile-safari artifact: frontend-dist-cloud - browser: desktop artifact: frontend-dist-desktop steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Download built frontend uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ matrix.artifact }} path: dist/ - name: Start ComfyUI server uses: ./.github/actions/start-comfyui-server - name: Setup frontend uses: ./.github/actions/setup-frontend with: node_cache: 'false' - name: Run Playwright tests (${{ matrix.browser }}) id: playwright run: pnpm exec playwright test --project=${{ matrix.browser }} --add-reporter=./tools/snapshot-updates/src/snapshotUpdatesReporter.ts env: PLAYWRIGHT_BLOB_OUTPUT_DIR: ./blob-report - name: Upload snapshot updates uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 if: ${{ !cancelled() && github.event_name == 'pull_request' }} continue-on-error: true with: name: snapshot-updates-${{ matrix.browser }}-${{ job.check_run_id }} path: snapshot-updates/ retention-days: 7 if-no-files-found: ignore overwrite: true - name: Generate HTML and JSON reports if: always() run: | # Generate HTML report from blob pnpm exec playwright merge-reports --reporter=html ./blob-report # Generate JSON report separately with explicit output path PLAYWRIGHT_JSON_OUTPUT_NAME=playwright-report/report.json \ pnpm exec playwright merge-reports --reporter=json ./blob-report - name: Upload Playwright report uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 if: always() with: name: playwright-report-${{ matrix.browser }} path: ./playwright-report/ retention-days: 30 # Merge sharded test reports (no container needed - only runs CLI) merge-reports: needs: [ changes, preflight, playwright-tests-chromium-sharded, playwright-tests-cloud-sharded ] runs-on: ubuntu-latest if: ${{ !cancelled() && needs.preflight.result == 'success' && needs.changes.outputs.should-run == 'true' }} strategy: fail-fast: false matrix: project: [chromium, cloud] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: sparse-checkout: /package.json sparse-checkout-cone-mode: false persist-credentials: false - name: Install pnpm uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - name: Download blob reports uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: ./all-blob-reports pattern: blob-report-${{ matrix.project }}-* merge-multiple: true - name: Merge into HTML Report run: | # Generate HTML report pnpm dlx @playwright/test merge-reports --reporter=html ./all-blob-reports # Generate JSON report separately with explicit output path PLAYWRIGHT_JSON_OUTPUT_NAME=playwright-report/report.json \ pnpm dlx @playwright/test merge-reports --reporter=json ./all-blob-reports - name: Upload HTML report uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: playwright-report-${{ matrix.project }} path: ./playwright-report/ retention-days: 30 # Gate job — single required check that passes whether the matrix ran or was # skipped. Branch rulesets require this instead of the individual matrix- # expanded check names so PRs with no e2e-relevant changes aren't stuck. e2e-status: if: ${{ always() }} needs: [ changes, preflight, playwright-tests-chromium-sharded, playwright-tests-cloud-sharded, playwright-tests ] runs-on: ubuntu-latest steps: - name: Check E2E results env: PREFLIGHT: ${{ needs.preflight.result }} CHANGES: ${{ needs.changes.result }} SHOULD_RUN: ${{ needs.changes.outputs.should-run }} SHARDED: ${{ needs.playwright-tests-chromium-sharded.result }} CLOUD: ${{ needs.playwright-tests-cloud-sharded.result }} BROWSERS: ${{ needs.playwright-tests.result }} run: | echo "changes=$CHANGES should_run=$SHOULD_RUN sharded=$SHARDED cloud=$CLOUD browsers=$BROWSERS" [[ "$PREFLIGHT" != "success" ]] && echo "CI prerequisites did not pass ($PREFLIGHT)" && exit 1 [[ "$CHANGES" != "success" ]] && echo "changes gate did not complete ($CHANGES)" && exit 1 [[ "$SHOULD_RUN" != "true" ]] && echo "E2E skipped" && exit 0 [[ "$SHARDED" == "failure" || "$CLOUD" == "failure" || "$BROWSERS" == "failure" ]] && echo "E2E failed" && exit 1 [[ "$SHARDED" == "cancelled" || "$CLOUD" == "cancelled" || "$BROWSERS" == "cancelled" ]] && echo "E2E cancelled, not failed" && exit 1 [[ "$SHARDED" != "success" || "$CLOUD" != "success" || "$BROWSERS" != "success" ]] && echo "E2E failed" && exit 1 echo "E2E passed" upload-e2e-coverage: needs: [changes, preflight, playwright-tests-chromium-sharded] if: >- ${{ always() && needs.preflight.result == 'success' && needs.changes.outputs.should-run == 'true' && needs.playwright-tests-chromium-sharded.result != 'cancelled' }} uses: ./.github/workflows/ci-tests-e2e-coverage-package.yaml with: source_sha: ${{ github.sha }} # A shard that died partway still uploads a tracefile, and one that # produced none fails its own upload, so the matrix verdict is what # makes the merge trustworthy for trend reporting. shards_succeeded: ${{ needs.playwright-tests-chromium-sharded.result == 'success' }} #### BEGIN Deployment and commenting (PRs with direct secret access) # when using pull_request event, we have permission to comment directly # otherwise, we use workflow_run in ci-tests-e2e-forks.yaml # Post starting section into the unified PR report comment comment-on-pr-start: needs: [changes, preflight] runs-on: ubuntu-latest concurrency: group: pr-comment-${{ github.event.pull_request.number }} cancel-in-progress: false if: >- ${{ !cancelled() && needs.preflight.result == 'success' && needs.changes.outputs.should-run == 'true' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false && github.event.pull_request.user.login != 'dependabot[bot]' }} permissions: pull-requests: write steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Upsert playwright starting section into unified report uses: ./.github/actions/upsert-comment-section with: pr-number: ${{ github.event.pull_request.number }} section-name: playwright section-content: '## 🎭 Playwright: ⏳ Running...' token: ${{ github.token }} # Deploy and upsert final playwright section deploy-and-comment: needs: [changes, preflight, playwright-tests, merge-reports] runs-on: ubuntu-latest concurrency: group: pr-comment-${{ github.event.pull_request.number }} cancel-in-progress: false if: >- ${{ always() && needs.preflight.result == 'success' && needs.changes.outputs.should-run == 'true' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false && github.event.pull_request.user.login != 'dependabot[bot]' }} permissions: pull-requests: write contents: read steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Download all playwright reports uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: playwright-report-* path: reports - name: Download snapshot update manifests uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 continue-on-error: true with: pattern: snapshot-updates-* path: snapshot-updates - name: Deploy reports and generate section env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} GITHUB_TOKEN: ${{ github.token }} GITHUB_SHA: ${{ github.event.pull_request.head.sha }} WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} SUMMARY_FILE: playwright-section.md BRANCH_NAME: ${{ github.head_ref }} run: | bash ./scripts/cicd/pr-playwright-deploy-and-comment.sh \ "${{ github.event.pull_request.number }}" \ "$BRANCH_NAME" \ "completed" - name: Read and upsert playwright section if: ${{ !cancelled() }} uses: ./.github/actions/read-and-upsert-comment-section with: pr-number: ${{ github.event.pull_request.number }} section-name: playwright section-file: playwright-section.md token: ${{ github.token }}