# Next.js adapter compatibility suite. # # The same suite Vercel's own adapter runs against, and the only thing that turns # "nextship supports Next.js" from an assertion into evidence. It is manual and # scheduled rather than per-push because it clones and builds Next.js from source # and then runs thirty-two parallel Playwright groups. Sixteen left the slowest # group at 69 minutes against a 90 minute cap, which the suite would outgrow. # # Two ways to run it. With no `tests` input it runs everything. With `tests` it # runs only those files, about ten to a job, which is how a fix is checked against # the suites it targets in minutes rather than an hour. The summary job scores # either. # # Contract: nextjs.org/docs/app/api-reference/adapters/testing-adapters # Author: Gowtham name: conformance on: workflow_dispatch: inputs: nextjsRef: description: 'Next.js repo ref (branch, tag or SHA)' default: 'canary' type: string tests: description: 'Test files to run instead of the whole suite, space separated (for example test/e2e/prerender.test.ts)' default: '' type: string schedule: # Weekly, so a Next.js regression is found before a release rather than after. - cron: '0 3 * * 1' jobs: build: name: Build Next.js and nextship runs-on: ubuntu-latest timeout-minutes: 45 outputs: nextjs-sha: ${{ steps.ref.outputs.sha }} groups: ${{ steps.groups.outputs.list }} steps: # The test matrix is computed here rather than from the input inside the # matrix expression: a conditional over inputs there made GitHub reject every # dispatch that set `tests` with an HTTP 500. - id: groups env: SELECTED_TESTS: ${{ inputs.tests }} run: | node -e ' const selected = (process.env.SELECTED_TESTS || "").split(/\s+/).filter(Boolean).length const total = selected ? Math.min(32, Math.ceil(selected / 10)) : 32 console.log("list=" + JSON.stringify(Array.from({ length: total }, (_, i) => `${i + 1}/${total}`))) ' >> "$GITHUB_OUTPUT" - uses: actions/checkout@v4 with: { path: adapter } - uses: actions/checkout@v4 with: repository: vercel/next.js ref: ${{ inputs.nextjsRef || 'canary' }} path: nextjs fetch-depth: 25 - id: ref run: echo "sha=$(git -C nextjs rev-parse HEAD)" >> "$GITHUB_OUTPUT" # Building Next.js from source is most of this job's time, and it depends on # nothing but the Next.js commit. Keyed on that commit, every run against the # same ref after the first skips it. - id: nextjs-cache uses: actions/cache/restore@v4 with: path: | nextjs ~/.cache/ms-playwright key: nextjs-build-${{ steps.ref.outputs.sha }} - uses: actions/setup-node@v4 with: { node-version: '22' } - run: npm i -g corepack@latest && corepack enable - name: Install and build Next.js if: steps.nextjs-cache.outputs.cache-hit != 'true' working-directory: nextjs run: pnpm install && pnpm build && pnpm install - name: Install Playwright if: steps.nextjs-cache.outputs.cache-hit != 'true' working-directory: nextjs run: pnpm playwright install --with-deps chromium - uses: actions/cache/save@v4 if: steps.nextjs-cache.outputs.cache-hit != 'true' with: path: | nextjs ~/.cache/ms-playwright key: nextjs-build-${{ steps.ref.outputs.sha }} - name: Build nextship working-directory: adapter run: pnpm install && pnpm build && pnpm test # nextship changes on every run, so it is cached per run and apart from Next.js, # which keeps the large entry from being stored twice. - uses: actions/cache/save@v4 with: path: adapter key: conformance-adapter-${{ github.sha }}-${{ github.run_id }} test: name: Tests ${{ matrix.group }} needs: build runs-on: ubuntu-latest timeout-minutes: 90 strategy: fail-fast: false matrix: group: ${{ fromJSON(needs.build.outputs.groups) }} steps: - uses: actions/cache/restore@v4 with: path: | nextjs ~/.cache/ms-playwright key: nextjs-build-${{ needs.build.outputs.nextjs-sha }} fail-on-cache-miss: true - uses: actions/cache/restore@v4 with: path: adapter key: conformance-adapter-${{ github.sha }}-${{ github.run_id }} fail-on-cache-miss: true - uses: actions/setup-node@v4 with: { node-version: '22' } # Playwright's own binary rather than `pnpm playwright`: nothing in a deploy # mode run needs pnpm, and fetching it through corepack is a download that # once crashed a group before any of its tests ran. - name: Ensure Playwright browser working-directory: nextjs run: node_modules/.bin/playwright install chromium - run: chmod +x adapter/conformance/*.sh - name: Run the suite working-directory: nextjs env: NEXT_TEST_MODE: deploy NEXT_E2E_TEST_TIMEOUT: 240000 # Next.js's own list, then ours: tests that assert what Vercel's CDN or # proxy does rather than what Next.js does, each with its reason in the # file. Bun's adapter keeps a list of its own the same way. NEXT_EXTERNAL_TESTS_FILTERS: test/deploy-tests-manifest.json,${{ github.workspace }}/adapter/conformance/deploy-tests-manifest.nextship.json ADAPTER_DIR: ${{ github.workspace }}/adapter IS_TURBOPACK_TEST: 1 # Also what makes the runner write a results file for every suite. NEXT_TEST_JOB: 1 NEXT_TELEMETRY_DISABLED: 1 NEXT_TEST_DEPLOY_SCRIPT_PATH: ${{ github.workspace }}/adapter/conformance/e2e-deploy.sh NEXT_TEST_DEPLOY_LOGS_SCRIPT_PATH: ${{ github.workspace }}/adapter/conformance/e2e-logs.sh NEXT_TEST_CLEANUP_SCRIPT_PATH: ${{ github.workspace }}/adapter/conformance/e2e-cleanup.sh # One key for the whole run, so Server Actions stay decryptable across # the many apps the harness builds. NEXT_SERVER_ACTIONS_ENCRYPTION_KEY: ${{ secrets.CONFORMANCE_ACTIONS_KEY }} # Read from the environment rather than interpolated into the script, so a # value in the input can only ever be test file arguments. SELECTED_TESTS: ${{ inputs.tests }} GROUP: ${{ matrix.group }} run: | if [ -n "$SELECTED_TESTS" ]; then # Word splitting is the point: each file is its own argument. # shellcheck disable=SC2086 node run-tests.js -g "$GROUP" -c 2 --type e2e $SELECTED_TESTS else node run-tests.js --timings -g "$GROUP" -c 2 --type e2e fi - name: Keep the results if: always() uses: actions/upload-artifact@v4 with: name: results-${{ strategy.job-index }} path: nextjs/test/**/*.results.json if-no-files-found: warn retention-days: 30 summary: name: Score the run needs: test # Scored whenever the suite ran, including when groups failed, which they do. if: ${{ !cancelled() && needs.test.result != 'skipped' }} runs-on: ubuntu-latest timeout-minutes: 10 steps: - uses: actions/checkout@v4 with: sparse-checkout: conformance/summarize.mjs sparse-checkout-cone-mode: false - uses: actions/setup-node@v4 with: { node-version: '22' } - uses: actions/download-artifact@v4 with: pattern: results-* merge-multiple: true path: results - run: node conformance/summarize.mjs results summary.json >> "$GITHUB_STEP_SUMMARY" - uses: actions/upload-artifact@v4 with: name: summary path: summary.json retention-days: 90