name: Tests on: push: branches: - main tags: - "!*" pull_request: types: - opened - reopened - synchronize concurrency: # Unique group for this workflow and branch group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: ci-scope: name: Detect CI Scope runs-on: ubuntu-latest outputs: fast-path: ${{ steps.scope.outputs.runtime-fast-path }} validation-fast-path: ${{ steps.scope.outputs.validation-fast-path }} e2e-vercel-slot: ${{ steps.e2e-slot.outputs.slot }} vercel-team-id: ${{ steps.vercel-matrix.outputs.vercel-team-id }} vercel-projects: ${{ steps.vercel-matrix.outputs.vercel-projects }} vercel-prod-matrix: ${{ steps.vercel-matrix.outputs.vercel-prod-matrix }} vercel-ws-transport-matrix: ${{ steps.vercel-matrix.outputs.vercel-ws-transport-matrix }} vercel-http-transport-matrix: ${{ steps.vercel-matrix.outputs.vercel-http-transport-matrix }} vercel-spec9-matrix: ${{ steps.vercel-matrix.outputs.vercel-spec9-matrix }} steps: - name: Checkout Repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Classify changed files id: scope uses: ./.github/actions/detect-ci-scope with: github-token: ${{ secrets.GITHUB_TOKEN }} # Slot index for the Vercel e2e lanes' concurrency groups (see the # `concurrency` block on e2e-vercel-prod for what the groups do). The # slot count is how many runs' worth of Vercel e2e traffic may be in # flight repo-wide at once; everything past that queues. # # The modulo happens here because GitHub expressions have no arithmetic # operators, so the lanes can only read a precomputed value through # `needs`. Keying on run_id rather than something random keeps a re-run # of a run in the same slot it had, so a re-run queues behind the run it # replaces instead of contending with a third one. # # Tunable without a code change: set the repository variable # E2E_VERCEL_CONCURRENCY_SLOTS (Settings > Secrets and variables > # Actions > Variables). Raise it if PRs are waiting too long, lower it # to 1 to serialize the Vercel lanes across the whole repo. - name: Pick Vercel e2e concurrency slot id: e2e-slot env: SLOTS: ${{ vars.E2E_VERCEL_CONCURRENCY_SLOTS }} RUN_ID: ${{ github.run_id }} run: | slots="${SLOTS:-2}" case "$slots" in '' | *[!0-9]*) slots=2 ;; esac if [ "$slots" -lt 1 ]; then slots=1 fi echo "slot=$((RUN_ID % slots))" >> "$GITHUB_OUTPUT" echo "Vercel e2e lanes use slot $((RUN_ID % slots)) of $slots" # Vercel projects and per-lane matrices for the e2e-vercel-* jobs. - name: Build Vercel e2e matrices id: vercel-matrix run: node .github/scripts/vercel-e2e-matrix.js >> "$GITHUB_OUTPUT" # Phase 0: Update PR comment to show tests are running pr-comment-start: name: Create PR Comment runs-on: ubuntu-latest needs: ci-scope if: github.event_name == 'pull_request' && needs.ci-scope.outputs.fast-path != 'true' timeout-minutes: 2 steps: - name: Find existing test comment uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0 id: find-comment with: issue-number: ${{ github.event.pull_request.number }} comment-author: 'github-actions[bot]' body-includes: '' - name: Get existing comment body if: steps.find-comment.outputs.comment-id != '' id: get-comment uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: STARTED_AT: ${{ github.run_started_at }} with: script: | const fs = require('fs'); const comment = await github.rest.issues.getComment({ owner: context.repo.owner, repo: context.repo.repo, comment_id: ${{ steps.find-comment.outputs.comment-id }} }); const body = comment.data.body; // Check if there are actual results (tables) if (body.includes('|') && body.includes('Passed')) { // Extract results section (everything after header) const resultsSection = body .replace(/\n## 🧪 E2E Test Results\n\n> ⚠️ \*\*Results below are stale\*\*[^\n]*\n\n/g, '') .replace(/\n## 🧪 E2E Test Results\n\n/g, '') .replace(/⏳ \*\*Tests are running\.\.\.\*\*\n\n---\n_Started at:[^_]*_\n\n---\n\n/g, '') .replace(/⏳ \*\*Tests are running\.\.\.\*\*\n\n---\n_Started at:[^_]*_/g, '') .trim(); if (resultsSection && resultsSection.includes('|')) { // Write the full stale-banner message to disk and pass the // path to the sticky-pull-request-comment action below. // Inlining the previous results via `message:` blew past // ARG_MAX once the matrix doubled (snapshot + replay). const startedAt = process.env.STARTED_AT; const message = '\n' + '## 🧪 E2E Test Results\n\n' + '> ⚠️ **Results below are stale** and not from the latest commit. This comment will be updated when CI completes on the latest run.\n\n' + '⏳ **Tests are running...**\n\n' + '---\n' + `_Started at: ${startedAt}_\n\n` + '---\n\n' + resultsSection + '\n'; const path = `${process.env.RUNNER_TEMP}/stale-comment.md`; fs.writeFileSync(path, message); core.setOutput('has-results', 'true'); core.setOutput('stale-comment-path', path); } else { core.setOutput('has-results', 'false'); } } else { core.setOutput('has-results', 'false'); } - name: Create new test comment if: steps.find-comment.outputs.comment-id == '' uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 with: header: e2e-test-results message: | ## 🧪 E2E Test Results ⏳ **Tests are running...** This comment will be updated with the results when the tests complete. --- _Started at: ${{ github.run_started_at }}_ - name: Update existing test comment with stale warning if: steps.find-comment.outputs.comment-id != '' && steps.get-comment.outputs.has-results == 'true' uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 with: header: e2e-test-results path: ${{ steps.get-comment.outputs.stale-comment-path }} - name: Update existing test comment without results if: steps.find-comment.outputs.comment-id != '' && steps.get-comment.outputs.has-results != 'true' uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 with: header: e2e-test-results message: | ## 🧪 E2E Test Results ⏳ **Tests are running...** This comment will be updated with the results when the tests complete. --- _Started at: ${{ github.run_started_at }}_ unit: name: Unit Tests (${{ matrix.os }}) runs-on: ${{ matrix.os }} timeout-minutes: 30 needs: ci-scope if: ${{ needs.ci-scope.outputs.validation-fast-path != 'true' && !contains(github.event.pull_request.labels.*.name, 'workflow-server-test') }} strategy: fail-fast: false matrix: os: [ubuntu-latest, windows-latest] env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} steps: - name: Checkout Repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Setup environment uses: ./.github/actions/setup-workflow-dev with: setup-rust: 'true' build-packages: 'false' - name: Run Unit Tests run: pnpm turbo test --filter='!./docs' --filter='!./workbench/vitest' serializable-revivers: name: Serializable Reviver Compatibility runs-on: ubuntu-latest timeout-minutes: 10 needs: ci-scope if: ${{ needs.ci-scope.outputs.validation-fast-path != 'true' && !contains(github.event.pull_request.labels.*.name, 'workflow-server-test') }} env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} steps: - name: Checkout Repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Setup environment uses: ./.github/actions/setup-workflow-dev with: build-packages: 'false' - name: Build reviver packages run: pnpm turbo run build --filter='@workflow/cli' --filter='@workflow/web-shared' - name: Check web-shared and CLI revivers run: pnpm vitest run packages/web-shared/test/serializable-revivers.test.ts build-error-messages: name: Node.js Module Build Errors Test runs-on: ubuntu-latest timeout-minutes: 30 needs: ci-scope if: ${{ needs.ci-scope.outputs.fast-path != 'true' && !contains(github.event.pull_request.labels.*.name, 'workflow-server-test') }} env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} steps: - name: Checkout Repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Setup pnpm uses: pnpm/setup@fa36a5ef7dcdefc3a7cb25655c122e00cf6e636e # v3.1.2 with: install: true require-lockfile: true runtime: node@24 cache: true - name: Run Initial Build run: pnpm turbo run build --filter='!./workbench/*' - name: Generate Workflows Registry run: node workbench/scripts/generate-workflows-registry.js workbench/nextjs-turbopack/workflows workbench/nextjs-turbopack/_workflows.ts - name: Run Build Error Tests run: pnpm vitest run packages/core/e2e/build-errors.test.ts env: APP_NAME: "nextjs-turbopack" # Regression test for the o2flow beta.26 incident: hook.resume() from a # plain API route must resolve the workflow world inside an isolated # (lambda-like) route bundle. Runs in this job because it also performs # a production Turbopack build of the nextjs-turbopack workbench. - name: Run Route Bundle Isolation Tests run: pnpm vitest run packages/core/e2e/route-bundle-isolation.test.ts env: APP_NAME: "nextjs-turbopack" vitest-plugin: name: Vitest Plugin Tests (v${{ matrix.vitest }}, ${{ matrix.vm }}) runs-on: ubuntu-latest needs: ci-scope if: ${{ needs.ci-scope.outputs.fast-path != 'true' }} strategy: fail-fast: false matrix: vitest: [3, 4, 5] # Workflow VM engines: node:vm (default) and the opt-in QuickJS # WASM engine (WORKFLOW_VM=quickjs). vm: [node, quickjs] env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} steps: - name: Checkout Repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup environment uses: ./.github/actions/setup-workflow-dev with: setup-rust: 'true' build-packages: 'false' - name: Install supported Vitest version # The plugin setup files and workbench must resolve the same Vitest instance. run: pnpm --filter @workflow/vitest --filter @workflow/vitest-workbench add --save-dev vitest@${{ matrix.vitest }} - name: Build packages run: pnpm turbo run build --filter='!./workbench/*' - name: Test Vitest plugin run: pnpm --filter @workflow/vitest test - name: Test workflow execution run: pnpm test working-directory: workbench/vitest env: WORKFLOW_VM: ${{ matrix.vm }} e2e-package-build: name: Build Shared E2E Packages runs-on: ubuntu-latest needs: ci-scope if: ${{ needs.ci-scope.outputs.fast-path != 'true' && !contains(github.event.pull_request.labels.*.name, 'workflow-server-test') }} timeout-minutes: 30 env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} steps: - name: Checkout Repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Setup environment and build packages uses: ./.github/actions/setup-workflow-dev with: build-packages: 'true' - name: Upload package build artifacts uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: e2e-package-build-artifacts # Keep this list aligned with package build outputs in turbo.json. # upload-artifact strips the common packages/ root; consumers restore into packages/. path: | packages/*/dist packages/*/.well-known packages/*/src/version.ts packages/core/src/runtime/quickjs-assets.generated.ts packages/swc-plugin-workflow/swc_plugin_workflow.wasm packages/swc-plugin-workflow/build-hash.json include-hidden-files: true retention-days: 1 e2e-vercel-prod: name: E2E Vercel Prod Tests (${{ matrix.app.name }} - ${{ matrix.vm }}) runs-on: ubuntu-latest timeout-minutes: 30 continue-on-error: ${{ matrix.app.advisory }} needs: ci-scope if: ${{ needs.ci-scope.outputs.fast-path != 'true' }} # Repo-wide cap on how much traffic CI can point at world-vercel at once. # # The group is per (lane, matrix cell, slot), not per run, so a single # PR still fans its whole matrix out in parallel: within one run every # cell is a distinct group. What collides is the *same* cell from a # *different* run, which waits instead of doubling the load on the # backend and on the shared Vercel project this cell deploys to. With # the default two slots that leaves at most two runs' worth of Vercel # e2e in flight repo-wide, however many PRs are pushed at once. # # `queue: max` is what makes this safe to apply to a required check. # The default (`queue: single`) keeps one pending entry per group and # cancels the rest, which would turn a burst of PRs into cancelled # jobs and a red E2E Required Check; `max` queues up to 100 instead. # It cannot be combined with `cancel-in-progress: true`: GitHub # rejects the workflow. Superseded runs are still cancelled by the # workflow-level concurrency at the top of this file, which releases # their queued jobs too. concurrency: group: e2e-vercel-prod-${{ matrix.app.name }}-${{ matrix.vm }}-${{ needs.ci-scope.outputs.e2e-vercel-slot }} queue: max permissions: id-token: write contents: read strategy: fail-fast: false # Projects and lane subsets live in .github/scripts/vercel-e2e-matrix.js. matrix: ${{ fromJSON(needs.ci-scope.outputs.vercel-prod-matrix) }} env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} WORKFLOW_PUBLIC_MANIFEST: '1' WORKFLOW_RETURN_VALUE_POLL_INTERVAL_MS: '5000' VERCEL_TEAM_ID: ${{ needs.ci-scope.outputs.vercel-team-id }} steps: - name: Checkout Repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Setup environment uses: ./.github/actions/setup-workflow-dev with: build-packages: 'false' - name: Build CLI run: pnpm turbo run build --filter='@workflow/cli' # Pull requests test their head commit's preview deployment and main # tests its production deployment. `changeset-release/*` PRs (the # changesets "Version Packages" PR) are deliberately not deployed: the # branch is force-pushed and can point at main's HEAD SHA, so every # project's vercel.json sets `git.deploymentEnabled` false for it. Their # content is main plus a version-bump commit, so the action resolves the # production deployment of their base commit instead. - name: Resolve Vercel deployment id: deployment uses: ./.github/actions/resolve-vercel-deployment with: project-id: ${{ matrix.app.project-id }} team-id: ${{ env.VERCEL_TEAM_ID }} token: ${{ secrets.VERCEL_LABS_TOKEN }} - name: Record E2E start time id: e2eStart run: echo "ms=$(($(date +%s) * 1000))" >> "$GITHUB_OUTPUT" - name: Run E2E Tests run: pnpm run test:e2e --reporter=verbose --reporter=json --reporter=./packages/core/e2e/github-reporter.ts "--outputFile=e2e-vercel-prod-$APP_NAME-$WORKFLOW_VM.json" env: NODE_OPTIONS: "--enable-source-maps" DEPLOYMENT_URL: ${{ steps.deployment.outputs.deployment-url }} VERCEL_DEPLOYMENT_ID: ${{ steps.deployment.outputs.deployment-id }} APP_NAME: ${{ matrix.app.name }} WORKFLOW_VM: ${{ matrix.vm }} WORKFLOW_VERCEL_ENV: ${{ steps.deployment.outputs.environment }} WORKFLOW_VERCEL_AUTH_TOKEN: ${{ secrets.VERCEL_LABS_TOKEN }} WORKFLOW_VERCEL_TEAM: ${{ env.VERCEL_TEAM_ID }} WORKFLOW_VERCEL_PROJECT: ${{ matrix.app.project-id }} WORKFLOW_VERCEL_PROJECT_SLUG: ${{ matrix.app.project-slug }} # The trusted-sources OIDC token (x-vercel-trusted-oidc-idp-token) # is minted on demand inside `scripts/trusted-sources-headers.mjs` # via the runner's ACTIONS_ID_TOKEN_REQUEST_URL/_TOKEN env vars # (exposed when `permissions: id-token: write` is set on the job), # and re-minted before its 5-minute expiry. This lets long e2e # suites keep a fresh token throughout the run instead of using # a single pre-minted token that would expire mid-suite. # See: https://vercel.com/docs/deployment-protection/methods-to-bypass-deployment-protection/trusted-sources # Preview deployments are wired to the protected world-vercel backend # preview and production deployments to the public # vercel-workflow.com server; the harness has to read run state # from the same server the app writes it to. VERCEL_WORKFLOW_SERVER_URL: ${{ steps.deployment.outputs.environment == 'preview' && secrets.VERCEL_WORKFLOW_SERVER_URL || '' }} - name: Report E2E results if: always() uses: ./.github/actions/report-vercel-e2e with: results-name: vercel-prod-${{ matrix.app.name }}-${{ matrix.vm }} app: ${{ matrix.app.name }} summary-title: E2E Vercel Prod (${{ matrix.app.name }} - ${{ matrix.vm }}) capture-runtime-logs: ${{ job.status == 'failure' }} deployment-id: ${{ steps.deployment.outputs.deployment-id }} environment: ${{ steps.deployment.outputs.environment }} project-id: ${{ matrix.app.project-id }} team-id: ${{ env.VERCEL_TEAM_ID }} token: ${{ secrets.VERCEL_LABS_TOKEN }} e2e-start-ms: ${{ steps.e2eStart.outputs.ms }} e2e-vercel-multi-region: # Vercel-specific multi-region routing suite (start()'s `region` # option). Deliberately separate from e2e-vercel-prod: that job runs # the shared suite as a matrix across all worlds/frameworks, while # this functionality only exists for @workflow/world-vercel and only # the nextjs-turbopack workbench is deployed multi-region # (workbench/nextjs-turbopack/vercel.json pins iad1+sfo1+fra1). name: E2E Vercel Multi-Region Tests (nextjs-turbopack) runs-on: ubuntu-latest # 19 regions: the implicit-route cases pay one cold start each, # sequentially. timeout-minutes: 35 needs: ci-scope if: ${{ needs.ci-scope.outputs.fast-path != 'true' }} # One cell, so one group per slot. See e2e-vercel-prod's block above. concurrency: group: e2e-vercel-multi-region-${{ needs.ci-scope.outputs.e2e-vercel-slot }} queue: max permissions: id-token: write contents: read env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} WORKFLOW_PUBLIC_MANIFEST: '1' WORKFLOW_RETURN_VALUE_POLL_INTERVAL_MS: '5000' VERCEL_TEAM_ID: ${{ needs.ci-scope.outputs.vercel-team-id }} VERCEL_PROJECT_ID: ${{ fromJSON(needs.ci-scope.outputs.vercel-projects)['nextjs-turbopack']['project-id'] }} steps: - name: Checkout Repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Setup environment uses: ./.github/actions/setup-workflow-dev with: build-packages: 'false' - name: Build CLI run: pnpm turbo run build --filter='@workflow/cli' # See the note on e2e-vercel-prod's deployment step. - name: Resolve Vercel deployment id: deployment uses: ./.github/actions/resolve-vercel-deployment with: project-id: ${{ env.VERCEL_PROJECT_ID }} team-id: ${{ env.VERCEL_TEAM_ID }} token: ${{ secrets.VERCEL_LABS_TOKEN }} - name: Record E2E start time id: e2eStart run: echo "ms=$(($(date +%s) * 1000))" >> "$GITHUB_OUTPUT" - name: Run Multi-Region E2E Tests run: pnpm vitest run packages/core/e2e/e2e-region.test.ts --reporter=verbose --reporter=json --reporter=./packages/core/e2e/github-reporter.ts "--outputFile=e2e-vercel-multi-region-$APP_NAME.json" env: NODE_OPTIONS: "--enable-source-maps" DEPLOYMENT_URL: ${{ steps.deployment.outputs.deployment-url }} VERCEL_DEPLOYMENT_ID: ${{ steps.deployment.outputs.deployment-id }} APP_NAME: nextjs-turbopack WORKFLOW_VERCEL_ENV: ${{ steps.deployment.outputs.environment }} WORKFLOW_VERCEL_AUTH_TOKEN: ${{ secrets.VERCEL_LABS_TOKEN }} WORKFLOW_VERCEL_TEAM: ${{ env.VERCEL_TEAM_ID }} WORKFLOW_VERCEL_PROJECT: ${{ env.VERCEL_PROJECT_ID }} WORKFLOW_VERCEL_PROJECT_SLUG: ${{ fromJSON(needs.ci-scope.outputs.vercel-projects)['nextjs-turbopack']['project-slug'] }} # See the note on e2e-vercel-prod. VERCEL_WORKFLOW_SERVER_URL: ${{ steps.deployment.outputs.environment == 'preview' && secrets.VERCEL_WORKFLOW_SERVER_URL || '' }} - name: Report E2E results if: always() uses: ./.github/actions/report-vercel-e2e with: results-name: vercel-multi-region-nextjs-turbopack app: nextjs-turbopack summary-title: E2E Vercel Multi-Region (nextjs-turbopack) capture-runtime-logs: ${{ job.status == 'failure' }} deployment-id: ${{ steps.deployment.outputs.deployment-id }} environment: ${{ steps.deployment.outputs.environment }} project-id: ${{ env.VERCEL_PROJECT_ID }} team-id: ${{ env.VERCEL_TEAM_ID }} token: ${{ secrets.VERCEL_LABS_TOKEN }} e2e-start-ms: ${{ steps.e2eStart.outputs.ms }} e2e-vercel-queue-hold: # Send-and-hold (`QueueOptions.experimental_hold`) against the real # queue service: packages/core/e2e/e2e-queue-hold.test.ts holds queue # health checks from this runner and checks who ran them. A World and # queue feature, the same for every framework, so one app is enough. # # Until send-and-hold is on for the target, every send comes back # `not_held` and the suite proves only the fallback (the queue delivers # the message). Set the repository variable # WORKFLOW_E2E_EXPECT_QUEUE_HOLD=1 once it is on, so a send that is not # held fails the job. name: E2E Vercel Queue Hold Tests (nextjs-turbopack) runs-on: ubuntu-latest timeout-minutes: 15 needs: ci-scope if: ${{ needs.ci-scope.outputs.fast-path != 'true' }} # One cell, so one group per slot. See e2e-vercel-prod's block above. concurrency: group: e2e-vercel-queue-hold-${{ needs.ci-scope.outputs.e2e-vercel-slot }} queue: max permissions: id-token: write contents: read env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} VERCEL_TEAM_ID: ${{ needs.ci-scope.outputs.vercel-team-id }} VERCEL_PROJECT_ID: ${{ fromJSON(needs.ci-scope.outputs.vercel-projects)['nextjs-turbopack']['project-id'] }} steps: - name: Checkout Repo uses: actions/checkout@v6 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Setup environment uses: ./.github/actions/setup-workflow-dev with: build-packages: 'false' # Builds the packages the suite imports (@workflow/core, world, ...). - name: Build CLI run: pnpm turbo run build --filter='@workflow/cli' # See the note on e2e-vercel-prod's deployment step. - name: Resolve Vercel deployment id: deployment uses: ./.github/actions/resolve-vercel-deployment with: project-id: ${{ env.VERCEL_PROJECT_ID }} team-id: ${{ env.VERCEL_TEAM_ID }} token: ${{ secrets.VERCEL_LABS_TOKEN }} - name: Record E2E start time id: e2eStart run: echo "ms=$(($(date +%s) * 1000))" >> "$GITHUB_OUTPUT" - name: Run Queue Hold E2E Tests run: pnpm vitest run packages/core/e2e/e2e-queue-hold.test.ts --reporter=verbose --reporter=json --reporter=./packages/core/e2e/github-reporter.ts "--outputFile=e2e-vercel-queue-hold-$APP_NAME.json" env: NODE_OPTIONS: "--enable-source-maps" DEPLOYMENT_URL: ${{ steps.deployment.outputs.deployment-url }} VERCEL_DEPLOYMENT_ID: ${{ steps.deployment.outputs.deployment-id }} APP_NAME: nextjs-turbopack WORKFLOW_VERCEL_ENV: ${{ steps.deployment.outputs.environment }} WORKFLOW_VERCEL_AUTH_TOKEN: ${{ secrets.VERCEL_LABS_TOKEN }} WORKFLOW_VERCEL_TEAM: ${{ env.VERCEL_TEAM_ID }} WORKFLOW_VERCEL_PROJECT: ${{ env.VERCEL_PROJECT_ID }} WORKFLOW_VERCEL_PROJECT_SLUG: ${{ fromJSON(needs.ci-scope.outputs.vercel-projects)['nextjs-turbopack']['project-slug'] }} WORKFLOW_E2E_EXPECT_QUEUE_HOLD: ${{ vars.WORKFLOW_E2E_EXPECT_QUEUE_HOLD }} # See the note on e2e-vercel-prod. VERCEL_WORKFLOW_SERVER_URL: ${{ steps.deployment.outputs.environment == 'preview' && secrets.VERCEL_WORKFLOW_SERVER_URL || '' }} - name: Report E2E results if: always() uses: ./.github/actions/report-vercel-e2e with: results-name: vercel-queue-hold-nextjs-turbopack app: nextjs-turbopack summary-title: E2E Vercel Queue Hold (nextjs-turbopack) capture-runtime-logs: ${{ job.status == 'failure' }} deployment-id: ${{ steps.deployment.outputs.deployment-id }} environment: ${{ steps.deployment.outputs.environment }} project-id: ${{ env.VERCEL_PROJECT_ID }} team-id: ${{ env.VERCEL_TEAM_ID }} token: ${{ secrets.VERCEL_LABS_TOKEN }} e2e-start-ms: ${{ steps.e2eStart.outputs.ms }} e2e-vercel-ws-transport: # Dedicated coverage for world-vercel's WebSocket events transport # (WORKFLOW_EVENTS_TRANSPORT=ws — see events-v4.ts's # isWsEventsTransportEnabled). Every other Vercel lane inherits the # default transport but checks nothing; this is the only lane that # *asserts* the socket carried the events. See # e2e-vercel-http-transport below for the other side. # # Runs the full e2e suite against its own `vercel deploy` rather than the # Git deployment: that is the only way to get a second, independent # deployment of a project with its own env var, without creating a whole # new project. name: E2E Vercel WS Transport Test (${{ matrix.app.name }}) runs-on: ubuntu-latest timeout-minutes: 30 needs: ci-scope if: ${{ needs.ci-scope.outputs.fast-path != 'true' }} # See e2e-vercel-prod's block above. This lane's cells are worth capping # for a second reason: each one is a full remote `vercel deploy`, so a # burst of PRs also queues builds on the shared project. concurrency: group: e2e-vercel-ws-transport-${{ matrix.app.name }}-${{ needs.ci-scope.outputs.e2e-vercel-slot }} queue: max permissions: id-token: write contents: read strategy: fail-fast: false matrix: ${{ fromJSON(needs.ci-scope.outputs.vercel-ws-transport-matrix) }} env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} WORKFLOW_PUBLIC_MANIFEST: '1' VERCEL_TEAM_ID: ${{ needs.ci-scope.outputs.vercel-team-id }} steps: - name: Checkout Repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Setup environment uses: ./.github/actions/setup-workflow-dev with: build-packages: 'false' - name: Build CLI run: pnpm turbo run build --filter='@workflow/cli' # Strict mode turns a silent HTTP demotion into a failed run, so this # lane asserts the socket carried traffic instead of only asserting the # feature was harmless. Scoped to `step_completed`; see # STRICT_WS_EVENT_TYPES in events-v4.ts for why other event types fall # back legitimately. Never set on e2e-vercel-prod. - name: Deploy a dedicated WS-transport preview id: deployment uses: ./.github/actions/deploy-vercel-preview with: project-id: ${{ matrix.app.project-id }} team-id: ${{ env.VERCEL_TEAM_ID }} token: ${{ secrets.VERCEL_LABS_TOKEN }} env: | WORKFLOW_EVENTS_TRANSPORT=ws WORKFLOW_INTERNAL_EVENTS_TRANSPORT_STRICT=1 - name: Record E2E start time id: e2eStart run: echo "ms=$(($(date +%s) * 1000))" >> "$GITHUB_OUTPUT" - name: Run E2E Tests run: pnpm run test:e2e --reporter=verbose --reporter=json --reporter=./packages/core/e2e/github-reporter.ts "--outputFile=e2e-vercel-ws-transport-$APP_NAME.json" env: NODE_OPTIONS: "--enable-source-maps" DEPLOYMENT_URL: ${{ steps.deployment.outputs.deployment-url }} VERCEL_DEPLOYMENT_ID: ${{ steps.deployment.outputs.deployment-id }} APP_NAME: ${{ matrix.app.name }} WORKFLOW_VERCEL_ENV: "preview" WORKFLOW_VERCEL_AUTH_TOKEN: ${{ secrets.VERCEL_LABS_TOKEN }} WORKFLOW_VERCEL_TEAM: ${{ env.VERCEL_TEAM_ID }} WORKFLOW_VERCEL_PROJECT: ${{ matrix.app.project-id }} WORKFLOW_VERCEL_PROJECT_SLUG: ${{ matrix.app.project-slug }} VERCEL_WORKFLOW_SERVER_URL: ${{ github.ref != 'refs/heads/main' && secrets.VERCEL_WORKFLOW_SERVER_URL || '' }} - name: Report E2E results if: always() uses: ./.github/actions/report-vercel-e2e with: results-name: vercel-ws-transport-${{ matrix.app.name }} app: ${{ matrix.app.name }} summary-title: E2E Vercel WS Transport (${{ matrix.app.name }}) capture-runtime-logs: ${{ job.status == 'failure' }} deployment-id: ${{ steps.deployment.outputs.deployment-id }} environment: preview project-id: ${{ matrix.app.project-id }} team-id: ${{ env.VERCEL_TEAM_ID }} token: ${{ secrets.VERCEL_LABS_TOKEN }} e2e-start-ms: ${{ steps.e2eStart.outputs.ms }} e2e-vercel-spec9: # Spec 9 runs (the single orchestrator) are opt-in with # WORKFLOW_SPEC_VERSION=9, so every other Vercel lane runs the default, # spec 8. This lane deploys its own preview with the variable set and runs # the full suite against it on both VMs. # # The variable is set on the deployment AND on the test runner: a # cross-deployment start() stamps the lower of the target's and the # caller's version, so with only the deployment env set this lane would # silently test spec 8. The e2e test "stamps the spec version the lane # opts in to" fails if either side stops seeing it. # # Same dedicated-deployment pattern as e2e-vercel-ws-transport above. name: E2E Vercel Spec 9 Test (${{ matrix.app.name }}, ${{ matrix.vm }}) runs-on: ubuntu-latest timeout-minutes: 30 needs: ci-scope if: ${{ needs.ci-scope.outputs.fast-path != 'true' }} concurrency: group: e2e-vercel-spec9-${{ matrix.app.name }}-${{ matrix.vm }}-${{ needs.ci-scope.outputs.e2e-vercel-slot }} queue: max permissions: id-token: write contents: read strategy: fail-fast: false matrix: ${{ fromJSON(needs.ci-scope.outputs.vercel-spec9-matrix) }} env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} WORKFLOW_PUBLIC_MANIFEST: '1' VERCEL_TEAM_ID: ${{ needs.ci-scope.outputs.vercel-team-id }} steps: - name: Checkout Repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Setup environment uses: ./.github/actions/setup-workflow-dev with: build-packages: 'false' - name: Build CLI run: pnpm turbo run build --filter='@workflow/cli' - name: Deploy a dedicated spec 9 preview id: deployment uses: ./.github/actions/deploy-vercel-preview with: project-id: ${{ matrix.app.project-id }} team-id: ${{ env.VERCEL_TEAM_ID }} token: ${{ secrets.VERCEL_LABS_TOKEN }} env: | WORKFLOW_SPEC_VERSION=9 - name: Record E2E start time id: e2eStart run: echo "ms=$(($(date +%s) * 1000))" >> "$GITHUB_OUTPUT" - name: Run E2E Tests run: pnpm run test:e2e --reporter=verbose --reporter=json --reporter=./packages/core/e2e/github-reporter.ts "--outputFile=e2e-vercel-spec9-$APP_NAME-$WORKFLOW_VM.json" env: NODE_OPTIONS: "--enable-source-maps" DEPLOYMENT_URL: ${{ steps.deployment.outputs.deployment-url }} VERCEL_DEPLOYMENT_ID: ${{ steps.deployment.outputs.deployment-id }} APP_NAME: ${{ matrix.app.name }} WORKFLOW_VM: ${{ matrix.vm }} WORKFLOW_SPEC_VERSION: '9' WORKFLOW_VERCEL_ENV: "preview" WORKFLOW_VERCEL_AUTH_TOKEN: ${{ secrets.VERCEL_LABS_TOKEN }} WORKFLOW_VERCEL_TEAM: ${{ env.VERCEL_TEAM_ID }} WORKFLOW_VERCEL_PROJECT: ${{ matrix.app.project-id }} WORKFLOW_VERCEL_PROJECT_SLUG: ${{ matrix.app.project-slug }} VERCEL_WORKFLOW_SERVER_URL: ${{ github.ref != 'refs/heads/main' && secrets.VERCEL_WORKFLOW_SERVER_URL || '' }} - name: Report E2E results if: always() uses: ./.github/actions/report-vercel-e2e with: results-name: vercel-spec9-${{ matrix.app.name }}-${{ matrix.vm }} app: ${{ matrix.app.name }} summary-title: E2E Vercel Spec 9 (${{ matrix.app.name }}, ${{ matrix.vm }}) capture-runtime-logs: ${{ job.status == 'failure' }} deployment-id: ${{ steps.deployment.outputs.deployment-id }} environment: preview project-id: ${{ matrix.app.project-id }} team-id: ${{ env.VERCEL_TEAM_ID }} token: ${{ secrets.VERCEL_LABS_TOKEN }} e2e-start-ms: ${{ steps.e2eStart.outputs.ms }} e2e-vercel-http-transport: # The counterpart to e2e-vercel-ws-transport. With # WORKFLOW_EVENTS_TRANSPORT defaulting to `ws`, e2e-vercel-prod — which # sets no transport env var at all — is a WebSocket lane, so nothing else # exercises the HTTP events transport against a real Vercel deployment. # # HTTP is the fallback path, and the fallback is *silent*: # resolveWsTransport returning null costs a write nothing and logs # nothing. A path that is both unexercised and quiet is the exact # shape of the durabench bug this lane came out of, so it is # unconditional and required. # # Six apps, chosen by server shape rather than by count; each row is a # real `vercel deploy` charged to every PR, so the full 14 is too # expensive, but two was too thin for a transport that is still # supported. The shapes, one fixture each: # example - baseline, no framework server # nextjs-turbopack - Next; also the only fixture emitting OTEL spans, # so the only one that can show which transport ran # vite - Vite SSR # express - Node req/res server # nitro - Nitro/h3, which also covers nuxt # hono - fetch-API Request/Response server, a genuinely # different mount shape from express # # Omitted because they duplicate a shape already here: nextjs-webpack # (same runtime as turbopack, different bundler), nuxt (nitro), fastify # and nest (both express-shaped), astro and tanstack-start (adapter # variants). `python` is omitted because it has no conformance gate and # needs routes this suite does not serve - see the e2e-python job. name: E2E Vercel HTTP Transport Test (${{ matrix.app.name }}) runs-on: ubuntu-latest timeout-minutes: 30 needs: ci-scope if: ${{ needs.ci-scope.outputs.fast-path != 'true' }} # See e2e-vercel-prod and e2e-vercel-ws-transport above. concurrency: group: e2e-vercel-http-transport-${{ matrix.app.name }}-${{ needs.ci-scope.outputs.e2e-vercel-slot }} queue: max permissions: id-token: write contents: read strategy: fail-fast: false matrix: ${{ fromJSON(needs.ci-scope.outputs.vercel-http-transport-matrix) }} env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} WORKFLOW_PUBLIC_MANIFEST: '1' VERCEL_TEAM_ID: ${{ needs.ci-scope.outputs.vercel-team-id }} steps: - name: Checkout Repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Setup environment uses: ./.github/actions/setup-workflow-dev with: build-packages: 'false' - name: Build CLI run: pnpm turbo run build --filter='@workflow/cli' # `http` is the one value that opts out; the gate compares against # exactly this string, so a typo here would silently test the default # instead of the fallback and this lane would quietly become a # duplicate of e2e-vercel-prod. - name: Deploy a dedicated HTTP-transport preview id: deployment uses: ./.github/actions/deploy-vercel-preview with: project-id: ${{ matrix.app.project-id }} team-id: ${{ env.VERCEL_TEAM_ID }} token: ${{ secrets.VERCEL_LABS_TOKEN }} env: | WORKFLOW_EVENTS_TRANSPORT=http - name: Record E2E start time id: e2eStart run: echo "ms=$(($(date +%s) * 1000))" >> "$GITHUB_OUTPUT" - name: Run E2E Tests run: pnpm run test:e2e --reporter=verbose --reporter=json --reporter=./packages/core/e2e/github-reporter.ts "--outputFile=e2e-vercel-http-transport-$APP_NAME.json" env: NODE_OPTIONS: "--enable-source-maps" DEPLOYMENT_URL: ${{ steps.deployment.outputs.deployment-url }} VERCEL_DEPLOYMENT_ID: ${{ steps.deployment.outputs.deployment-id }} APP_NAME: ${{ matrix.app.name }} WORKFLOW_VERCEL_ENV: "preview" WORKFLOW_VERCEL_AUTH_TOKEN: ${{ secrets.VERCEL_LABS_TOKEN }} WORKFLOW_VERCEL_TEAM: ${{ env.VERCEL_TEAM_ID }} WORKFLOW_VERCEL_PROJECT: ${{ matrix.app.project-id }} WORKFLOW_VERCEL_PROJECT_SLUG: ${{ matrix.app.project-slug }} VERCEL_WORKFLOW_SERVER_URL: ${{ github.ref != 'refs/heads/main' && secrets.VERCEL_WORKFLOW_SERVER_URL || '' }} - name: Report E2E results if: always() uses: ./.github/actions/report-vercel-e2e with: results-name: vercel-http-transport-${{ matrix.app.name }} app: ${{ matrix.app.name }} summary-title: E2E Vercel HTTP Transport (${{ matrix.app.name }}) capture-runtime-logs: ${{ job.status == 'failure' }} deployment-id: ${{ steps.deployment.outputs.deployment-id }} environment: preview project-id: ${{ matrix.app.project-id }} team-id: ${{ env.VERCEL_TEAM_ID }} token: ${{ secrets.VERCEL_LABS_TOKEN }} e2e-start-ms: ${{ steps.e2eStart.outputs.ms }} getTestMatrix: name: Get Test Matrix runs-on: ubuntu-latest needs: ci-scope if: ${{ needs.ci-scope.outputs.fast-path != 'true' && !contains(github.event.pull_request.labels.*.name, 'workflow-server-test') }} outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - name: Checkout Repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Setup environment uses: ./.github/actions/setup-workflow-dev with: install-dependencies: 'false' build-packages: 'false' cache-pnpm: 'false' - id: set-matrix run: | MATRIX_JSON="$(node ./scripts/create-test-matrix.mjs | jq -c .)" echo "matrix=$MATRIX_JSON" >> "$GITHUB_OUTPUT" e2e-local-dev: name: E2E Local Dev Tests (${{ matrix.app.name }} - ${{ matrix.app.runLabel }}) runs-on: ubuntu-latest timeout-minutes: 30 if: ${{ needs.ci-scope.outputs.fast-path != 'true' && !contains(github.event.pull_request.labels.*.name, 'workflow-server-test') }} needs: [ci-scope, getTestMatrix, e2e-package-build] strategy: fail-fast: false matrix: ${{fromJson(needs.getTestMatrix.outputs.matrix)}} env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} WORKFLOW_PUBLIC_MANIFEST: '1' steps: - name: Checkout Repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Setup environment uses: ./.github/actions/setup-workflow-dev with: install-dependencies: ${{ !matrix.app.canary }} build-packages: 'false' - name: Setup canary if: ${{ matrix.app.canary }} env: APP_NAME: ${{ matrix.app.name }} run: | cat packages/next/package.json | jq '.dependencies.next|="16.3.1-canary.17"' > packages/next/package.json.new && mv packages/next/package.json.new packages/next/package.json cat "workbench/$APP_NAME/package.json" | jq '.dependencies.next|="16.3.1-canary.17"' > "workbench/$APP_NAME/package.json.new" && mv "workbench/$APP_NAME/package.json.new" "workbench/$APP_NAME/package.json" pnpm install --no-frozen-lockfile - name: Download shared package builds if: ${{ !matrix.app.canary }} uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: e2e-package-build-artifacts path: packages - name: Run Initial Build for Next.js canary if: ${{ matrix.app.canary }} run: pnpm turbo run build --filter='!./workbench/*' - name: Prepare workbench path id: prepare-workbench uses: ./.github/actions/prepare-workbench-path with: app-name: ${{ matrix.app.name }} - name: Run E2E Tests run: | export DEV_SERVER_LOG_PATH="$GITHUB_WORKSPACE/dev-server-${{ matrix.app.name }}-${{ matrix.app.artifactSuffix }}.log" rm -f "$DEV_SERVER_LOG_PATH" # Queue auth stays off on this lane until the team decides to run # the Local lanes with it on. (The runner shares the app's data # directory, so its start()/resumeHook() deliveries carry the app's # key either way.) Off requires dynamic workflows off: the runtime # refuses WORKFLOW_QUEUE_AUTH=off under them. Dynamic workflows are # covered by e2e-local-dynamic, with queue auth on. (cd "$WORKBENCH_APP_PATH" && WORKFLOW_QUEUE_AUTH=off pnpm dev 2>&1 | tee "$DEV_SERVER_LOG_PATH") & echo "starting tests in 10 seconds" && sleep 10 # dev.test.ts hands the server on only once the rebuilds its fixture # restores set off have finished (see waitForDevServerToSettle). pnpm vitest run packages/core/e2e/dev.test.ts pnpm run test:e2e --reporter=verbose --reporter=json --reporter=./packages/core/e2e/github-reporter.ts --outputFile=e2e-local-dev-${{ matrix.app.name }}-${{ matrix.app.artifactSuffix }}.json env: NODE_OPTIONS: "--enable-source-maps" APP_NAME: ${{ matrix.app.name }} WORKBENCH_APP_PATH: ${{ steps.prepare-workbench.outputs.workbench_app_path }} DEPLOYMENT_URL: "http://localhost:${{ matrix.app.name == 'sveltekit' && '5173' || (matrix.app.name == 'astro' && '4321' || '3000') }}" DEV_TEST_CONFIG: ${{ toJSON(matrix.app) }} WORKFLOW_DEV_HMR_LOGS: "1" NEXT_CANARY: ${{ matrix.app.canary && '1' || '' }} WORKFLOW_VM: ${{ matrix.app.vm || '' }} WORKFLOW_SNAPSHOT_THRESHOLD: ${{ matrix.app.snapshotThreshold || '' }} # Local worlds have no run encryption keys; snapshot legs opt in. WORKFLOW_SNAPSHOT_ALLOW_UNENCRYPTED: ${{ matrix.app.snapshotThreshold && '1' || '' }} # Snapshot legs log each restore so the check below can count them. DEBUG: ${{ matrix.app.snapshotThreshold && 'workflow:runtime:info' || '' }} - name: Check VM snapshots were restored if: ${{ matrix.app.snapshotThreshold }} run: | # A snapshot leg that never restores would still pass the suite # (every miss falls back to full replay), so require restores. restores=$(grep -c "QuickJS runtime: restored VM snapshot" "$GITHUB_WORKSPACE/dev-server-${{ matrix.app.name }}-${{ matrix.app.artifactSuffix }}.log" || true) echo "VM snapshot restores: $restores" test "$restores" -gt 0 - name: Generate E2E summary if: always() run: node .github/scripts/aggregate-e2e-results.js . --job-name "E2E Local Dev (${{ matrix.app.name }} - ${{ matrix.app.runLabel }})" >> $GITHUB_STEP_SUMMARY || true - name: Upload E2E results if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: e2e-results-local-dev-${{ matrix.app.name }}-${{ matrix.app.artifactSuffix }} path: | e2e-local-dev-${{ matrix.app.name }}-${{ matrix.app.artifactSuffix }}.json e2e-local-dev-${{ matrix.app.name }}-${{ matrix.app.artifactSuffix }}.flaky.json e2e-infra-${{ matrix.app.name }}-local.json retention-days: 7 if-no-files-found: ignore e2e-local-prod: name: E2E Local Prod Tests (${{ matrix.app.name }} - ${{ matrix.app.runLabel }}) runs-on: ubuntu-latest timeout-minutes: 30 if: ${{ needs.ci-scope.outputs.fast-path != 'true' && !contains(github.event.pull_request.labels.*.name, 'workflow-server-test') }} needs: [ci-scope, getTestMatrix, e2e-package-build] strategy: fail-fast: false matrix: ${{fromJson(needs.getTestMatrix.outputs.matrix)}} env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} WORKFLOW_PUBLIC_MANIFEST: '1' steps: - name: Checkout Repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Setup environment uses: ./.github/actions/setup-workflow-dev with: install-dependencies: ${{ !matrix.app.canary }} build-packages: 'false' - name: Setup canary if: ${{ matrix.app.canary }} env: APP_NAME: ${{ matrix.app.name }} run: | cat packages/next/package.json | jq '.dependencies.next|="16.3.1-canary.17"' > packages/next/package.json.new && mv packages/next/package.json.new packages/next/package.json cat "workbench/$APP_NAME/package.json" | jq '.dependencies.next|="16.3.1-canary.17"' > "workbench/$APP_NAME/package.json.new" && mv "workbench/$APP_NAME/package.json.new" "workbench/$APP_NAME/package.json" pnpm install --no-frozen-lockfile - name: Download shared package builds if: ${{ !matrix.app.canary }} uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: e2e-package-build-artifacts path: packages - name: Run Initial Build for Next.js canary if: ${{ matrix.app.canary }} run: pnpm turbo run build --filter='!./workbench/*' - name: Prepare workbench path id: prepare-workbench uses: ./.github/actions/prepare-workbench-path with: app-name: ${{ matrix.app.name }} - name: Run Build Tests run: pnpm vitest run packages/core/e2e/local-build.test.ts env: APP_NAME: ${{ matrix.app.name }} WORKBENCH_APP_PATH: ${{ steps.prepare-workbench.outputs.workbench_app_path }} # Asserts the shape of the manifest the step above just built. It needs # its own invocation rather than another file in that one: local-build # is what runs `pnpm build`, and vitest would run both files # concurrently, against a manifest that does not exist yet. - name: Run Manifest Tests run: pnpm vitest run packages/core/e2e/manifest.test.ts env: APP_NAME: ${{ matrix.app.name }} WORKBENCH_APP_PATH: ${{ steps.prepare-workbench.outputs.workbench_app_path }} - name: Run E2E Tests run: | export PROD_SERVER_LOG_PATH="$GITHUB_WORKSPACE/prod-server-${{ matrix.app.name }}-${{ matrix.app.artifactSuffix }}.log" # Queue auth stays off on this lane until the team decides to run # the Local lanes with it on. (The runner shares the app's data # directory, so its start()/resumeHook() deliveries carry the app's # key either way.) Off requires dynamic workflows off: the runtime # refuses WORKFLOW_QUEUE_AUTH=off under them. Dynamic workflows are # covered by e2e-local-dynamic, with queue auth on. (cd "$WORKBENCH_APP_PATH" && WORKFLOW_QUEUE_AUTH=off pnpm start 2>&1 | tee "$PROD_SERVER_LOG_PATH") & echo "starting tests in 10 seconds" && sleep 10 pnpm run test:e2e --reporter=verbose --reporter=json --reporter=./packages/core/e2e/github-reporter.ts --outputFile=e2e-local-prod-${{ matrix.app.name }}-${{ matrix.app.artifactSuffix }}.json env: NODE_OPTIONS: "--enable-source-maps" APP_NAME: ${{ matrix.app.name }} WORKBENCH_APP_PATH: ${{ steps.prepare-workbench.outputs.workbench_app_path }} DEPLOYMENT_URL: "http://localhost:${{ matrix.app.name == 'sveltekit' && '4173' || (matrix.app.name == 'astro' && '4321' || '3000') }}" NEXT_CANARY: ${{ matrix.app.canary && '1' || '' }} WORKFLOW_VM: ${{ matrix.app.vm || '' }} WORKFLOW_SNAPSHOT_THRESHOLD: ${{ matrix.app.snapshotThreshold || '' }} # Local worlds have no run encryption keys; snapshot legs opt in. WORKFLOW_SNAPSHOT_ALLOW_UNENCRYPTED: ${{ matrix.app.snapshotThreshold && '1' || '' }} # Snapshot legs log each restore so the check below can count them. DEBUG: ${{ matrix.app.snapshotThreshold && 'workflow:runtime:info' || '' }} - name: Check VM snapshots were restored if: ${{ matrix.app.snapshotThreshold }} run: | # A snapshot leg that never restores would still pass the suite # (every miss falls back to full replay), so require restores. restores=$(grep -c "QuickJS runtime: restored VM snapshot" "$GITHUB_WORKSPACE/prod-server-${{ matrix.app.name }}-${{ matrix.app.artifactSuffix }}.log" || true) echo "VM snapshot restores: $restores" test "$restores" -gt 0 - name: Generate E2E summary if: always() run: node .github/scripts/aggregate-e2e-results.js . --job-name "E2E Local Prod (${{ matrix.app.name }} - ${{ matrix.app.runLabel }})" >> $GITHUB_STEP_SUMMARY || true - name: Upload E2E results if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: e2e-results-local-prod-${{ matrix.app.name }}-${{ matrix.app.artifactSuffix }} path: | e2e-local-prod-${{ matrix.app.name }}-${{ matrix.app.artifactSuffix }}.json e2e-local-prod-${{ matrix.app.name }}-${{ matrix.app.artifactSuffix }}.flaky.json e2e-infra-${{ matrix.app.name }}-local.json retention-days: 7 if-no-files-found: ignore e2e-local-spec9: # The Local World counterpart of e2e-vercel-spec9: spec 9 runs (the single # orchestrator) are opt-in with WORKFLOW_SPEC_VERSION=9, so the other local # lanes run the default, spec 8. This lane builds the Next.js workbench, # serves it with the variable set, and runs the full suite against it on # both VMs. The variable is set on the server AND on the test runner (a # start() from the runner stamps the lower of the two), and the e2e test # "stamps the spec version the lane opts in to" fails if either side # stops seeing it. name: E2E Local Spec 9 Tests (nextjs-turbopack, ${{ matrix.vm }}) runs-on: ubuntu-latest timeout-minutes: 30 if: ${{ needs.ci-scope.outputs.fast-path != 'true' && !contains(github.event.pull_request.labels.*.name, 'workflow-server-test') }} needs: [ci-scope, e2e-package-build] strategy: fail-fast: false matrix: # Workflow VM engines: node:vm (default) and the opt-in QuickJS # WASM engine (WORKFLOW_VM=quickjs). vm: [node, quickjs] env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} WORKFLOW_PUBLIC_MANIFEST: '1' WORKFLOW_SPEC_VERSION: '9' APP_NAME: nextjs-turbopack steps: - name: Checkout Repo uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Setup environment uses: ./.github/actions/setup-workflow-dev with: build-packages: 'false' - name: Download shared package builds uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: e2e-package-build-artifacts path: packages - name: Prepare workbench path id: prepare-workbench uses: ./.github/actions/prepare-workbench-path with: app-name: nextjs-turbopack - name: Build workbench run: pnpm vitest run packages/core/e2e/local-build.test.ts env: WORKBENCH_APP_PATH: ${{ steps.prepare-workbench.outputs.workbench_app_path }} - name: Run E2E Tests run: | export PROD_SERVER_LOG_PATH="$GITHUB_WORKSPACE/prod-server-spec9-${{ matrix.vm }}.log" (cd "$WORKBENCH_APP_PATH" && WORKFLOW_EXPERIMENTAL_DYNAMIC_WORKFLOWS=1 pnpm start 2>&1 | tee "$PROD_SERVER_LOG_PATH") & echo "starting tests in 10 seconds" && sleep 10 pnpm run test:e2e --reporter=verbose --reporter=json --reporter=./packages/core/e2e/github-reporter.ts --outputFile=e2e-local-spec9-nextjs-turbopack-${{ matrix.vm }}.json env: NODE_OPTIONS: "--enable-source-maps" WORKBENCH_APP_PATH: ${{ steps.prepare-workbench.outputs.workbench_app_path }} DEPLOYMENT_URL: "http://localhost:3000" WORKFLOW_VM: ${{ matrix.vm }} # The server above opts in, so a dynamic E2E opt-in refusal fails. WORKFLOW_E2E_EXPECT_DYNAMIC_WORKFLOWS: "1" - name: Generate E2E summary if: always() run: node .github/scripts/aggregate-e2e-results.js . --job-name "E2E Local Spec 9 (nextjs-turbopack, ${{ matrix.vm }})" >> $GITHUB_STEP_SUMMARY || true - name: Upload E2E results if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: e2e-results-local-spec9-nextjs-turbopack-${{ matrix.vm }} path: | e2e-local-spec9-nextjs-turbopack-${{ matrix.vm }}.json e2e-local-spec9-nextjs-turbopack-${{ matrix.vm }}.flaky.json e2e-infra-nextjs-turbopack-local.json retention-days: 7 if-no-files-found: ignore - name: Upload server logs if: failure() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: prod-server-logs-local-spec9-${{ matrix.vm }} path: prod-server-spec9-${{ matrix.vm }}.log retention-days: 7 if-no-files-found: ignore # Dynamic workflows on a local World, with queue auth on. The local dev/prod # lanes run with dynamic workflows off and queue auth off until the team # decides to run them with auth on (see e2e-local-dev). Here the runner # starts each parent fixture through the app (`/api/e2e-dynamic-start`, # WORKFLOW_E2E_START_VIA_APP=1), so the app's own queue sends the first # delivery, as in a real app. e2e-local-dynamic: name: E2E Local Dynamic Workflows (nextjs-turbopack) runs-on: ubuntu-latest timeout-minutes: 20 if: ${{ needs.ci-scope.outputs.fast-path != 'true' && !contains(github.event.pull_request.labels.*.name, 'workflow-server-test') }} needs: [ci-scope, e2e-package-build] env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} WORKFLOW_PUBLIC_MANIFEST: '1' steps: - name: Checkout Repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Setup environment uses: ./.github/actions/setup-workflow-dev with: build-packages: 'false' - name: Download shared package builds uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: e2e-package-build-artifacts path: packages - name: Prepare workbench path id: prepare-workbench uses: ./.github/actions/prepare-workbench-path with: app-name: nextjs-turbopack - name: Run dynamic workflow E2E tests run: | export DEV_SERVER_LOG_PATH="$GITHUB_WORKSPACE/dev-server-nextjs-turbopack-dynamic.log" # Queue auth is left at its default (on). (cd "$WORKBENCH_APP_PATH" && WORKFLOW_EXPERIMENTAL_DYNAMIC_WORKFLOWS=1 pnpm dev 2>&1 | tee "$DEV_SERVER_LOG_PATH") & echo "starting tests in 10 seconds" && sleep 10 pnpm vitest run packages/core/e2e/e2e-dynamic-workflow.test.ts --reporter=verbose env: NODE_OPTIONS: "--enable-source-maps" APP_NAME: nextjs-turbopack WORKBENCH_APP_PATH: ${{ steps.prepare-workbench.outputs.workbench_app_path }} DEPLOYMENT_URL: "http://localhost:3000" # The server above opts in, so a dynamic E2E opt-in refusal fails. WORKFLOW_E2E_EXPECT_DYNAMIC_WORKFLOWS: "1" WORKFLOW_E2E_START_VIA_APP: "1" - name: Upload dev server log if: failure() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: e2e-local-dynamic-dev-server-log path: dev-server-nextjs-turbopack-dynamic.log retention-days: 7 if-no-files-found: ignore e2e-local-postgres: name: E2E Local Postgres Tests (${{ matrix.app.name }} - ${{ matrix.app.runLabel }}) runs-on: ubuntu-latest timeout-minutes: 30 if: ${{ needs.ci-scope.outputs.fast-path != 'true' && !contains(github.event.pull_request.labels.*.name, 'workflow-server-test') }} needs: [ci-scope, getTestMatrix, e2e-package-build] strategy: fail-fast: false matrix: ${{fromJson(needs.getTestMatrix.outputs.matrix)}} services: postgres: image: postgres:18-alpine env: POSTGRES_USER: world POSTGRES_PASSWORD: world POSTGRES_DB: world ports: - 5432:5432 options: >- --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} WORKFLOW_PUBLIC_MANIFEST: '1' WORKFLOW_TARGET_WORLD: "@workflow/world-postgres" WORKFLOW_POSTGRES_URL: "postgres://world:world@localhost:5432/world" steps: - name: Checkout Repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Setup environment uses: ./.github/actions/setup-workflow-dev with: install-dependencies: ${{ !matrix.app.canary }} build-packages: 'false' - name: Setup canary if: ${{ matrix.app.canary }} env: APP_NAME: ${{ matrix.app.name }} run: | cat packages/next/package.json | jq '.dependencies.next|="16.3.1-canary.17"' > packages/next/package.json.new && mv packages/next/package.json.new packages/next/package.json cat "workbench/$APP_NAME/package.json" | jq '.dependencies.next|="16.3.1-canary.17"' > "workbench/$APP_NAME/package.json.new" && mv "workbench/$APP_NAME/package.json.new" "workbench/$APP_NAME/package.json" pnpm install --no-frozen-lockfile - name: Download shared package builds if: ${{ !matrix.app.canary }} uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: e2e-package-build-artifacts path: packages - name: Run Initial Build for Next.js canary if: ${{ matrix.app.canary }} run: pnpm turbo run build --filter='!./workbench/*' - name: Setup PostgreSQL Database run: ./packages/world-postgres/bin/setup.js - name: Prepare workbench path id: prepare-workbench uses: ./.github/actions/prepare-workbench-path with: app-name: ${{ matrix.app.name }} - name: Run Build Tests run: pnpm vitest run packages/core/e2e/local-build.test.ts env: APP_NAME: ${{ matrix.app.name }} WORKBENCH_APP_PATH: ${{ steps.prepare-workbench.outputs.workbench_app_path }} - name: Run E2E Tests run: | export PROD_SERVER_LOG_PATH="$GITHUB_WORKSPACE/prod-server-postgres-${{ matrix.app.name }}-${{ matrix.app.artifactSuffix }}.log" (cd "$WORKBENCH_APP_PATH" && WORKFLOW_EXPERIMENTAL_DYNAMIC_WORKFLOWS=1 pnpm start 2>&1 | tee "$PROD_SERVER_LOG_PATH") & echo "starting tests in 10 seconds" && sleep 10 pnpm run test:e2e --reporter=verbose --reporter=json --reporter=./packages/core/e2e/github-reporter.ts --outputFile=e2e-local-postgres-${{ matrix.app.name }}-${{ matrix.app.artifactSuffix }}.json env: NODE_OPTIONS: "--enable-source-maps" APP_NAME: ${{ matrix.app.name }} WORKBENCH_APP_PATH: ${{ steps.prepare-workbench.outputs.workbench_app_path }} DEPLOYMENT_URL: "http://localhost:${{ matrix.app.name == 'sveltekit' && '4173' || (matrix.app.name == 'astro' && '4321' || '3000') }}" NEXT_CANARY: ${{ matrix.app.canary && '1' || '' }} WORKFLOW_VM: ${{ matrix.app.vm || '' }} # The server above opts in, so a dynamic E2E opt-in refusal fails. WORKFLOW_E2E_EXPECT_DYNAMIC_WORKFLOWS: "1" WORKFLOW_SNAPSHOT_THRESHOLD: ${{ matrix.app.snapshotThreshold || '' }} # Local worlds have no run encryption keys; snapshot legs opt in. WORKFLOW_SNAPSHOT_ALLOW_UNENCRYPTED: ${{ matrix.app.snapshotThreshold && '1' || '' }} # Snapshot legs log each restore so the check below can count them. DEBUG: ${{ matrix.app.snapshotThreshold && 'workflow:runtime:info' || '' }} - name: Check VM snapshots were restored if: ${{ matrix.app.snapshotThreshold }} run: | # A snapshot leg that never restores would still pass the suite # (every miss falls back to full replay), so require restores. restores=$(grep -c "QuickJS runtime: restored VM snapshot" "$GITHUB_WORKSPACE/prod-server-postgres-${{ matrix.app.name }}-${{ matrix.app.artifactSuffix }}.log" || true) echo "VM snapshot restores: $restores" test "$restores" -gt 0 # The only CI run of the start({ hook }) block on the single-orchestrator # engine: the spec 9 Vercel lane skips it until world-vercel's start Hook # flag is on there. Spec 9 is a runtime choice (WORKFLOW_SPEC_VERSION is # read when the World is created), so the built app is restarted with it # rather than rebuilt; the spec 8 server is stopped first so its # in-process queue workers don't pick the spec 9 runs up. - name: Run start({ hook }) E2E at spec 9 if: ${{ matrix.app.name == 'nextjs-turbopack' && !matrix.app.canary && !matrix.app.snapshotThreshold }} run: | # Stop the spec 8 server by its port: Next.js renames its server # process, so its command line no longer says `next start`. listening() { lsof -ti tcp:3000 -sTCP:LISTEN; } pids=$(listening || true) if [ -n "$pids" ]; then echo "$pids" | xargs kill || true; fi for _ in $(seq 1 30); do listening >/dev/null || break; sleep 1; done if listening >/dev/null; then echo "port 3000 is still in use"; exit 1; fi (cd "$WORKBENCH_APP_PATH" && pnpm start > "$GITHUB_WORKSPACE/prod-server-postgres-spec9-${{ matrix.app.artifactSuffix }}.log" 2>&1) & for _ in $(seq 1 60); do curl -s -o /dev/null http://localhost:3000/ && break; sleep 1; done pnpm vitest run packages/core/e2e/e2e.test.ts -t 'start\(\{ hook \}\)' --reporter=verbose --reporter=json --reporter=./packages/core/e2e/github-reporter.ts --outputFile=e2e-local-postgres-spec9-${{ matrix.app.name }}-${{ matrix.app.artifactSuffix }}.json env: NODE_OPTIONS: "--enable-source-maps" APP_NAME: ${{ matrix.app.name }} WORKBENCH_APP_PATH: ${{ steps.prepare-workbench.outputs.workbench_app_path }} DEPLOYMENT_URL: "http://localhost:3000" WORKFLOW_VM: ${{ matrix.app.vm || '' }} WORKFLOW_SPEC_VERSION: '9' # Every test of the block must run here, not skip. WORKFLOW_E2E_EXPECT_START_HOOK: '1' - name: Generate E2E summary if: always() run: node .github/scripts/aggregate-e2e-results.js . --job-name "E2E Local Postgres (${{ matrix.app.name }} - ${{ matrix.app.runLabel }})" >> $GITHUB_STEP_SUMMARY || true - name: Upload E2E results if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: e2e-results-local-postgres-${{ matrix.app.name }}-${{ matrix.app.artifactSuffix }} path: | e2e-local-postgres-${{ matrix.app.name }}-${{ matrix.app.artifactSuffix }}.json e2e-local-postgres-${{ matrix.app.name }}-${{ matrix.app.artifactSuffix }}.flaky.json e2e-local-postgres-spec9-${{ matrix.app.name }}-${{ matrix.app.artifactSuffix }}.json e2e-infra-${{ matrix.app.name }}-local.json retention-days: 7 if-no-files-found: ignore # @workflow/world-cloudflare: each run is a Durable Object. The suite builds # the package's e2e Worker (packages/world-cloudflare/e2e) against the built # package and runs it under `wrangler dev` (local workerd with local Durable # Object storage); nothing is deployed. It covers steps, fan-out, sleep, # Promise.race, retries, streams, hooks and cross-run token conflicts, VM # snapshots, and crash recovery by killing workerd mid-run. e2e-cloudflare: name: E2E Cloudflare (workerd) runs-on: ubuntu-latest timeout-minutes: 20 if: ${{ needs.ci-scope.outputs.fast-path != 'true' && !contains(github.event.pull_request.labels.*.name, 'workflow-server-test') }} needs: [ci-scope, e2e-package-build] env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} steps: - name: Checkout Repo uses: actions/checkout@v6 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Setup environment uses: ./.github/actions/setup-workflow-dev with: build-packages: 'false' - name: Download shared package builds uses: actions/download-artifact@v8 with: name: e2e-package-build-artifacts path: packages # Runs packages/world-cloudflare/e2e/test/e2e.test.ts. - name: Run Cloudflare World e2e tests run: pnpm --filter @workflow/world-cloudflare test:e2e # Cross-language conformance: the same TypeScript e2e suite, driven against the # Python workbench app instead of a JavaScript one. Deliberately its own job # rather than a `getTestMatrix` row — every step in the two local lanes is JS # toolchain (`pnpm dev` / `pnpm start`, the canary `next` patch, `dev.test.ts` # and `local-build.test.ts`, and the per-framework `DEV_TEST_CONFIG` of # compiler-generated paths), and none of it has a Python counterpart. The # `e2e-vercel-prod` lane, whose rows only need a URL, does carry `python` as an # ordinary matrix row. # # This lane is advisory while the Python runtime catches up with the current # protocol. A test that stops skipping correctly, or a fixture declared in # `e2e-conformance.json` that the app stops registering, still fails visibly # here and nowhere else. e2e-python: name: E2E Python Conformance runs-on: ubuntu-latest timeout-minutes: 20 if: ${{ needs.ci-scope.outputs.fast-path != 'true' && !contains(github.event.pull_request.labels.*.name, 'workflow-server-test') }} needs: [ci-scope, e2e-package-build] env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} WORKFLOW_PUBLIC_MANIFEST: '1' steps: - name: Checkout Repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Setup environment uses: ./.github/actions/setup-workflow-dev with: build-packages: 'false' # The Python app needs nothing from `packages/`, but the driver does: the # vitest suite imports `@workflow/core` and shells out to the `workflow` # CLI to inspect runs. - name: Download shared package builds uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: e2e-package-build-artifacts path: packages # Pinned rather than floating to latest: `uv sync --locked` below is only # a meaningful gate if the tool reading the lock is a known quantity. # Pinning here rather than via `[tool.uv] required-version` is deliberate # — that key would also bind the uv that `@vercel/python` bundles on the # build image, which we do not control. - name: Setup uv uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 with: version: '0.13.5' enable-cache: true cache-dependency-glob: workbench/python/uv.lock # `--locked` rather than `--frozen`: a `pyproject.toml` edit that was never # relocked should fail the job, not be silently ignored. - name: Install Python dependencies working-directory: workbench/python run: uv sync --locked - name: Run E2E Tests run: | APP_LOG="$GITHUB_WORKSPACE/python-app.log" rm -f "$APP_LOG" (cd workbench/python && uv run uvicorn app:app --port 3000 > "$APP_LOG" 2>&1) & for _ in $(seq 1 30); do if curl -sf http://localhost:3000/.well-known/workflow/v1/manifest.json -o /dev/null; then echo "app is up" break fi sleep 1 done curl -sf http://localhost:3000/.well-known/workflow/v1/manifest.json | head -c 2000 echo # Only `e2e.test.ts` — `e2e-agent.test.ts` (which `pnpm test:e2e` also # runs) has no conformance gate and needs routes this app does not serve. pnpm vitest run packages/core/e2e/e2e.test.ts \ --reporter=verbose --reporter=json \ --reporter=./packages/core/e2e/github-reporter.ts \ --outputFile=e2e-conformance-python.json env: NODE_OPTIONS: "--enable-source-maps" APP_NAME: "python" WORKBENCH_APP_PATH: ${{ github.workspace }}/workbench/python DEPLOYMENT_URL: "http://localhost:3000" - name: Print Python app logs if: always() run: cat "$GITHUB_WORKSPACE/python-app.log" || true - name: Generate E2E summary if: always() run: node .github/scripts/aggregate-e2e-results.js . --job-name "E2E Python Conformance" >> $GITHUB_STEP_SUMMARY || true - name: Upload E2E results if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: e2e-results-conformance-python path: | e2e-conformance-python.json e2e-conformance-python.flaky.json e2e-infra-python-local.json e2e-diagnostics-python-local.json retention-days: 7 if-no-files-found: ignore e2e-windows: name: E2E Windows Tests (${{ matrix.vm }}) runs-on: windows-latest timeout-minutes: 30 needs: ci-scope if: ${{ needs.ci-scope.outputs.fast-path != 'true' && !contains(github.event.pull_request.labels.*.name, 'workflow-server-test') }} strategy: fail-fast: false matrix: # Workflow VM engines: node:vm (default) and the opt-in QuickJS # WASM engine (WORKFLOW_VM=quickjs). vm: [node, quickjs] env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} WORKFLOW_PUBLIC_MANIFEST: '1' steps: - name: Checkout Repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Setup Rust uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 with: toolchain: stable target: wasm32-unknown-unknown - name: Setup pnpm uses: pnpm/setup@fa36a5ef7dcdefc3a7cb25655c122e00cf6e636e # v3.1.2 with: install: true require-lockfile: true runtime: node@24 cache: true - name: Run Initial Build run: pnpm turbo run build --filter='!./workbench/*' --filter='!./docs' - name: Run E2E Tests (Next.js) run: | cd workbench/nextjs-turbopack $logFile = "$env:GITHUB_WORKSPACE/nextjs-server.log" $env:DEV_SERVER_LOG_PATH = $logFile # `$using:` only resolves PowerShell variables, not env vars, so # copy MATRIX_VM into a session variable before Start-Job. $matrixVm = $env:MATRIX_VM # Queue auth stays off on this lane until the team decides to run # the Local lanes with it on (dynamic workflows stay off, as the # runtime requires for that). $job = Start-Job -ScriptBlock { Set-Location $using:PWD; $env:WORKFLOW_VM = $using:matrixVm; $env:WORKFLOW_QUEUE_AUTH = 'off'; pnpm dev *>&1 | Tee-Object -FilePath $using:logFile } Start-Sleep -Seconds 15 cd ../.. # Fail-fast guards: if the Next.js dev server gets into a corrupted # state during HMR (a recurring Turbopack-on-Windows issue where # `packages/core/dist/*.js` becomes "MODULE_UNPARSABLE" and every # subsequent request returns 500), the e2e suite would otherwise burn # the full 30-minute job window polling stuck workflow runs before # finally being cancelled. Below we (1) bail out if `dev.test.ts` # fails, since that's the first thing to start hitting 500s, and # (2) probe the server before the long e2e run to catch any other # silent breakage. # # Get-DevServerStatus returns the HTTP status code of the workflow # flow route's health endpoint, or 0 if the request never produced a # response. A healthy dev server answers `?__health` with 200; we # only treat 5xx (and no-response) as unhealthy. # # Probe the flow route rather than /api/chat: the flow route is what # every workflow dispatch hits, and it is the route that breaks. Its # generated `__step_registrations.js` imports each discovered step # file by path, so a step file the dev server failed to forget after # a `dev.test.ts` fixture was deleted leaves it importing a path that # no longer exists. /api/chat compiles independently and keeps # answering 405 while every dispatch gets a 500. function Get-DevServerStatus { try { $resp = Invoke-WebRequest -Uri "http://localhost:3000/.well-known/workflow/v1/flow?__health" -Method POST -UseBasicParsing -TimeoutSec 15 -ErrorAction Stop return [int]$resp.StatusCode } catch { if ($_.Exception.Response) { try { return [int]$_.Exception.Response.StatusCode } catch { return 0 } } return 0 } } $devExit = 0 pnpm vitest run packages/core/e2e/dev.test.ts $devExit = $LASTEXITCODE if ($devExit -ne 0) { Write-Host "::warning title=dev.test.ts failed::dev.test.ts exited with code $devExit. Skipping the remainder of the e2e suite to avoid waiting for the 30-minute job timeout." Stop-Job $job -ErrorAction SilentlyContinue exit $devExit } # Sanity-check the server before kicking off the long e2e suite. This # catches the case where Turbopack quietly went sideways (e.g. lost # track of files in `packages/core/dist` after HMR on Windows) and is # returning 500 for every request. Without this check the e2e suite # would burn the full 30-minute job window polling stuck workflow # runs before getting cancelled. $status = Get-DevServerStatus Write-Host "[health-check:pre-e2e] POST /.well-known/workflow/v1/flow?__health -> $status" if ($status -eq 0 -or $status -ge 500) { Write-Host "::error title=Next.js dev server unhealthy::Dev server returned $status before e2e tests. Aborting to avoid the 30-minute job timeout. See the 'Print Next.js server logs' step for the underlying Turbopack error." Stop-Job $job -ErrorAction SilentlyContinue exit 1 } pnpm run test:e2e --reporter=verbose --reporter=json --reporter=./packages/core/e2e/github-reporter.ts --outputFile=e2e-windows-nextjs-turbopack-$env:MATRIX_VM.json $e2eExit = $LASTEXITCODE Stop-Job $job -ErrorAction SilentlyContinue exit $e2eExit shell: powershell env: NODE_OPTIONS: "--enable-source-maps" APP_NAME: "nextjs-turbopack" DEPLOYMENT_URL: "http://localhost:3000" # Use a real workflow file for the Windows HMR test. Most of this # workbench's workflows are symlinks into workbench/example, and # Windows/Turbopack can rebuild from the symlink path while still # serving the previous module contents from the realpath cache. DEV_TEST_CONFIG: '{"generatedStepRegistrationPath":"app/.well-known/workflow/v1/flow/__step_registrations.js","generatedWorkflowPath":"app/.well-known/workflow/v1/flow/route.js","apiFilePath":"app/api/chat/route.ts","apiFileImportPath":"../../..","port":3000,"testWorkflowFile":"96_many_steps.ts"}' DEV_SERVER_LOG_PATH: "${{ github.workspace }}/nextjs-server.log" WORKFLOW_DEV_HMR_LOGS: "1" WORKFLOW_VM: ${{ matrix.vm }} MATRIX_VM: ${{ matrix.vm }} - name: Print Next.js server logs if: always() shell: powershell run: | $logFile = "$env:GITHUB_WORKSPACE/nextjs-server.log" if (Test-Path $logFile) { Write-Host "=== Next.js Server Logs ===" -ForegroundColor Cyan Get-Content $logFile Write-Host "=== End of Server Logs ===" -ForegroundColor Cyan } else { Write-Host "No server log file found at $logFile" -ForegroundColor Yellow } - name: Generate E2E summary if: always() shell: bash env: MATRIX_VM: ${{ matrix.vm }} run: node .github/scripts/aggregate-e2e-results.js . --job-name "E2E Windows (nextjs-turbopack - $MATRIX_VM)" >> $GITHUB_STEP_SUMMARY || true - name: Upload E2E results if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: e2e-results-windows-nextjs-turbopack-${{ matrix.vm }} path: | e2e-windows-nextjs-turbopack-${{ matrix.vm }}.json e2e-windows-nextjs-turbopack-${{ matrix.vm }}.flaky.json e2e-infra-nextjs-turbopack-local.json retention-days: 7 if-no-files-found: ignore - name: Upload Next.js server logs if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: nextjs-server-logs-windows-${{ matrix.vm }} path: nextjs-server.log retention-days: 7 if-no-files-found: ignore # Community World E2E Tests (dynamically generated from worlds-manifest.json) # Disabled on main: community worlds target spec v2 and do not yet support the # CBOR queue transport used by world-vercel (see #1627 / #1658). They continue # to run on `stable`, where spec v2 is still the primary compat target. Once # the community worlds (mizzle-dev/workflow-worlds) ship CBOR support we can # re-enable these jobs here. getCommunityWorldsMatrix: name: Get Community Worlds Matrix runs-on: ubuntu-latest if: false outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Setup environment uses: ./.github/actions/setup-workflow-dev with: install-dependencies: 'false' build-packages: 'false' cache-pnpm: 'false' - id: set-matrix run: | MATRIX_JSON="$(node ./scripts/create-community-worlds-matrix.mjs | jq -c .)" echo "matrix=$MATRIX_JSON" >> "$GITHUB_OUTPUT" e2e-community: name: E2E Community World (${{ matrix.world.name }}) if: false needs: getCommunityWorldsMatrix strategy: fail-fast: false matrix: ${{ fromJson(needs.getCommunityWorldsMatrix.outputs.matrix) }} uses: ./.github/workflows/e2e-community-world.yml with: world-id: ${{ matrix.world.id }} world-name: ${{ matrix.world.name }} world-package: ${{ matrix.world.package }} package-version: ${{ matrix.world.version }} service-type: ${{ matrix.world.service-type }} env-vars: ${{ matrix.world.env-vars }} services: ${{ matrix.world.services }} secrets: inherit # Final job: Aggregate all E2E results and update PR comment summary: name: E2E Summary runs-on: ubuntu-latest needs: [ci-scope, e2e-vercel-prod, e2e-vercel-multi-region, e2e-vercel-ws-transport, e2e-vercel-http-transport, e2e-vercel-spec9, e2e-local-dev, e2e-local-prod, e2e-local-spec9, e2e-local-postgres, e2e-python, e2e-windows] if: always() && !cancelled() && needs.ci-scope.outputs.fast-path != 'true' timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Download all E2E artifacts uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: e2e-results-* path: e2e-results merge-multiple: true - name: List downloaded files run: find e2e-results -type f -name "*.json" | sort || echo "No files found" - name: Aggregate E2E results id: aggregate run: | node .github/scripts/aggregate-e2e-results.js e2e-results --mode aggregate --run-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | tee e2e-summary.md >> $GITHUB_STEP_SUMMARY - name: Update PR comment with results if: github.event_name == 'pull_request' uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 with: header: e2e-test-results path: e2e-summary.md # Final required check: passes only when unit + required E2E jobs succeed. # Python conformance is advisory, and community worlds are disabled on main. e2e-required-check: name: E2E Required Check runs-on: ubuntu-latest needs: [ci-scope, unit, e2e-package-build, e2e-vercel-prod, e2e-vercel-ws-transport, e2e-vercel-http-transport, e2e-vercel-spec9, e2e-local-dev, e2e-local-prod, e2e-local-spec9, e2e-local-dynamic, e2e-local-postgres, e2e-windows, e2e-cloudflare] if: always() timeout-minutes: 5 steps: - name: Validate E2E job statuses env: UNIT_STATUS: ${{ needs.unit.result }} BUILD_STATUS: ${{ needs.e2e-package-build.result }} VERCEL_STATUS: ${{ needs.e2e-vercel-prod.result }} VERCEL_WS_STATUS: ${{ needs.e2e-vercel-ws-transport.result }} VERCEL_HTTP_STATUS: ${{ needs.e2e-vercel-http-transport.result }} VERCEL_SPEC9_STATUS: ${{ needs.e2e-vercel-spec9.result }} LOCAL_DEV_STATUS: ${{ needs.e2e-local-dev.result }} LOCAL_PROD_STATUS: ${{ needs.e2e-local-prod.result }} LOCAL_SPEC9_STATUS: ${{ needs.e2e-local-spec9.result }} LOCAL_DYNAMIC_STATUS: ${{ needs.e2e-local-dynamic.result }} POSTGRES_STATUS: ${{ needs.e2e-local-postgres.result }} WINDOWS_STATUS: ${{ needs.e2e-windows.result }} CLOUDFLARE_STATUS: ${{ needs.e2e-cloudflare.result }} FAST_PATH: ${{ needs.ci-scope.outputs.fast-path }} VALIDATION_FAST_PATH: ${{ needs.ci-scope.outputs.validation-fast-path }} HAS_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'workflow-server-test') }} run: | FAILED_JOBS=() if [[ "$FAST_PATH" == "true" ]]; then if [[ "$VALIDATION_FAST_PATH" != "true" && "$HAS_LABEL" != "true" && "$UNIT_STATUS" != "success" ]]; then echo "UI-only fast path still requires unit tests; unit status was $UNIT_STATUS." exit 1 fi echo "Docs/UI-only change detected - no runtime E2E lanes are required." exit 0 fi # When workflow-server-test label is present, only check vercel-prod (others should be skipped) # Otherwise, all jobs must succeed if [[ "$HAS_LABEL" == "true" ]]; then echo "workflow-server-test label detected - only checking required jobs" [[ "$VERCEL_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-vercel-prod ($VERCEL_STATUS)") # This label is itself opt-in for the WS lane (see that job's `if`), # so it runs here and is required. [[ "$VERCEL_WS_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-vercel-ws-transport ($VERCEL_WS_STATUS)") # Unconditional, exactly like e2e-vercel-prod above: the HTTP # lane has no opt-in label, so there is no case in which it is # legitimately skipped while the rest of this branch runs. [[ "$VERCEL_HTTP_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-vercel-http-transport ($VERCEL_HTTP_STATUS)") # Verify other jobs were actually skipped [[ "$UNIT_STATUS" == "skipped" ]] || echo "Warning: unit was not skipped ($UNIT_STATUS)" [[ "$LOCAL_DEV_STATUS" == "skipped" ]] || echo "Warning: e2e-local-dev was not skipped ($LOCAL_DEV_STATUS)" [[ "$LOCAL_PROD_STATUS" == "skipped" ]] || echo "Warning: e2e-local-prod was not skipped ($LOCAL_PROD_STATUS)" [[ "$LOCAL_SPEC9_STATUS" == "skipped" ]] || echo "Warning: e2e-local-spec9 was not skipped ($LOCAL_SPEC9_STATUS)" [[ "$LOCAL_DYNAMIC_STATUS" == "skipped" ]] || echo "Warning: e2e-local-dynamic was not skipped ($LOCAL_DYNAMIC_STATUS)" [[ "$POSTGRES_STATUS" == "skipped" ]] || echo "Warning: e2e-local-postgres was not skipped ($POSTGRES_STATUS)" [[ "$WINDOWS_STATUS" == "skipped" ]] || echo "Warning: e2e-windows was not skipped ($WINDOWS_STATUS)" [[ "$CLOUDFLARE_STATUS" == "skipped" ]] || echo "Warning: e2e-cloudflare was not skipped ($CLOUDFLARE_STATUS)" else echo "Standard PR - checking all required jobs" [[ "$UNIT_STATUS" == "success" ]] || FAILED_JOBS+=("unit ($UNIT_STATUS)") [[ "$BUILD_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-package-build ($BUILD_STATUS)") [[ "$VERCEL_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-vercel-prod ($VERCEL_STATUS)") [[ "$VERCEL_WS_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-vercel-ws-transport ($VERCEL_WS_STATUS)") [[ "$VERCEL_HTTP_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-vercel-http-transport ($VERCEL_HTTP_STATUS)") [[ "$VERCEL_SPEC9_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-vercel-spec9 ($VERCEL_SPEC9_STATUS)") [[ "$LOCAL_DEV_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-local-dev ($LOCAL_DEV_STATUS)") [[ "$LOCAL_PROD_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-local-prod ($LOCAL_PROD_STATUS)") [[ "$LOCAL_SPEC9_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-local-spec9 ($LOCAL_SPEC9_STATUS)") [[ "$LOCAL_DYNAMIC_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-local-dynamic ($LOCAL_DYNAMIC_STATUS)") [[ "$POSTGRES_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-local-postgres ($POSTGRES_STATUS)") [[ "$WINDOWS_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-windows ($WINDOWS_STATUS)") [[ "$CLOUDFLARE_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-cloudflare ($CLOUDFLARE_STATUS)") fi if [[ ${#FAILED_JOBS[@]} -gt 0 ]]; then echo "The following E2E jobs did not succeed:" printf '%s\n' "${FAILED_JOBS[@]}" exit 1 fi echo "All required E2E jobs succeeded." # Publish E2E results to GitHub Pages (main branch only) publish-results: name: Publish E2E Results runs-on: ubuntu-latest needs: [summary, e2e-vercel-multi-region] # Use always() because summary uses always() - without it this job gets skipped when any E2E test fails if: always() && !cancelled() && github.event_name == 'push' && github.ref == 'refs/heads/main' timeout-minutes: 5 permissions: actions: read contents: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Download all E2E artifacts uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: e2e-results-* path: e2e-results merge-multiple: true - name: Generate docs data run: | mkdir -p docs-data node .github/scripts/generate-docs-data.js \ --type e2e \ --results-dir e2e-results \ --output docs-data/e2e-results.json \ --commit "${{ github.sha }}" \ --branch "${{ github.ref_name }}" # Main Tests runs are serialized by the workflow concurrency group, so # reading the previous file and publishing its successor cannot lose a # concurrent main-branch update. - name: Fetch previous E2E flake history id: previous-flake-history continue-on-error: true env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | if gh api --header 'Accept: application/vnd.github.raw+json' \ "repos/${{ github.repository }}/contents/ci/e2e-flake-history.json?ref=gh-pages" \ > previous-e2e-flake-history.json 2>history-fetch-error.log; then test -s previous-e2e-flake-history.json elif grep -q 'HTTP 404' history-fetch-error.log; then : > previous-e2e-flake-history.json else cat history-fetch-error.log >&2 exit 1 fi RUN_STARTED_AT=$(gh api "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}" --jq .run_started_at) echo "run-started-at=$RUN_STARTED_AT" >> "$GITHUB_OUTPUT" - name: Generate E2E flake history if: steps.previous-flake-history.outcome == 'success' continue-on-error: true run: | node .github/scripts/generate-e2e-flake-history.js \ --results-dir e2e-results \ --previous previous-e2e-flake-history.json \ --output docs-data/e2e-flake-history.json \ --run-id "${{ github.run_id }}" \ --attempt "${{ github.run_attempt }}" \ --sha "${{ github.sha }}" \ --started-at "${{ steps.previous-flake-history.outputs.run-started-at }}" \ --run-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - name: Deploy to GitHub Pages uses: ./.github/actions/publish-to-gh-pages with: github-token: ${{ secrets.GITHUB_TOKEN }} source-dir: ./docs-data destination-dir: ci