name: Action Test # Exercises packages/executable-stories-action against the formatter binary built # from THIS commit (via formatter-binary), not a published release. This is the # guard that keeps the action and the CLI in sync: if a CLI change removes or # renames a subcommand/flag the action relies on (format, review, gate-release, # deploy), this workflow goes red instead of the break surfacing in production. # Runs on pull requests when the action or the CLI it drives changes. Steps that # reach the comment stage set post-comment: false, so this workflow tests # detect, format and artifact and stays out of the PR conversation. on: pull_request: branches: - main paths: - "packages/executable-stories-action/**" - "packages/executable-stories-formatters/**" - ".github/workflows/action-test.yml" workflow_dispatch: permissions: contents: read jobs: build-binary: name: Build formatter binary runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - uses: actions/setup-node@v6 with: node-version: 22 - name: Enable Corepack run: corepack enable - name: Install pnpm run: corepack prepare pnpm@10.28.1 --activate - name: Install dependencies run: pnpm install --frozen-lockfile - name: Build packages run: pnpm build - name: Setup Bun uses: oven-sh/setup-bun@v2 with: bun-version: "1.2.0" - name: Compile formatter binary run: cd packages/executable-stories-formatters && pnpm run compile - name: Upload binary uses: actions/upload-artifact@v7 with: name: action-test-binary path: packages/executable-stories-formatters/executable-stories retention-days: 1 comment-builder: name: "unit: PR comment, job summary, annotations, assets" runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - uses: actions/setup-node@v6 with: node-version: 22 - name: Test the comment builder and the asset collector run: | node --test \ packages/executable-stories-action/test/asset-refs.test.mjs \ packages/executable-stories-action/test/build-comment.test.mjs \ packages/executable-stories-action/test/collect-assets.test.mjs \ packages/executable-stories-action/test/own-comment.test.mjs \ packages/executable-stories-action/test/ref-retry.test.mjs raw-json: name: "report: format raw run JSON with the built binary" needs: build-binary runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - uses: actions/download-artifact@v7 with: name: action-test-binary path: bin - name: Stage raw run fixture run: | mkdir -p .executable-stories cp packages/executable-stories-action/test/fixtures/raw-run/raw-run.json .executable-stories/raw-run.json - name: Run action (raw JSON → report) uses: ./packages/executable-stories-action with: post-comment: "false" report-dir: reports output-name: test-results formatter-binary: bin/executable-stories - name: Assert reports were generated run: | test -f reports/test-results.html || { echo "missing reports/test-results.html"; exit 1; } test -f reports/test-results.md || { echo "missing reports/test-results.md"; exit 1; } echo "Action produced HTML + Markdown from raw-run.json" review: name: "review: evidence review produces findings the comment can render" needs: build-binary runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - uses: actions/download-artifact@v7 with: name: action-test-binary path: bin - name: Stage a run and a diff run: | mkdir -p .executable-stories cp packages/executable-stories-action/test/fixtures/raw-run/raw-run.json .executable-stories/raw-run.json # JSON, not "git diff --name-status" text: that format is tab-separated, # and a shell quoting slip turns the tab into a literal backslash-t that # the parser silently drops, leaving a green job that asserted nothing. cat > changed-files.json <<'JSON' [ { "path": "src/cart/checkout.ts", "changeKind": "modified" }, { "path": "src/cart/discount.ts", "changeKind": "added" } ] JSON - name: Run action (review) uses: ./packages/executable-stories-action with: post-comment: "false" mode: review report-dir: reports output-name: test-results changed-files: changed-files.json formatter-binary: bin/executable-stories - name: Assert the machine contract the PR comment renders from run: | test -f reports/test-results.review.json || { echo "missing reports/test-results.review.json"; exit 1; } node -e ' const r = require("./reports/test-results.review.json"); if (r.version !== 1) throw new Error("unexpected contract version: " + r.version); if (!Array.isArray(r.findings) || r.findings.length === 0) { throw new Error("expected findings for two uncovered files"); } const uncovered = r.findings.filter((f) => f.kind === "uncovered"); if (uncovered.length !== 2) { throw new Error("expected 2 uncovered findings, got " + uncovered.length); } for (const f of r.findings) { for (const field of ["severity", "title", "file", "detail", "remedy"]) { if (!f[field]) throw new Error("finding missing " + field); } } console.log("Review contract holds: " + r.findings.length + " finding(s)"); ' pre-generated: name: "report: pass through pre-generated reports" runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - name: Stage pre-generated fixtures run: cp -r packages/executable-stories-action/test/fixtures/reports reports - name: Run action (pre-generated) uses: ./packages/executable-stories-action with: post-comment: "false" report-dir: reports output-name: test-results stale-state: name: "report: a second invocation ignores the first's leftovers" runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - name: Stage a report referencing a bundled screenshot run: | mkdir -p reports/assets printf 'iVBORw0KGgo=' | base64 -d > reports/assets/dashboard.png cat > reports/test-results.md <<'MD' # User Stories ![Dashboard](assets/dashboard.png) MD cp packages/executable-stories-action/test/fixtures/reports/test-results.html reports/test-results.html # What an earlier invocation of the action leaves in RUNNER_TEMP when it # hosted its own suite's media. Both suites reference `assets/…` by # convention, so the paths collide and the swap would be silent. - name: Leave a previous invocation's asset URLs and comment behind run: | echo '{"assets/dashboard.png":"https://raw.example/PREVIOUS-SUITE.png"}' \ > "$RUNNER_TEMP/executable-stories-asset-urls.json" echo 'PREVIOUS SUITE COMMENT' > "$RUNNER_TEMP/executable-stories-comment.md" - name: Run action (host-images off, so it hosts nothing of its own) uses: ./packages/executable-stories-action with: post-comment: "false" report-dir: reports output-name: test-results - name: Assert nothing from the previous invocation survived run: | body="$RUNNER_TEMP/executable-stories-comment.md" if grep -q 'PREVIOUS-SUITE.png' "$body"; then echo "::error::This report is showing the previous invocation's screenshot" exit 1 fi if grep -q 'PREVIOUS SUITE COMMENT' "$body"; then echo "::error::The previous invocation's comment body was not replaced" exit 1 fi # Not hosted, so it must be labelled rather than left as a dead path. grep -q '📎 Dashboard (see HTML report)' "$body" \ || { echo "::error::expected the screenshot to be labelled"; cat "$body"; exit 1; } echo "Second invocation is independent of the first" publish-run-missing: name: "publish-run: fail when no run JSON exists" runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - name: Run action (should fail before touching the API) id: action continue-on-error: true uses: ./packages/executable-stories-action with: mode: publish-run - name: Verify failure run: | if [[ "${{ steps.action.outcome }}" != "failure" ]]; then echo "Expected publish-run to fail when no run JSON exists" exit 1 fi echo "Action correctly failed" publish-run-invalid: name: "publish-run: reject structurally invalid run JSON" runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - name: Stage invalid run fixture run: | mkdir -p reports printf '%s\n' '{"testCases":[null]}' > reports/raw-run.json - name: Run action (should fail before touching the API) id: action continue-on-error: true uses: ./packages/executable-stories-action with: mode: publish-run run-json: reports/raw-run.json - name: Verify failure run: | if [[ "${{ steps.action.outcome }}" != "failure" ]]; then echo "Expected publish-run to reject structurally invalid run JSON" exit 1 fi echo "Action correctly rejected invalid run JSON" missing-files: name: "report: fail when no input exists" runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - name: Run action (should fail) id: action continue-on-error: true uses: ./packages/executable-stories-action - name: Verify failure run: | if [[ "${{ steps.action.outcome }}" != "failure" ]]; then echo "Expected the action to fail when no reports or raw JSON exist" exit 1 fi echo "Action correctly failed"