import { expect } from "@playwright/test"; import { Comment } from "../apps/backend/src/database/models"; import { createFlakyTestScenario, createIgnoredChangeScenario, createTestChangeScenario, } from "../apps/backend/src/database/seeds"; import { formatTestId } from "../apps/backend/src/util/test-id"; import { loggedTest } from "./logged-test"; import { ensureTeamOwner, phoneScreenshot, screenshot } from "./util"; /** A one-line rich-text comment body. */ function doc(text: string) { return { type: "doc", content: [{ type: "paragraph", content: [{ type: "text", text }] }], }; } /** * The test ID in the breadcrumb is a sqid of the `tests` row id, so the Postgres * sequence decides it and it differs from one run to the next. Mask it, or every * screenshot of the test view reports a change. */ function maskTestId(testId: string) { return { replacements: { [testId]: "SPARKLE-XXX" } }; } loggedTest.beforeEach(async ({ auth, team }) => { await ensureTeamOwner({ team: team.team, user: auth.user }); }); loggedTest("tests", async ({ page, team, project, builds }) => { void builds; await page.goto(`/${team.account.slug}/${project.name}/tests`); await expect(page.getByRole("heading", { name: "Tests" })).toBeVisible(); await expect( page.getByRole("link", { name: /penelope-argos\.jpg/ }), ).toBeVisible(); await screenshot(page, "project-tests"); // The team's tests page lists the same tests across its projects. await page.goto(`/${team.account.slug}/~/tests`); await expect( page.getByRole("link", { name: /penelope-argos\.jpg/ }), ).toBeVisible(); await screenshot(page, "account-tests"); }); loggedTest("test detail", async ({ page, team, project, builds }) => { void builds; await page.goto(`/${team.account.slug}/${project.name}/tests`); await page.getByRole("link", { name: /penelope-argos\.jpg/ }).click(); await expect( page.getByRole("heading", { name: "penelope-argos.jpg" }), ).toBeVisible(); await expect(page.getByRole("heading", { name: /Changes/ })).toBeVisible(); const match = page.url().match(/\/tests\/([^/?#]+)/); if (!match?.[1]) { throw new Error("Test ID should be present in the URL"); } const testId = match[1]; await screenshot(page, "test-detail", maskTestId(testId)); await phoneScreenshot(page, "test-detail", { ready: (phone) => expect(phone.getByRole("heading", { name: /Changes/ })).toBeVisible(), ...maskTestId(testId), }); }); loggedTest("test view with a change", async ({ page, team, project }) => { const { test } = await createTestChangeScenario({ projectId: project.id }); const testId = formatTestId({ projectName: project.name, testId: test.id }); await page.goto(`/${team.account.slug}/${project.name}/tests/${testId}`); // The change's snapshot diff viewer is rendered. await expect( page.getByRole("heading", { name: "penelope-argos.jpg" }), ).toBeVisible(); await expect(page.getByText("Occurrences")).toBeVisible(); await expect(page.getByText("Baseline", { exact: false })).toBeVisible(); // No comment affordances in this view, despite the user being able to review. await expect(page.getByRole("button", { name: "Comment tool" })).toHaveCount( 0, ); await expect(page.getByRole("button", { name: "Move tool" })).toHaveCount(0); await expect(page.getByRole("button", { name: /comments$/ })).toHaveCount(0); // The change image is served from the CDN and actually renders (it isn't a // broken image). await expect .poll(() => page .getByRole("img", { name: "Changes screenshot" }) .first() .evaluate((el: HTMLImageElement) => el.complete && el.naturalWidth > 0), ) .toBe(true); await screenshot(page, "test-view-change", maskTestId(testId)); }); loggedTest( "test view flags ignored changes and filters on them", async ({ page, team, project, auth }) => { const { test } = await createIgnoredChangeScenario({ projectId: project.id, userId: auth.user.id, }); const testId = formatTestId({ projectName: project.name, testId: test.id }); await page.goto(`/${team.account.slug}/${project.name}/tests/${testId}`); // The badge is on the card in the list, without hovering it. const badge = page .getByRole("region", { name: "Changes" }) .getByText("Ignored", { exact: true }); await expect(badge).toBeVisible(); // Ignored changes are part of the default view, and the filter narrows it // down to them. `Changes over` rather than `Changes`, which the snapshot // pane's own heading also answers to. await expect( page.getByRole("heading", { name: /^Changes over/ }), ).toBeVisible(); await page.getByRole("button", { name: "Ignored", exact: true }).click(); await expect( page.getByRole("heading", { name: /^Ignored changes/ }), ).toBeVisible(); await expect(badge).toBeVisible(); // The filter is in the URL, so the view survives a reload and can be shared. await expect(page).toHaveURL(/changes=ignored/); await screenshot(page, "test-view-ignored-changes", maskTestId(testId)); }, ); loggedTest( "test view lets a reviewer comment on the test", async ({ page, team, project }) => { const { test } = await createTestChangeScenario({ projectId: project.id }); const testId = formatTestId({ projectName: project.name, testId: test.id }); await page.goto(`/${team.account.slug}/${project.name}/tests/${testId}`); // The right column carries the change history and the activity feed. await expect(page.getByText("First change")).toBeVisible(); await expect(page.getByText("Last change")).toBeVisible(); const activity = page.getByRole("heading", { name: "Activity" }); await expect(activity).toBeVisible(); await expect(page.getByText("Test created")).toBeVisible(); // Nobody follows the test until they engage with it. await expect(page.getByRole("button", { name: "Subscribe" })).toBeVisible(); // Posting a comment adds it to the feed. const editor = page.getByLabel("Add a comment"); await editor.click(); await editor.fill("This test keeps flapping on CI."); await page.getByRole("button", { name: "Submit the comment" }).click(); await expect( page.getByText("This test keeps flapping on CI."), ).toBeVisible(); // Commenting subscribes the author, so the bell flips to "Unsubscribe". const unsubscribe = page.getByRole("button", { name: "Unsubscribe" }); await expect(unsubscribe).toBeVisible(); await screenshot(page, "test-view-comment", maskTestId(testId)); // And the toggle opts back out. await unsubscribe.click(); await expect(page.getByRole("button", { name: "Subscribe" })).toBeVisible(); }, ); loggedTest( "test view marks a comment an agent posted for its author", async ({ page, auth, team, project }) => { const { test } = await createTestChangeScenario({ projectId: project.id }); const testId = formatTestId({ projectName: project.name, testId: test.id }); // Two comments from the same person: one they typed, one Claude Code posted // with their credentials. Only the second is marked, which is the whole // point — the author is identical on both. await Comment.query().insert([ { testId: test.id, userId: auth.user.id, content: doc("This one keeps flapping on CI."), }, { testId: test.id, userId: auth.user.id, content: doc("Retried it 20 times — the header animation races."), agent: "claude-code", }, ]); await page.goto(`/${team.account.slug}/${project.name}/tests/${testId}`); await expect( page.getByText("Retried it 20 times — the header animation races."), ).toBeVisible(); await expect( page.getByRole("img", { name: "Posted through Claude Code on behalf of this user", }), ).toBeVisible(); // The comment the same person typed carries no marker. await expect( page.getByRole("img", { name: /^Posted through/ }), ).toHaveCount(1); await screenshot(page, "test-view-agent-comment", maskTestId(testId)); }, ); loggedTest( "test view hands out a prompt to fix the flakiness with an agent", async ({ page, team, project, browserName }) => { const { test } = await createFlakyTestScenario({ projectId: project.id }); const testId = formatTestId({ projectName: project.name, testId: test.id }); // Reading back what was copied needs the clipboard permissions, and only // Chromium exposes them to Playwright — Firefox rejects the names outright. const canReadClipboard = browserName === "chromium"; await page.goto(`/${team.account.slug}/${project.name}/tests/${testId}`); if (canReadClipboard) { await page .context() .grantPermissions(["clipboard-read", "clipboard-write"], { origin: new URL(page.url()).origin, }); } // The test is flaky, so the section opens itself: the actions are reachable // without expanding anything. The button is named after what it does, and // Claude is the agent offered by default. await expect( page.getByRole("heading", { name: "Fix with AI" }), ).toBeVisible(); const fixButton = page.getByRole("link", { name: "Fix flakiness" }); await expect(fixButton).toBeVisible(); // The agent is opened through its own deep link, which carries the prompt: // it names the test and the API endpoints the agent has to call. const href = (await fixButton.getAttribute("href")) ?? ""; expect(href).toMatch(/^claude:\/\/code\/new\?q=/); const prompt = new URLSearchParams(href.split("?")[1]).get("q") ?? ""; expect(prompt).toContain("Fix the flaky Argos visual test"); expect(prompt).toContain("penelope-argos.jpg"); expect(prompt).toContain(`Test id: ${testId}`); expect(prompt).toContain( `/projects/${team.account.slug}/${project.name}/tests/${testId}?metricsPeriod=LAST_7_DAYS`, ); expect(prompt).toContain(`/tests/${testId}/changes`); expect(prompt).toContain("listTestChanges"); // The other agents, and the clipboard, are one menu away. const openMenu = () => page .getByRole("button", { name: "Choose what to do with the prompt" }) .click(); await openMenu(); await expect( page.getByRole("option", { name: "Open in Codex" }), ).toHaveAttribute("href", /^codex:\/\/new\?prompt=/); const cursorItem = page.getByRole("option", { name: "Open in Cursor" }); await expect(cursorItem).toHaveAttribute( "href", /^cursor:\/\/anysphere\.cursor-deeplink\/prompt\?text=/, ); // Copying is an action like opening an agent, so it puts the prompt on the // clipboard and becomes the one the button offers. await page.getByRole("option", { name: "Copy prompt" }).click(); const copyButton = page.getByRole("button", { name: /Cop(y prompt|ied)/ }); await expect(copyButton).toBeVisible(); if (canReadClipboard) { // The button only confirms itself once the write resolved, so it stands in // for the copy having landed. await expect(page.getByRole("button", { name: "Copied" })).toBeVisible(); await expect .poll(() => page.evaluate(() => navigator.clipboard.readText())) .toContain(`Test id: ${testId}`); } // And the choice is remembered, so the next flaky test starts there. await page.reload(); await expect( page.getByRole("button", { name: "Copy prompt" }), ).toBeVisible(); // Picking an agent moves the button back to it — the label stays the action, // so the deep link's scheme is what says where it now goes. Only Chromium // ignores the unknown scheme the click navigates to; the others would stop // on a dialog. if (browserName === "chromium") { await openMenu(); await page.getByRole("option", { name: "Open in Cursor" }).click(); await page.reload(); await expect(fixButton).toHaveAttribute( "href", /^cursor:\/\/anysphere\.cursor-deeplink\/prompt\?text=/, ); } }, ); loggedTest( "test view folds the AI prompt away on a stable test", async ({ page, team, project }) => { const { test } = await createTestChangeScenario({ projectId: project.id }); const testId = formatTestId({ projectName: project.name, testId: test.id }); await page.goto(`/${team.account.slug}/${project.name}/tests/${testId}`); // Nothing to fix here, so the section is folded into its header: still // there, but not competing with the metrics saying the test is fine. const heading = page.getByRole("heading", { name: "Fix with AI" }); await expect(heading).toBeVisible(); const fixButton = page.getByRole("link", { name: "Fix flakiness" }); await expect(fixButton).toBeHidden(); // And opening it hands out the same prompt. await heading.click(); await expect(fixButton).toBeVisible(); }, ); loggedTest( "test view explains an empty ignored list", async ({ page, team, project }) => { const { test } = await createTestChangeScenario({ projectId: project.id }); const testId = formatTestId({ projectName: project.name, testId: test.id }); await page.goto( `/${team.account.slug}/${project.name}/tests/${testId}?changes=ignored`, ); // The test has a change, but none of them is ignored. await expect( page.getByRole("heading", { name: "No ignored changes" }), ).toBeVisible(); // The empty state hands back the unfiltered list. await page.getByRole("button", { name: "See all changes" }).click(); await expect(page.getByRole("heading", { name: /^Changes/ })).toBeVisible(); await expect(page.getByText("Occurrences")).toBeVisible(); }, );