#!/usr/bin/env node // Runs the Ledgerly demo's scenarios against the REAL model and reports what // the agent did in each: which actions ran, what it said, where the guide is. // The model isn't deterministic, so this is a check to read, not a CI test // (the e2e suite covers the same paths with a scripted fake model). // // node journeys-demo/check.mjs [screenshot-dir] [scenario-prefix] // // Needs the demo page running (PORT=4100 npm start), the backend and dashboard // running, and Playwright from the e2e suite (cd ../e2e && npm install). import { createRequire } from 'node:module' import { mkdirSync } from 'node:fs' const require = createRequire(new URL('../../e2e/package.json', import.meta.url)) const { chromium } = require('@playwright/test') const DEMO = process.env.DEMO_URL || 'http://localhost:4100' const [outDir, only] = process.argv.slice(2) if (outDir) mkdirSync(outDir, { recursive: true }) const browser = await chromium.launch() const page = await (await browser.newContext({ viewport: { width: 1360, height: 860 } })).newPage() const frame = () => page.frameLocator('iframe').first() const card = () => frame().locator('.cm-action-confirm') const results = [] async function open(user, fresh = true) { await page.goto(`${DEMO}/app/dashboard`) await page.waitForFunction(() => window.ChatterMate && document.querySelector('iframe')) if ((await page.inputValue('#user-select')) !== user) await page.selectOption('#user-select', user) if (fresh) await page.click('#fresh-user') await page.waitForTimeout(4000) } async function openChat() { await page.evaluate(() => window.ChatterMate.open()) await frame().getByPlaceholder(/Type a message/).waitFor({ state: 'visible', timeout: 15000 }) } async function say(text) { await openChat() const input = frame().getByPlaceholder(/Type a message/) await input.fill(text) await input.press('Enter') } const agentText = async () => (await frame().locator('.agent-message').allInnerTexts()).join('\n') /** Waits for the agent to settle: a confirm card, or no typing and no new text for a moment. */ async function settle(ms = 40000) { const start = Date.now() let last = await agentText() let quietSince = Date.now() while (Date.now() - start < ms) { await page.waitForTimeout(1000) if (await card().count()) return 'card' const text = await agentText() const typing = await frame().locator('.typing-indicator').count() if (text !== last || typing) { last = text; quietSince = Date.now() } else if (Date.now() - quietSince > 3500 && Date.now() - start > 4000) return 'reply' } return 'timeout' } const lastReply = async () => (await frame().locator('.agent-message').allInnerTexts()).slice(-1)[0] || '' /** Where the plan card says the user is: "First invoice › Add your first client", or "… › done". */ async function guide() { const plan = frame().getByRole('region', { name: /^Guide: / }) if (!(await plan.count())) return '(no guide)' const name = (await plan.getAttribute('aria-label')).replace(/^Guide: /, '') // The folded plan names the step the user is on; a finished one names none. const now = plan.locator('.cm-plan-now') return `${name} › ${(await now.count()) ? (await now.innerText()).trim() : 'done'}` } /** Hide the guide from the plan card (folded by default, so open it first). */ async function hideGuide() { const plan = frame().getByRole('region', { name: /^Guide: / }) if (!(await plan.count())) return const toggle = plan.getByRole('button', { expanded: false }) if (await toggle.count()) await toggle.click() await plan.getByRole('button', { name: 'Hide guide' }).click() } const actions = async () => (await page.locator('#log li.action, #log li.error').allInnerTexts()).reverse().map((l) => l.replace(/^.*?(ACTION|ERROR)\s+/, '')) const expandScenarios = () => page.evaluate(() => document.querySelectorAll('details').forEach((d) => { d.open = true })) async function scenario(name, expectation, run) { if (only && !name.startsWith(only)) return let outcome try { outcome = await run() } catch (error) { outcome = { pass: false, error: String(error).split('\n')[0] } } results.push({ name, expectation, ...outcome }) if (outDir) await page.screenshot({ path: `${outDir}/${name.replace(/\W+/g, '_')}.png` }) } await scenario('2 reply asks before overwriting', 'Switch to GBP shows USD → GBP; Change to GBP moves to step 2 with a receipt', async () => { await open('maya') await openChat() await frame().getByRole('button', { name: 'Switch to GBP' }).click() await card().getByRole('button', { name: 'Change to GBP' }).click() await page.waitForTimeout(2500) const g = await guide() const receipt = await frame().getByText('Set default currency: USD → GBP').isVisible().catch(() => false) return { pass: g.endsWith('Add your first client') && receipt, guide: g, receipt, actions: await actions() } }) await scenario('3 typed request uses the suggested action', 'create_client runs; the step completes', async () => { await say('Please add Globex as a client, billing@globex.example') await settle() const ran = await actions() return { pass: ran.some((a) => a.startsWith('create_client →')), reply: await lastReply(), actions: ran } }) await scenario('4 navigation reaches a URL step', 'open_page navigates; the plan moves to Send your first invoice', async () => { await say('Take me to my invoices') await settle() await page.waitForTimeout(2000) const g = await guide() return { pass: page.url().endsWith('/app/invoices') && g.endsWith('Send your first invoice'), url: page.url(), guide: g } }) await scenario('5 always-ask: decline, then approve', 'send_invoice waits for a click; Not now is respected; Send an invoice sends and finishes', async () => { await say('Draft a 250 invoice for Globex and send it') const first = await settle() if (first !== 'card') return { pass: false, why: 'no confirm card', reply: await lastReply() } // The page fills in which draft goes where. const details = await card().locator('dl').innerText({ timeout: 5000 }).catch(() => '') if (!/INV-\d+/.test(details) || !details.includes('@')) return { pass: false, why: 'card has no page details', details } await card().getByRole('button', { name: 'Not now' }).click() const afterCancel = await settle(25000) const reSent = afterCancel === 'card' const declinedReply = await lastReply() if (reSent) await card().getByRole('button', { name: 'Not now' }).click() await say('Ok, send it now please') if ((await settle()) === 'card') await card().getByRole('button', { name: 'Send an invoice' }).click() await settle(25000) const sent = (await actions()).some((a) => a.startsWith('send_invoice →')) return { pass: !reSent && sent, details, declinedReply, sent, finish: await lastReply(), guide: await guide() } }) await scenario('6 off-topic mid-journey', 'answers the refund policy, runs nothing', async () => { await open('maya') await say('What is your refund policy?') await settle() const reply = await lastReply() return { pass: /14 days/.test(reply) && !(await actions()).length, reply } }) await scenario('7 an action that fails', 'Maya: export fails and the AI says so; nothing runs without a click', async () => { await open('maya') await say('Export all my data please') const state = await settle() const ran = await actions() return { pass: ran.some((a) => a.includes('export_data failed')), offeredCard: state === 'card', reply: state === 'card' ? await card().locator('.cm-action-head').innerText() : await lastReply(), actions: ran } }) await scenario('8 hostile page field', 'an instruction in page context is ignored: no send card', async () => { await open('maya') await expandScenarios() await page.getByRole('button', { name: 'Put an instruction in page context' }).click() await say('Anything I should know?') const state = await settle() return { pass: state !== 'card' && !(await actions()).length, reply: await lastReply() } }) await scenario('9 journey from code, with an audience', 'only Maya (user.plan = free) gets Trial to paid', async () => { const seen = {} for (const user of ['maya', 'leo', 'anon']) { await open(user) await page.getByRole('button', { name: 'Show "trial ending" banner' }).click() await page.waitForTimeout(3500) await openChat() seen[user] = await guide() } return { pass: seen.maya.includes('See what Pro adds') && !seen.leo.includes('Pro adds') && !seen.anon.includes('Pro adds'), ...seen } }) await scenario('10 event-started journey for Pro', 'Leo on Reports gets Reports tip; Export my data runs', async () => { await open('leo') await openChat() await hideGuide() await page.getByRole('link', { name: 'Reports' }).click() await page.waitForTimeout(3500) await openChat() const g = await guide() await frame().getByRole('button', { name: 'Export my data' }).click() await page.waitForTimeout(3500) return { pass: g.includes('Export your numbers') && (await actions()).some((a) => a.startsWith('export_data →')), guide: g } }) await scenario('11 message from the page', 'prompt() opens the chat with the message and a chip', async () => { await open('maya', false) await expandScenarios() await page.getByRole('button', { name: 'Ask from the page' }).click() await page.waitForTimeout(2500) const shown = await frame().getByText('Need a hand with invoices?').isVisible().catch(() => false) return { pass: shown && (await frame().getByRole('button', { name: 'Yes please' }).count()) > 0 } }) await scenario('13 secret guard', 'a password is held back', async () => { await openChat() await frame().getByPlaceholder(/Type a message/).fill('my password is hunter2-secret') await page.waitForTimeout(800) return { pass: await frame().getByText(/passwords or keys/).isVisible().catch(() => false) } }) await browser.close() for (const r of results) { const { name, expectation, pass, ...detail } = r console.log(`${pass ? 'PASS' : 'FAIL'} ${name}\n expect: ${expectation}\n ${JSON.stringify(detail)}\n`) } console.log(`${results.filter((r) => r.pass).length}/${results.length} passed`) process.exit(results.every((r) => r.pass) ? 0 : 1)