/** * images — the browser's hands for FILES: a canvas brand image, an upload that sets a file input, and * a download that takes an image out of a page. The page is faked (page.evaluate / page.$ return * canned values), so this proves the tool logic — the one live edge (a real canvas, a real file * input) is what a walk confirms. What matters now: made/downloaded bytes land in the CROSS-SESSION * store (fileAssets), so a picture generated on one login is uploadable from another — the fix that * lets the channel art and the video frames actually reach the next step. */ import { describe, it, expect, beforeEach, afterAll } from 'vitest'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { makeAssetStore, extFor } from '../src/assets.js'; // Point the persistent store at a temp dir BEFORE importing anything that binds it at load. const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gb-img-')); process.env.PROFILE_DIR = dir; const fileAssets = await import('../src/fileAssets.js'); const images = (await import('../src/tools/images.js')).default; const roles = (await import('../src/roles.js')).default; const agent = (await import('../src/agent.js')).default; afterAll(() => fs.rmSync(dir, { recursive: true, force: true })); beforeEach(() => { try { for (const f of fs.readdirSync(fileAssets.DIR)) fs.rmSync(path.join(fileAssets.DIR, f), { force: true }); } catch { /* fresh */ } }); // A 1x1 PNG as a data URL, so a faked canvas/fetch returns real bytes. const PNG_DATA_URL = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M8AAAMBAQDJ/pLvAAAAAElFTkSuQmCC'; // A fake image element handle: reports a src, and can be screenshotted. function fakeHandle({ src = 'https://lh3.googleusercontent.com/generated/x', screenshot = null } = {}) { return { evaluate: async () => src, screenshot: async () => screenshot, $: async () => null }; } function fakeCtx({ evaluate, query, handle, evaluateHandle } = {}) { const steps = []; const notes = []; let resets = 0; return { _steps: steps, _notes: notes, get _resets() { return resets; }, session: () => ({}), page: () => ({ evaluate: evaluate || (async () => PNG_DATA_URL), // download_image finds the image element via evaluateHandle → asElement(). evaluateHandle: evaluateHandle || (async () => ({ asElement: () => (handle || null) })), $: query || (async () => null), $$: async () => [], }), observe: (t) => notes.push(t), step: (k, t) => steps.push([k, t]), settle: async () => {}, sleep: async () => {}, resetClickLoop: () => { resets += 1; }, }; } describe('the per-session asset store (assets.js, still its own module)', () => { it('puts bytes and gets them back; list is metadata only (no bytes)', () => { const s = makeAssetStore(); const id = s.put({ kind: 'profile', mime: 'image/png', bytes: Buffer.from([1, 2, 3]) }); expect(s.get(id).bytes).toHaveLength(3); expect(s.latest('profile').id).toBe(id); expect(s.list()[0].bytes).toBe(3); // a COUNT, not the bytes }); it('extFor maps mimes', () => { expect(extFor('image/png')).toBe('png'); expect(extFor('application/x')).toBe('bin'); }); }); describe('make_brand_image → cross-session store', () => { it('draws a profile asset that lands in fileAssets, ready for any later step to upload', async () => { const ctx = fakeCtx(); await images.make_brand_image(ctx, { kind: 'profile', name: 'CaseHandoff', accent: '#d62b2b' }); const items = fileAssets.list(); expect(items).toHaveLength(1); expect(items[0].kind).toBe('profile'); expect(fileAssets.latest('profile').bytes.length).toBeGreaterThan(0); expect(ctx._notes.join(' ')).toMatch(/upload_image/); }); }); describe('upload_image', () => { it('sets the file input from a stored (cross-session) asset', async () => { let setWith = null; const input = { setInputFiles: async (f) => { setWith = f; } }; const ctx = fakeCtx({ query: async (sel) => (String(sel).includes('file') ? input : null) }); fileAssets.put({ kind: 'profile', mime: 'image/png', name: 'profile.png', bytes: Buffer.from([9, 9, 9]) }); await images.upload_image(ctx, { kind: 'profile' }); // no id → newest of the kind expect(setWith).toBeTruthy(); expect(setWith.name).toBe('profile.png'); expect(setWith.buffer).toHaveLength(3); expect(ctx._resets).toBe(1); // an upload is progress — resets the click-loop }); it('uploads by explicit assetId from a DIFFERENT session (the whole point of the store)', async () => { const id = fileAssets.put({ kind: 'cover', mime: 'image/png', name: 'cover.png', bytes: Buffer.from([7, 7]) }); let setWith = null; const ctx = fakeCtx({ query: async () => ({ setInputFiles: async (f) => { setWith = f; } }) }); await images.upload_image(ctx, { assetId: id }); expect(setWith.buffer).toHaveLength(2); }); it('with no image, says so instead of throwing', async () => { const ctx = fakeCtx(); await images.upload_image(ctx, { kind: 'profile' }); expect(ctx._notes.join(' ')).toMatch(/No stored image/); }); it('with no file input on the page, tells the walk to open the editor first', async () => { fileAssets.put({ kind: 'cover', mime: 'image/png', bytes: Buffer.from([1]) }); const ctx = fakeCtx({ query: async () => null }); await images.upload_image(ctx, { kind: 'cover' }); expect(ctx._notes.join(' ')).toMatch(/Open the photo editor first|No file input/); }); }); describe('download_image → cross-session store', () => { it('full-res fast path: fetches the image bytes in-page and stores them for a later login to upload', async () => { // fetch succeeds (page.evaluate returns a data URL) → the fetched bytes are stored. const ctx = fakeCtx({ handle: fakeHandle(), evaluate: async () => PNG_DATA_URL }); await images.download_image(ctx, { kind: 'cover' }); const items = fileAssets.list(); expect(items).toHaveLength(1); expect(items[0].kind).toBe('cover'); expect(ctx._notes.join(' ')).toMatch(/upload_image|upload_file/); }); it('reliable fallback: when the fetch cannot read the bytes (blob/CORS), it SCREENSHOTS the element', async () => { const shot = Buffer.from([5, 5, 5, 5]); // fetch returns null (a Gemini blob URL the in-page fetch cannot read) → screenshot the element. const ctx = fakeCtx({ handle: fakeHandle({ screenshot: shot }), evaluate: async () => null }); await images.download_image(ctx, { kind: 'profile' }); const item = fileAssets.latest('profile'); expect(item).toBeTruthy(); expect(item.bytes).toHaveLength(4); // the screenshot bytes, not an empty store expect(item.mime).toBe('image/png'); }); it('when no image element is present yet, says to wait rather than storing nothing', async () => { const ctx = fakeCtx({ evaluateHandle: async () => ({ asElement: () => null }) }); await images.download_image(ctx, {}); expect(fileAssets.list()).toHaveLength(0); expect(ctx._notes.join(' ')).toMatch(/No generated image/); }); }); describe('the roles carry the tools', () => { it('herald.facebook.onboard can make and upload images', () => { const picked = roles.toolsFor('herald.facebook.onboard', agent.TOOLS).map((t) => t.function.name); expect(picked).toContain('make_brand_image'); expect(picked).toContain('upload_image'); expect(picked).toContain('act'); }); it('the three image tools are declared in the model tool set', () => { const names = agent.TOOLS.map((t) => t.function.name); expect(names).toEqual(expect.arrayContaining(['make_brand_image', 'upload_image', 'download_image'])); }); }); /* * GIVING THE GENERATOR THE MARK TO MATCH. * * A cover asked for "in and around #5fe87f" comes back as a green rectangle. The same request with * the brand's own mark attached comes back as a banner that shares its weight, its geometry and its * exact greens — because the model can see the thing it is matching. * * The obstacle was never the picture, which has been on the shelf all along. It was that a chat's * attach button opens the OPERATING SYSTEM's file window, which a browser automation cannot answer. * Its paste handler can be reached, and pasting a screenshot is how people use these tools anyway. */ describe('pasting a reference image into a prompt box', () => { it('pastes the stored image of a named kind and reports what the page did with it', async () => { fileAssets.put({ kind: 'brandmark', mime: 'image/png', name: 'm.png', bytes: Buffer.from([1, 2, 3, 4]) }); /* The page half runs in a real browser; here it stands in, and returns what it would return. */ const ctx = fakeCtx({ evaluate: async () => ({ ok: true, handled: true, tag: 'div', label: 'Ask Gemini' }) }); await images.paste_image(ctx, { kind: 'brandmark' }); expect(ctx._steps.some(([k]) => k === 'image')).toBe(true); expect(ctx._notes.join(' ')).toMatch(/accepted the paste/); }); /* Delivered is not attached. Only a look can tell, so the walk is sent to look. */ it('never claims the image is attached — it sends the walk to check', async () => { fileAssets.put({ kind: 'brandmark', mime: 'image/png', name: 'm.png', bytes: Buffer.from([1, 2, 3, 4]) }); const ctx = fakeCtx({ evaluate: async () => ({ ok: true, handled: false, tag: 'textarea', label: '' }) }); await images.paste_image(ctx, { kind: 'brandmark' }); const note = ctx._notes.join(' '); expect(note).toMatch(/did not visibly take it/); expect(note).toMatch(/call look/i); expect(note).toMatch(/thumbnail or file chip/); }); /* A missing reference must never stop a run — the picture is better with it, not impossible without. */ it('no stored image of that kind is a note, not a failure', async () => { const ctx = fakeCtx(); await images.paste_image(ctx, { kind: 'brandmark' }); expect(ctx._steps).toHaveLength(0); expect(ctx._notes.join(' ')).toMatch(/carry on without a reference image/); }); it('and a page that refuses the paste says why, and says what to do instead', async () => { fileAssets.put({ kind: 'brandmark', mime: 'image/png', name: 'm.png', bytes: Buffer.from([1, 2, 3, 4]) }); const ctx = fakeCtx({ evaluate: async () => ({ ok: false, why: 'there is no writing box on this page to paste into' }) }); await images.paste_image(ctx, { kind: 'brandmark' }); expect(ctx._notes.join(' ')).toMatch(/was not pasted: there is no writing box/); expect(ctx._notes.join(' ')).toMatch(/describe the brand in words and carry on/); }); it('an exception in the page is survivable too', async () => { fileAssets.put({ kind: 'brandmark', mime: 'image/png', name: 'm.png', bytes: Buffer.from([1, 2, 3, 4]) }); const ctx = fakeCtx({ evaluate: async () => { throw new Error('detached frame'); } }); await images.paste_image(ctx, { kind: 'brandmark' }); expect(ctx._notes.join(' ')).toMatch(/could not be pasted \(detached frame\)/); expect(ctx._notes.join(' ')).toMatch(/not a reason to stop/); }); }); describe('the image role knows how to use a reference', () => { const r = roles.ROLES['gemini.image']; it('it is allowed the tool at all', () => { expect(r.tools).toContain('paste_image'); }); /* The attach button is a dead end, and a walk that tries it burns its steps on a file dialog. */ it('and is told to paste rather than hunt for a paperclip', () => { expect(r.prompt).toMatch(/do NOT hunt for the "\+" or a paperclip/); expect(r.prompt).toMatch(/open the computer's own file window, which you cannot use/); }); it('the reference goes in BEFORE the prompt is typed', () => { const paste = r.prompt.indexOf('paste_image'); const type = r.prompt.indexOf('TYPE the prompt into the box and RUN it'); expect(paste).toBeGreaterThan(0); expect(paste).toBeLessThan(type); }); it('and the walk must confirm the thumbnail before writing the prompt', () => { expect(r.prompt).toMatch(/A thumbnail or a file chip should be sitting in or just above the box/); expect(r.prompt).toMatch(/Only once the thumbnail is there, type the prompt/); }); /* Two runs that produced different pictures must not read the same in the record. */ it('the note says whether the reference actually went in', () => { expect(r.prompt).toMatch(/SAY WHETHER THE REFERENCE WAS ATTACHED/); }); it('the model is offered the tool', () => { const tool = agent.TOOLS ? agent.TOOLS.find((t) => t.function && t.function.name === 'paste_image') : null; if (tool) expect(tool.function.description).toMatch(/REFERENCE picture/); }); }); /* * IT DID THE WHOLE JOB INTO A SIGNED-OUT CHAT. * * Live: the walk opened Gemini in the google profile, pasted the mark, typed the full prompt and * sent it — while a "Sign in" button sat in the corner of the very first screenshot. The attachment * stopped at "Uploading file: 50%" and never moved. Nothing errored; it simply never finished. * * The order was wrong. Checking whether the door is open costs one look and is the cheapest thing * the walk can do; doing it last costs the whole run. */ describe('the image role checks the door before spending the run on it', () => { const r = roles.ROLES['gemini.image']; it('the FIRST question at each door is whether it is signed in', () => { expect(r.prompt).toMatch(/IS THIS SIGNED IN\?/); const ask = r.prompt.indexOf('IS THIS SIGNED IN'); const paste = r.prompt.indexOf('paste_image'); expect(ask).toBeGreaterThan(0); expect(ask).toBeLessThan(paste); }); it('and a signed-out chat is left alone, not filled in', () => { expect(r.prompt).toMatch(/do NOT paste, do NOT type, do NOT send/); }); /* The order that cost every run: the working door is tried first, not last. */ it('the door that is signed in is the one it starts at', () => { expect(r.prompt).toMatch(/THE FIRST DOOR — GOOGLE AI STUDIO/); expect(r.prompt).toMatch(/THE SECOND DOOR — GEMINI/); }); /* Both doors shut is a person's problem. Filling in a sign-in form is never the walk's job. */ it('and both doors shut ends the run instead of attempting a login', () => { expect(r.prompt).toMatch(/do NOT fill in a sign-in form/); expect(r.prompt).toMatch(/Somebody has to sign that browser profile in once/); }); /* A stuck percentage is not a slow generation, and telling them apart is what ends the waiting. */ it('a page that will never answer is recognised as one', () => { expect(r.prompt).toMatch(/A SIGNED-OUT SESSION IS NOT A SLOW ONE/); expect(r.prompt).toMatch(/Waiting longer has never once turned this into a picture/); }); /* The organ reads this marker to tell "try again later" apart from "nobody can fix this by trying". */ it('the one unfixable failure is marked so an organ can act on it', () => { expect(r.prompt).toMatch(/BEGIN YOUR NOTE WITH "SIGNED OUT:"/); expect(r.prompt).toMatch(/needs a person to sign in/); }); }); /* * A FINDING THAT IS NOT A NUMBER. * * Performance was the only thing a Search Console reader could file, so on a young property the * channel's entire output was "nought clicks, one impression" — true, and useless. The actionable * half of that console is counts, refusal reasons, sitemap statuses, manual actions and Google's own * messages, and none of them fitted through the reach door. * * One call per finding, as it is read: a walk that is stopped halfway still delivers what it saw. */ describe('recording what Search Console says about the property', () => { const records = require('../src/tools/records.js'); const ctxFor = () => { const steps = []; const notes = []; const rows = []; return { _steps: steps, _notes: notes, _rows: rows, addGscHealth: (r) => { if (!r || !r.kind || !r.label) return null; if (rows.some((x) => x.kind === r.kind && x.label === r.label && x.value === r.value)) return null; const row = { kind: r.kind, label: r.label, value: String(r.value ?? ''), detail: r.detail || '' }; rows.push(row); return row; }, step: (k, t) => steps.push([k, t]), observe: (t) => notes.push(t), }; }; it('a finding is recorded and named back to the walk', async () => { const ctx = ctxFor(); await records.save_gsc_health(ctx, { kind: 'indexing', label: 'not indexed', value: '12' }); expect(ctx._rows).toHaveLength(1); expect(ctx._steps[0][1]).toMatch(/indexing: not indexed = 12/); }); /* A walk that re-reads a tab must not double a finding, and must be told to move on. */ it('the same finding twice is the same finding', async () => { const ctx = ctxFor(); const row = { kind: 'sitemap', label: '/sitemap.xml', value: 'Geslaagd' }; await records.save_gsc_health(ctx, row); await records.save_gsc_health(ctx, row); expect(ctx._rows).toHaveLength(1); expect(ctx._notes.join(' ')).toMatch(/already down/); }); it('a finding with no kind is refused rather than stored empty', async () => { const ctx = ctxFor(); await records.save_gsc_health(ctx, { label: 'orphan' }); expect(ctx._rows).toHaveLength(0); expect(ctx._steps).toHaveLength(0); }); }); /* * ── SOMEBODY ELSE'S TOOLBAR ENDED UP IN OUR DIAGRAM ────────────────────────────────────────────── * * The first drawn diagram this platform published came out with three small icons in its top-right * corner — the generator's own share, copy and download buttons. Nothing grabbed them by accident. * An element screenshot captures whatever is painted over that RECTANGLE, and the toolbar sits on * top of the picture, so the fallback could not avoid photographing another product's interface onto * our page. No amount of aiming fixes that; it is the wrong kind of capture. * * The order is now: fetch the source, then read the image's own pixels through a canvas, and only * then photograph the screen. The canvas reads the IMAGE rather than the screen, at its real * resolution, with nothing on top of it — and it fails loudly (a tainted canvas throws) exactly when * the screenshot really is the only thing left. */ describe('a captured image is the image, not the region it sits in', () => { const src = fs.readFileSync(new URL('../src/tools/images.js', import.meta.url), 'utf8'); /* Scoped to download_image — screenshot_page photographs the screen on purpose, and always will. */ const grab = src.slice(src.indexOf('async download_image')); it('the pixels are read through a canvas before anything photographs the screen', () => { const canvasAt = grab.indexOf("c.getContext('2d').drawImage(el, 0, 0, w, h)"); const shotAt = grab.indexOf("handle.screenshot({ type: 'png' })"); expect(canvasAt).toBeGreaterThan(0); expect(shotAt).toBeGreaterThan(0); expect(canvasAt).toBeLessThan(shotAt); }); /* Full resolution, not display size — the canvas is sized from the image, not from the layout. */ it('at the image\'s own size rather than the size it happens to be shown at', () => { expect(src).toMatch(/const w = el\.naturalWidth \|\| el\.width, h = el\.naturalHeight \|\| el\.height/); }); /* A tainted canvas is the signal that the screenshot is genuinely the last resort. */ it('and a cross-origin image falls through rather than throwing', () => { expect(src).toMatch(/throws on a tainted canvas/); }); /* * The other way chrome got in: a numbered element that is a CARD, not an image. Capturing the card * captures its border, its padding and its buttons. */ it('a numbered element is used only when it is an image', () => { expect(src).toMatch(/handle = await marked\.\$\('img'\)\.catch/); expect(src).toMatch(/el\.tagName === 'IMG'/); expect(src).not.toMatch(/\(await marked\.\$\('img'\)\) \|\| marked/); }); });