// @vitest-environment happy-dom // Talk to Billion, the page (public/modules/talk.js, talk-turns.js): each // stretch of speech transcribed as WAV, the turn sent once after a real pause // (two stretches with a short pause between are one turn), an acknowledgment // the moment it ends, only the reply bound to it spoken (piece by piece from // the server), words added before the reply continuing the turn, speaking // over Billion cutting it off and dropping what was not played, mute and End // stopping the mic, the browser's voice when the server has none, and a clear // refusal where the browser cannot do it. The voice detector, fetch and the // audio element are stand-ins. import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { encodeWav, shouldSpeak, talkState, progressWait, talkStatus, PROGRESS_QUIET_MS, SEGMENT_SILENCE_MS, talkBar, talkButton, startTalk, endTalk, talkHeard, setMuted, talkOn, earpieceId, _resetTalk, _talkInternals, } from '../public/modules/talk.js'; import { unfinished, turnPause, pickAck, createTurns, PAUSE_MS } from '../public/modules/talk-turns.js'; import { setChatMessages, setBillionStatus } from '../public/modules/state.js'; const flush = async (n = 6) => { for (let i = 0; i < n; i++) await new Promise(r => setTimeout(r, 0)); }; let vadOpts, fakeVad, routes, requests, played, theAudio; class FakeAudio { // talk.js keeps one element for the page's life (unlocked in the first tap). constructor() { this.paused = true; this.volume = 1; theAudio = this; } play() { this.paused = false; if (this.blobText !== undefined) played.push(this.blobText); return Promise.resolve(); } pause() { this.paused = true; } set src(url) { this._src = url; this.blobText = blobs.get(url); } get src() { return this._src; } end() { this.onended?.(); } setSinkId(id) { this.sinkId = id; return Promise.resolve(); } } const blobs = new Map(); let blobN = 0; function response(body, { status = 200, headers = {} } = {}) { return { ok: status < 400, status, headers: { get: (k) => headers[k] ?? null }, json: async () => body, blob: async () => ({ text: String(body) }), }; } beforeEach(() => { document.body.innerHTML = ''; mountComposer(); requests = []; played = []; vadOpts = null; fakeVad = { start: vi.fn(async () => {}), pause: vi.fn(async () => {}), destroy: vi.fn(async () => {}), setOptions: vi.fn() }; window.ort = {}; window.vad = { MicVAD: { new: vi.fn(async (opts) => { vadOpts = opts; return fakeVad; }) } }; routes = { '/api/talk': () => response({ stt: 'whisper', tts: 'say' }), '/api/talk/segment': () => response({ text: 'How is it going?', transcribeMs: 300 }), '/api/talk/text': () => response({ ok: true, id: 'm1' }), }; vi.stubGlobal('fetch', vi.fn(async (url, init = {}) => { requests.push({ url, init }); if (init.signal?.aborted) throw new DOMException('aborted', 'AbortError'); const audio = url.match(/^\/api\/talk\/audio\/([^/]+)\/(\d+)$/); if (audio) return routes.audio ? routes.audio(audio[1], Number(audio[2])) : response(`${audio[1]}#${audio[2]}`, { headers: { 'X-Pieces': '2' } }); return routes[url](); })); vi.stubGlobal('Audio', FakeAudio); URL.createObjectURL = vi.fn((blob) => { const url = `blob:${blobN++}`; if (typeof blob.text === 'string') blobs.set(url, blob.text); return url; }); URL.revokeObjectURL = vi.fn(); Object.defineProperty(navigator, 'mediaDevices', { value: { getUserMedia: vi.fn() }, configurable: true }); Object.defineProperty(window, 'isSecureContext', { value: true, configurable: true }); setChatMessages([]); setBillionStatus(null); }); afterEach(() => { _resetTalk(); vi.unstubAllGlobals(); delete navigator.audioSession; try { localStorage.clear(); } catch {} }); // The composer as waiting.js builds it: the bar above the row, the phone button in it. function mountComposer() { document.body.innerHTML = ''; const row = document.createElement('div'); row.id = 'chat-compose-row'; const input = document.createElement('textarea'); input.id = 'chat-input'; const mic = document.createElement('button'); mic.id = 'chat-mic'; const send = document.createElement('button'); send.id = 'chat-send'; row.append(input, mic, talkButton(), send); document.body.append(talkBar(), row); } const bar = () => document.getElementById('talk-bar'); const stateWord = () => bar().dataset.state; const reply = (id, replyTo, text = 'It went fine.') => ({ id, from: 'billion', replyTo, text, at: new Date().toISOString() }); async function talking() { await startTalk(); await flush(); expect(stateWord()).toBe('listening'); } const posts = (url) => requests.filter(r => r.url === url); const turnsSent = () => posts('/api/talk/text').map(r => JSON.parse(r.init.body)); // The owner says something and pauses long enough to finish the turn. async function utterance() { const fake = vi.isFakeTimers(); if (!fake) vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] }); vadOpts.onSpeechEnd(new Float32Array(1600)); await vi.advanceTimersByTimeAsync(PAUSE_MS.unfinished); if (!fake) vi.useRealTimers(); await flush(); } describe('pure pieces', () => { it('encodeWav writes a 16 kHz 16-bit mono WAV', () => { const buf = encodeWav(new Float32Array([0, 1, -1])); const v = new DataView(buf); expect(String.fromCharCode(...new Uint8Array(buf, 0, 4))).toBe('RIFF'); expect(v.getUint32(24, true)).toBe(16000); expect(v.getUint16(22, true)).toBe(1); expect(v.getUint16(34, true)).toBe(16); expect(buf.byteLength).toBe(44 + 6); expect(v.getInt16(46, true)).toBe(0x7fff); expect(v.getInt16(48, true)).toBe(-0x8000); }); it('shouldSpeak: only Billion\'s reply to a message of this conversation, once', () => { const awaiting = new Set(['m1']); expect(shouldSpeak(reply('r1', 'm1'), awaiting, new Set())).toBe(true); expect(shouldSpeak(reply('r1', 'm1'), awaiting, new Set(['r1']))).toBe(false); expect(shouldSpeak(reply('r2', 'typed'), awaiting, new Set())).toBe(false); expect(shouldSpeak({ ...reply('r3', 'm1'), notice: true }, awaiting, new Set())).toBe(false); expect(shouldSpeak({ id: 'o', from: 'owner', replyTo: 'm1' }, awaiting, new Set())).toBe(false); }); it('talkState names each state', () => { const base = { on: true, connected: true }; expect(talkState({})).toBe('off'); expect(talkState(base)).toBe('listening'); expect(talkState({ ...base, uploads: 1 })).toBe('thinking'); expect(talkState({ ...base, awaiting: 1, playing: 'r1' })).toBe('speaking'); expect(talkState({ ...base, muted: true })).toBe('muted'); expect(talkState({ ...base, connected: false })).toBe('disconnected'); }); }); describe('a turn', () => { it('transcribes the speech as WAV, sends the turn once, speaks only its own reply piece by piece, then listens', async () => { await talking(); expect(window.vad.MicVAD.new).toHaveBeenCalledWith(expect.objectContaining({ baseAssetPath: '/vendor/vad/', onnxWASMBasePath: '/vendor/ort/', redemptionMs: SEGMENT_SILENCE_MS })); expect(fakeVad.start).toHaveBeenCalled(); await utterance(); const post = requests.find(r => r.url === '/api/talk/segment'); expect(post.init.headers).toMatchObject({ 'X-Agent007-Talk': '1', 'Content-Type': 'audio/wav', 'X-Segment-Id': expect.any(String) }); expect(post.init.body.byteLength).toBe(44 + 3200); expect(turnsSent()).toEqual([{ utterance: expect.any(String), text: 'How is it going?' }]); expect(stateWord()).toBe('thinking'); talkHeard(reply('rX', 'typed-message', 'Not yours.')); await flush(); expect(requests.some(r => r.url.includes('/audio/rX/'))).toBe(false); talkHeard(reply('r1', 'm1')); await flush(); expect(stateWord()).toBe('speaking'); expect(fakeVad.setOptions).toHaveBeenCalledWith({ positiveSpeechThreshold: 0.8 }); expect(played).toEqual(['r1#0']); theAudio.end(); await flush(); expect(played).toEqual(['r1#0', 'r1#1']); theAudio.end(); await flush(); expect(stateWord()).toBe('listening'); // The same reply seen again (a reconnect's chat-list) is not spoken twice. talkHeard(reply('r1', 'm1')); await flush(); expect(played).toHaveLength(2); }); it('speaking over Billion stops it, drops the rest, and marks the next utterance as maybe-echo', async () => { await talking(); await utterance(); talkHeard(reply('r1', 'm1')); await flush(); expect(played).toEqual(['r1#0']); vadOpts.onSpeechRealStart(); await flush(); expect(theAudio.paused).toBe(true); expect(stateWord()).toBe('listening'); theAudio.end(); await flush(); expect(played).toEqual(['r1#0']); routes['/api/talk/text'] = () => response({ ok: true, id: 'm2' }); await utterance(); expect(posts('/api/talk/segment')[1].init.headers['X-Echo-Of']).toBe('r1'); // The reply came, so this is a new turn, not more of the last. expect(turnsSent()[1].continues).toBeUndefined(); expect(_talkInternals().awaiting).toEqual(['m2']); }); it('a network failure retries the same turn id, never a new one', async () => { await talking(); let fails = 1; routes['/api/talk/text'] = () => { if (fails-- > 0) throw new TypeError('network'); return response({ ok: true, id: 'm1' }); }; vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] }); vadOpts.onSpeechEnd(new Float32Array(160)); await vi.advanceTimersByTimeAsync(PAUSE_MS.unfinished + 1500); vi.useRealTimers(); await flush(); const sent = turnsSent(); expect(sent).toHaveLength(2); expect(sent[1].utterance).toBe(sent[0].utterance); expect(_talkInternals().awaiting).toEqual(['m1']); }); it('mute stops the mic, End stops everything and speaks nothing more', async () => { await talking(); setMuted(true); expect(fakeVad.pause).toHaveBeenCalled(); expect(stateWord()).toBe('muted'); const starts = fakeVad.start.mock.calls.length; setMuted(false); expect(fakeVad.start).toHaveBeenCalledTimes(starts + 1); await utterance(); endTalk(); expect(fakeVad.destroy).toHaveBeenCalled(); expect(talkOn()).toBe(false); talkHeard(reply('r1', 'm1')); await flush(); expect(played).toEqual([]); expect(bar().querySelector('.talk-call').hidden).toBe(true); }); it('without say on the server, the browser speaks the reply', async () => { const spoken = []; vi.stubGlobal('speechSynthesis', { speak: (u) => { spoken.push(u.text); setTimeout(() => u.onend?.()); }, cancel: vi.fn(), resume: vi.fn(), getVoices: () => [] }); vi.stubGlobal('SpeechSynthesisUtterance', class { constructor(t) { this.text = t; } }); routes.audio = () => response({ error: 'no say' }, { status: 503 }); await talking(); await utterance(); talkHeard(reply('r1', 'm1', 'All **green**. Merging now.')); await flush(10); expect(spoken).toEqual(['All green. Merging now.']); expect(bar().querySelector('.talk-privacy').textContent).toMatch(/spoken by this browser/); }); }); describe('races', () => { it('End then Talk while the first start still waits leaves exactly one detector, and End stops it', async () => { let release; const gate = new Promise(r => { release = r; }); routes['/api/talk'] = () => gate.then(() => response({ stt: 'whisper', tts: 'say' })); const made = []; window.vad.MicVAD.new = vi.fn(async (opts) => { vadOpts = opts; const v = { ...fakeVad, start: vi.fn(async () => {}), destroy: vi.fn(async () => {}) }; made.push(v); return v; }); const first = startTalk(); endTalk(); const second = startTalk(); release(); await Promise.all([first, second]); await flush(); expect(made).toHaveLength(1); expect(stateWord()).toBe('listening'); endTalk(); expect(made[0].destroy).toHaveBeenCalled(); }); it('an utterance dropped as Billion\'s own voice says so', async () => { await talking(); routes['/api/talk/segment'] = () => response({ text: '', echo: true }); await utterance(); expect(bar().querySelector('.talk-detail').textContent).toMatch(/Billion's own voice/); expect(turnsSent()).toEqual([]); expect(_talkInternals().awaiting).toEqual([]); }); }); describe('progress updates while Billion works', () => { const status = (text) => { const s = { type: 'billion-status', text, working: true, currentRequest: 'm1', progress: { m1: [] } }; setBillionStatus(s); talkStatus(s); }; const said = () => played.filter(p => p.startsWith('status#')); const wait = (ms) => vi.advanceTimersByTimeAsync(ms); afterEach(() => vi.useRealTimers()); it('progressWait: quiet only at first, then now; never a repeat or nothing', () => { const p = { since: 0, said: 'Old news' }; expect(progressWait('Reading', p, 500)).toBe(PROGRESS_QUIET_MS - 500); expect(progressWait('Reading', p, PROGRESS_QUIET_MS)).toBe(0); expect(progressWait('Old news', p, 60000)).toBeNull(); expect(progressWait('', p, 60000)).toBeNull(); }); it('speaks a new status line at once after 0.8 s, newest only, no repeats, no filler, gives way to the answer', async () => { await talking(); routes.audio = (id, i) => response(`${id}#${i}`, { headers: { 'X-Pieces': id === 'status' ? '1' : '2' } }); vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] }); status('Looking at the board'); // from before the turn: not news vadOpts.onSpeechEnd(new Float32Array(1600)); await wait(PAUSE_MS.finished - SEGMENT_SILENCE_MS); expect(PROGRESS_QUIET_MS).toBe(800); await wait(PROGRESS_QUIET_MS); expect(said()).toEqual([]); status('Reading the talk code'); await wait(0); expect(said()).toEqual(['status#0']); expect(_talkInternals().playing).toBe('status'); expect(fakeVad.setOptions).toHaveBeenLastCalledWith({ positiveSpeechThreshold: 0.8 }); // Several changes while one is spoken: only the newest follows, right away. status('Checking the board'); status('Running the tests'); await wait(5000); expect(said()).toHaveLength(1); theAudio.end(); await wait(300); expect(said()).toHaveLength(2); expect(requests.filter(r => r.url.includes('/audio/status/'))).toHaveLength(2); theAudio.end(); await wait(0); // The same line again is not repeated. status('Running the tests'); await wait(5000); expect(said()).toHaveLength(2); // Never over the owner: an update waits while they speak. vadOpts.onSpeechRealStart(); status('Writing the fix'); await wait(5000); expect(said()).toHaveLength(2); vadOpts.onVADMisfire(); await wait(300); expect(said()).toHaveLength(3); // The answer arrives mid-update: the update stops, the answer plays next. talkHeard(reply('r1', 'm1')); await wait(0); expect(_talkInternals().playing).toBe('r1'); expect(played.at(-1)).toBe('r1#0'); // Answered: nothing more, and no filler cue ever. theAudio.end(); await wait(0); theAudio.end(); await wait(0); status('Tidying up'); await wait(60000); expect(said()).toHaveLength(3); expect(requests.some(r => r.url.includes('/audio/cue/'))).toBe(false); }); it('speaks the box heading when there is no status line; a status line wins; each heading once', async () => { await talking(); routes.audio = (id, i) => response(`${id}#${i}`, { headers: { 'X-Pieces': '1' } }); vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] }); const heading = (working, text = '') => { const s = { type: 'billion-status', text, working, currentRequest: 'm1', progress: { m1: [] } }; setBillionStatus(s); talkStatus(s); }; heading(false); vadOpts.onSpeechEnd(new Float32Array(1600)); await wait(PAUSE_MS.finished - SEGMENT_SILENCE_MS); await wait(PROGRESS_QUIET_MS - 100); expect(said()).toEqual([]); await wait(100); expect(said()).toEqual(['status#0']); theAudio.end(); await wait(0); heading(true); // the same heading: not said again await wait(5000); expect(said()).toHaveLength(1); heading(true, 'Reading the talk code'); // a status line wins await wait(300); expect(said()).toHaveLength(2); theAudio.end(); await wait(0); heading(true); // cleared: the heading again await wait(300); expect(said()).toHaveLength(3); expect(requests.some(r => r.url.includes('/audio/cue/'))).toBe(false); }); }); describe('hang up focus', () => { it.each([['coarse', true], ['fine', false]])('does not focus the textarea on a %s pointer; the Talk button gets it', async (_, coarse) => { vi.stubGlobal('matchMedia', (q) => ({ matches: coarse && q === '(pointer: coarse)' })); await talking(); endTalk(); await flush(); expect(document.activeElement).not.toBe(document.getElementById('chat-input')); expect(document.activeElement).toBe(document.getElementById('chat-talk')); }); }); describe('speaker or earpiece (iPhone)', () => { const OUTPUTS = [ { kind: 'audioinput', label: 'iPhone Microphone', deviceId: 'mic' }, { kind: 'audiooutput', label: 'Speaker', deviceId: 'spk' }, { kind: 'audiooutput', label: 'iPhone', deviceId: 'rcv' }, ]; const outButton = () => bar().querySelector('.talk-output'); function iPhone() { Object.defineProperty(navigator, 'audioSession', { value: { type: 'auto' }, configurable: true }); vi.stubGlobal('matchMedia', (q) => ({ matches: q === '(pointer: coarse)' })); navigator.mediaDevices.enumerateDevices = vi.fn(async () => OUTPUTS); } it('earpieceId picks the receiver, never the loudspeaker or a mic', () => { expect(earpieceId(OUTPUTS)).toBe('rcv'); expect(earpieceId([{ kind: 'audiooutput', label: 'Receiver', deviceId: 'r' }])).toBe('r'); expect(earpieceId(OUTPUTS.slice(0, 2))).toBe(''); }); it('a desktop browser shows no output button', async () => { await talking(); expect(outButton().hidden).toBe(true); }); it('defaults to Speaker, holds play-and-record for the call, and remembers Earpiece', async () => { iPhone(); await talking(); expect(outButton().hidden).toBe(false); expect(outButton().textContent).toBe('Speaker'); expect(navigator.audioSession.type).toBe('play-and-record'); expect(theAudio.sinkId).toBe(''); outButton().click(); await flush(); expect(outButton().textContent).toBe('Earpiece'); expect(theAudio.sinkId).toBe('rcv'); expect(localStorage.getItem('agent007-talk-output')).toBe('earpiece'); endTalk(); expect(navigator.audioSession.type).toBe('auto'); await talking(); await flush(); expect(outButton().textContent).toBe('Earpiece'); expect(theAudio.sinkId).toBe('rcv'); outButton().click(); await flush(); expect(theAudio.sinkId).toBe(''); expect(localStorage.getItem('agent007-talk-output')).toBe('speaker'); }); it('says so when the earpiece cannot be picked', async () => { iPhone(); navigator.mediaDevices.enumerateDevices = vi.fn(async () => OUTPUTS.slice(0, 2)); localStorage.setItem('agent007-talk-output', 'earpiece'); await talking(); await flush(); expect(bar().querySelector('.talk-detail').textContent).toMatch(/does not let the page pick the earpiece/); }); }); describe('the mic stays open for the call', () => { const fakeStream = () => { const track = { enabled: true, readyState: 'live', stop: vi.fn(function () { this.readyState = 'ended'; }) }; return { track, getTracks: () => [track] }; }; it('mute disables the track instead of stopping it; unmute re-enables the same stream', async () => { const s = fakeStream(); navigator.mediaDevices.getUserMedia = vi.fn(async () => s); await talking(); expect(await vadOpts.getStream()).toBe(s); await vadOpts.pauseStream(s); expect(s.track.enabled).toBe(false); expect(s.track.stop).not.toHaveBeenCalled(); expect(await vadOpts.resumeStream(s)).toBe(s); expect(s.track.enabled).toBe(true); expect(navigator.mediaDevices.getUserMedia).toHaveBeenCalledTimes(1); }); it('End stops the track, even while muted, and a stopped track is reopened on resume', async () => { const s = fakeStream(); navigator.mediaDevices.getUserMedia = vi.fn(async () => s); await talking(); await vadOpts.getStream(); setMuted(true); await vadOpts.pauseStream(s); endTalk(); expect(s.track.stop).toHaveBeenCalled(); const fresh = fakeStream(); navigator.mediaDevices.getUserMedia = vi.fn(async () => fresh); expect(await vadOpts.resumeStream(s)).toBe(fresh); }); }); describe('when the owner has finished (talk-turns.js)', () => { afterEach(() => vi.useRealTimers()); it('unfinished: a phrase that stops mid-thought, whatever whisper\'s full stop says', () => { for (const t of ['Check the release and', 'I want to.', 'Look at the, um', 'Merge it because', 'So the plan is,', 'Then…', 'Open the PR for the']) expect(unfinished(t), t).toBe(true); for (const t of ['Merge it.', 'Is the build green?', 'Turn it on.', 'Log in', 'Thank you', 'I think so.', '']) expect(unfinished(t), t).toBe(false); }); it('turnPause: 1.2 s after a finished sentence, 1.5 s usually, longer after an unfinished phrase', () => { expect(PAUSE_MS.finished).toBeGreaterThanOrEqual(1200); expect(PAUSE_MS.usual).toBeLessThanOrEqual(1800); expect(turnPause('Is the build green?')).toBe(PAUSE_MS.finished); expect(turnPause('Merge the release now.')).toBe(PAUSE_MS.finished); expect(turnPause('')).toBe(PAUSE_MS.usual); expect(turnPause('Okay')).toBe(PAUSE_MS.usual); expect(turnPause('Check the release and')).toBe(PAUSE_MS.unfinished); expect(PAUSE_MS.unfinished).toBeGreaterThan(PAUSE_MS.usual); }); it('pickAck never picks the one before', () => { for (let last = 0; last < 4; last++) { for (const r of [0, 0.3, 0.6, 0.99]) { const i = pickAck(4, last, () => r); expect(i).not.toBe(last); expect(i).toBeGreaterThanOrEqual(0); expect(i).toBeLessThan(4); } } expect(pickAck(1, 0)).toBe(0); expect(pickAck(0, -1)).toBe(-1); }); // A gatherer whose transcripts are the strings handed to segment(), after `ms`. function gatherer({ ms = 100, lead = SEGMENT_SILENCE_MS } = {}) { vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] }); const sent = [], echoes = []; let n = 0; const turns = createTurns({ transcribe: (input) => new Promise(r => setTimeout(() => r(String(input) === 'ECHO' ? { text: '', echo: true } : { text: 'Heard.' }), ms)), send: (t) => sent.push(t), onEcho: () => echoes.push(true), newId: () => `turn-${++n}`, lead, }); return { turns, sent, echoes }; } it('pause, then more words before the pause is up: one turn with both', async () => { const { turns, sent } = gatherer(); // ("Check the release." reads finished: 1.2 s of silence ends it, 0.5 s of which the detector waited.) turns.segment('Check the release.'); await vi.advanceTimersByTimeAsync(600); expect(sent).toEqual([]); turns.speechStart(); await vi.advanceTimersByTimeAsync(3000); // talking: never sent mid-speech expect(sent).toEqual([]); turns.segment('And the open PRs.'); await vi.advanceTimersByTimeAsync(PAUSE_MS.finished - SEGMENT_SILENCE_MS - 1); expect(sent).toEqual([]); await vi.advanceTimersByTimeAsync(1); expect(sent).toEqual([{ text: 'Check the release. And the open PRs.', utterance: 'turn-1' }]); }); it('waits longer after an unfinished phrase', async () => { const { turns, sent } = gatherer(); turns.segment('Can you check the release and'); await vi.advanceTimersByTimeAsync(PAUSE_MS.usual); expect(sent).toEqual([]); await vi.advanceTimersByTimeAsync(PAUSE_MS.unfinished - PAUSE_MS.usual - SEGMENT_SILENCE_MS); expect(sent).toHaveLength(1); }); it('a noise that was not speech does not restart the pause', async () => { const { turns, sent } = gatherer(); turns.segment('Merge it now.'); await vi.advanceTimersByTimeAsync(400); turns.speechStart(); await vi.advanceTimersByTimeAsync(200); turns.speechMisfire(); await vi.advanceTimersByTimeAsync(PAUSE_MS.finished - SEGMENT_SILENCE_MS - 600); expect(sent).toEqual([{ text: 'Merge it now.', utterance: 'turn-1' }]); }); it('a slow transcript holds the turn until it is in, then sends at once', async () => { const { turns, sent } = gatherer({ ms: 4000 }); turns.segment(new Float32Array(1)); // audio: transcribed await vi.advanceTimersByTimeAsync(3999); expect(sent).toEqual([]); await vi.advanceTimersByTimeAsync(1); expect(sent).toHaveLength(1); }); it('only the page\'s own voice heard: nothing sent, and it says so; End drops a turn half heard', async () => { const { turns, sent, echoes } = gatherer(); turns.segment(Object.assign(new Float32Array(1), { toString: () => 'ECHO' })); await vi.advanceTimersByTimeAsync(5000); expect(sent).toEqual([]); expect(echoes).toHaveLength(1); turns.segment('Something'); turns.reset(); await vi.advanceTimersByTimeAsync(5000); expect(sent).toEqual([]); expect(turns.gathering()).toBe(false); }); }); describe('a turn, an acknowledgment, and more words', () => { const ACKS = ['Got it, one sec.', 'Okay, let me check.', 'On it.']; beforeEach(() => { routes['/api/talk'] = () => response({ stt: 'whisper', tts: 'elevenlabs', acks: ACKS }); routes.audio = (id, i) => response(`${id}#${i}`, { headers: { 'X-Pieces': '1', 'Content-Type': 'audio/mpeg' } }); }); afterEach(() => vi.useRealTimers()); const acksPlayed = () => played.filter(p => p.startsWith('ack-')); it('says a different acknowledgment the moment each turn ends, from audio fetched at the start', async () => { await talking(); await flush(); expect(requests.filter(r => /\/audio\/ack-\d\/0$/.test(r.url))).toHaveLength(3); const fetched = requests.length; vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] }); vadOpts.onSpeechEnd(new Float32Array(1600)); await vi.advanceTimersByTimeAsync(PAUSE_MS.finished - SEGMENT_SILENCE_MS); // At the end of the turn, from what was fetched: no request in between. expect(acksPlayed()).toHaveLength(1); expect(requests.slice(fetched).filter(r => r.url.includes('/audio/'))).toEqual([]); expect(stateWord()).toBe('speaking'); // The reply comes while it plays: it waits for the acknowledgment. talkHeard(reply('r1', 'm1')); await vi.advanceTimersByTimeAsync(0); expect(_talkInternals().playing).toMatch(/^ack-/); theAudio.end(); await vi.advanceTimersByTimeAsync(0); expect(played.at(-1)).toBe('r1#0'); theAudio.end(); await vi.advanceTimersByTimeAsync(0); routes['/api/talk/text'] = () => response({ ok: true, id: 'm2' }); for (let i = 0; i < 6; i++) { vadOpts.onSpeechEnd(new Float32Array(1600)); await vi.advanceTimersByTimeAsync(PAUSE_MS.unfinished); theAudio.end(); await vi.advanceTimersByTimeAsync(0); } const said = acksPlayed(); expect(said).toHaveLength(7); for (let i = 1; i < said.length; i++) expect(said[i]).not.toBe(said[i - 1]); }); it('more words before the reply continue the turn; its answer answers both', async () => { await talking(); await utterance(); expect(_talkInternals().awaiting).toEqual(['m1']); theAudio.end(); // the acknowledgment await flush(); routes['/api/talk/segment'] = () => response({ text: 'And the open PRs too.' }); routes['/api/talk/text'] = () => response({ ok: true, id: 'm2' }); await utterance(); expect(turnsSent()[1]).toMatchObject({ text: 'And the open PRs too.', continues: 'm1' }); expect(_talkInternals().awaiting).toEqual(['m1', 'm2']); theAudio.end(); await flush(); talkHeard(reply('r2', 'm2', 'The release is out, and two PRs are open.')); await flush(); expect(played.at(-1)).toBe('r2#0'); expect(_talkInternals().awaiting).toEqual([]); theAudio.end(); await flush(); expect(stateWord()).toBe('listening'); }); it('words merged into a turn Billion has not had yet stay that one turn', async () => { await talking(); await utterance(); theAudio.end(); await flush(); routes['/api/talk/text'] = () => response({ ok: true, id: 'm1', merged: true }); await utterance(); expect(turnsSent()[1].continues).toBe('m1'); expect(_talkInternals().awaiting).toEqual(['m1']); }); it('speaking over the acknowledgment stops it at once; the words continue the turn', async () => { await talking(); await utterance(); expect(_talkInternals().playing).toMatch(/^ack-/); const ack = _talkInternals().playing; vadOpts.onSpeechStart(); expect(theAudio.volume).toBeLessThan(1); // ducked the moment speech may have begun vadOpts.onSpeechRealStart(); expect(theAudio.paused).toBe(true); expect(_talkInternals().playing).toBe(null); routes['/api/talk/text'] = () => response({ ok: true, id: 'm2' }); await utterance(); expect(posts('/api/talk/segment')[1].init.headers['X-Echo-Of']).toBe(ack); expect(turnsSent()[1].continues).toBe('m1'); }); it('barge-in on the reply: playback stops, it keeps listening, and the words are sent', async () => { await talking(); await utterance(); theAudio.end(); await flush(); talkHeard(reply('r1', 'm1', 'A long answer. '.repeat(40))); await flush(); expect(_talkInternals().playing).toBe('r1'); vadOpts.onSpeechStart(); vadOpts.onSpeechRealStart(); expect(theAudio.paused).toBe(true); expect(stateWord()).toBe('listening'); expect(fakeVad.pause).not.toHaveBeenCalled(); routes['/api/talk/text'] = () => response({ ok: true, id: 'm2' }); await utterance(); expect(turnsSent()).toHaveLength(2); expect(turnsSent()[1].continues).toBeUndefined(); }); it('a noise while Billion speaks only ducks it for a moment', async () => { await talking(); await utterance(); vadOpts.onSpeechStart(); expect(theAudio.volume).toBe(0.3); vadOpts.onVADMisfire(); expect(theAudio.volume).toBe(1); expect(_talkInternals().playing).toMatch(/^ack-/); }); it('a reply that comes while the owner is still speaking waits until the turn is sent, then plays, with no acknowledgment', async () => { await talking(); await utterance(); theAudio.end(); // the acknowledgment await flush(); const acks = acksPlayed().length; vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] }); vadOpts.onSpeechStart(); vadOpts.onSpeechRealStart(); talkHeard(reply('r1', 'm1')); await vi.advanceTimersByTimeAsync(0); expect(_talkInternals().playing).toBe(null); // not over the owner expect(_talkInternals().replyQueue).toEqual(['r1']); routes['/api/talk/text'] = () => response({ ok: true, id: 'm2' }); vadOpts.onSpeechEnd(new Float32Array(1600)); await vi.advanceTimersByTimeAsync(PAUSE_MS.finished - SEGMENT_SILENCE_MS - 100); expect(_talkInternals().playing).toBe(null); // nor in their pause await vi.advanceTimersByTimeAsync(100); expect(_talkInternals().playing).toBe('r1'); expect(acksPlayed()).toHaveLength(acks); }); it('a reply held for a noise plays once it proves not to be speech', async () => { await talking(); await utterance(); theAudio.end(); await flush(); vadOpts.onSpeechStart(); talkHeard(reply('r1', 'm1')); await flush(); expect(_talkInternals().playing).toBe(null); vadOpts.onVADMisfire(); await flush(); expect(_talkInternals().playing).toBe('r1'); }); it('muting mid-speech still sends what was heard', async () => { await talking(); vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] }); vadOpts.onSpeechEnd(new Float32Array(1600)); vadOpts.onSpeechStart(); // more speech, then Mute before it ends setMuted(true); await vi.advanceTimersByTimeAsync(PAUSE_MS.unfinished); expect(turnsSent()).toEqual([{ utterance: expect.any(String), text: 'How is it going?' }]); }); it('fetches the acknowledgments one at a time', async () => { let open = 0, most = 0; routes.audio = async (id, i) => { open++; most = Math.max(most, open); await new Promise(r => setTimeout(r, 5)); open--; return response(`${id}#${i}`, { headers: { 'X-Pieces': '1' } }); }; await talking(); await new Promise(r => setTimeout(r, 40)); expect(requests.filter(r => r.url.includes('/audio/ack-'))).toHaveLength(3); expect(most).toBe(1); }); it('the bar says ElevenLabs speaks the replies', async () => { await talking(); expect(bar().querySelector('.talk-privacy').textContent).toMatch(/spoken by ElevenLabs/); }); }); describe('acknowledgments and turns, edge cases', () => { const ACKS = ['Got it, one sec.', 'Okay, let me check.']; afterEach(() => vi.useRealTimers()); it('after the answer, the next turn is a turn of its own, not a continuation', async () => { routes['/api/talk'] = () => response({ stt: 'whisper', tts: 'say', acks: [] }); await talking(); await utterance(); talkHeard(reply('r1', 'm1')); await flush(); theAudio.end(); await flush(); theAudio.end(); await flush(); routes['/api/talk/text'] = () => response({ ok: true, id: 'm2' }); await utterance(); expect(turnsSent()).toHaveLength(2); expect(turnsSent()[1].continues).toBeUndefined(); }); it('the browser\'s voice says the acknowledgment itself: no ack audio is fetched', async () => { const spoken = []; vi.stubGlobal('speechSynthesis', { speak: (u) => { spoken.push(u.text); }, cancel: vi.fn(), resume: vi.fn(), getVoices: () => [] }); vi.stubGlobal('SpeechSynthesisUtterance', class { constructor(t) { this.text = t; } }); routes['/api/talk'] = () => response({ stt: 'whisper', tts: null, ttsMissing: 'no say', acks: ACKS }); await talking(); await utterance(); expect(requests.filter(r => r.url.includes('/audio/ack-'))).toEqual([]); expect(spoken).toHaveLength(1); expect(ACKS).toContain(spoken[0]); expect(turnsSent()).toHaveLength(1); }); it('acknowledgment audio that could not be fetched: no acknowledgment, the turn still goes and the reply plays', async () => { routes['/api/talk'] = () => response({ stt: 'whisper', tts: 'elevenlabs', acks: ACKS }); routes.audio = (id, i) => (id.startsWith('ack-') ? response({ error: 'down' }, { status: 503 }) : response(`${id}#${i}`, { headers: { 'X-Pieces': '1' } })); await talking(); await flush(); await utterance(); expect(played.filter(p => p.startsWith('ack-'))).toEqual([]); expect(turnsSent()).toHaveLength(1); talkHeard(reply('r1', 'm1')); await flush(); expect(played).toEqual(['r1#0']); }); it('a stretch of speech that could not be transcribed says why and sends nothing', async () => { routes['/api/talk/segment'] = () => response({ error: 'Could not transcribe that; say it again.' }, { status: 400 }); await talking(); await utterance(); expect(turnsSent()).toEqual([]); expect(bar().textContent).toMatch(/Could not transcribe that/); }); });