import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState, } from 'react' import { buildConversationTurns, discardUserTranscript, mergeConversationHistory, upsertAssistantTranscript, upsertUserTranscript, } from './message-order.js' import MessageContent from './MessageContent.jsx' import MultimodalComposer from './MultimodalComposer.jsx' import DesktopFluidOrb from './DesktopFluidOrb.jsx' import DesktopSpriteOrb from './DesktopSpriteOrb.jsx' import DomainLibraryPanel from './DomainLibraryPanel.jsx' import { desktopOrbClassName, resolveOrbVisualState } from './orb-presentation.js' import { isBuiltinOrbSkin, } from '../../shared/orb-skin-catalog.mjs' import { supportsComposerInput } from '../../shared/client-input-capabilities.mjs' import { resultLabel } from './presentation.js' import { setRuntimeLanguage, t } from './i18n.js' import { removeDeliveredTask, removeTaskInPhase, taskDeliverySettled, taskDetail, taskNeedsPresentation, taskLabel, taskView, } from './task-view.js' import useRealtimeVoice, { realtimeModelStatus, realtimeProviderForConnection, realtimeProviderSelection, shouldClaimReleasedVoice, } from './useRealtimeVoice.js' import { requestedSessionId } from './session.js' import { initialVoiceEnabled } from './voice-defaults.js' import { applyDesktopClientState, desktopCanFinishWaking, desktopCanHide, desktopHideDeadline, desktopTasksActive, desktopWorkSettled, desktopTasksWorking, performDesktopClientAction, } from './desktop-hide.js' import { desktopTaskCards, } from './desktop-task-cards.js' import { advanceDesktopRuntimePresentation, desktopBackendRuntime, desktopRealtimeRuntime, resolveDesktopRuntime, } from './desktop-runtime.js' import { spriteAnimationEventForGatewayEvent, spriteAnimationForEvent, } from './sprite-orb.js' import { applyDesktopClientSettings, initialDesktopClientSettings, } from './desktop-client-settings.js' const desktopOrbMode = ( new URLSearchParams(window.location.search).get('desktop') === 'orb' ) const initialDesktopSurfaceMode = ( new URLSearchParams(window.location.search).get('surface') === 'panel' ? 'panel' : 'orb' ) const composerEnabled = supportsComposerInput(desktopOrbMode ? 'desktop' : 'web') function getSessionId() { const requested = requestedSessionId(window.location.search) if (requested) { localStorage.setItem('qwen-audio-agent.session', requested) return requested } const current = localStorage.getItem('qwen-audio-agent.session') if (current) return current const created = crypto.randomUUID() localStorage.setItem('qwen-audio-agent.session', created) return created } function labelFor(state) { return { idle: t('待命'), listening: t('正在听'), processing: t('正在处理'), speaking: t('正在说'), working: t('正在处理任务'), starting: t('正在启动'), connecting: t('正在连接语音前台'), occupied: t('其他入口正在使用'), hidden: t('已隐藏'), waking: t('正在显示'), }[state] || state } function frontendLabel(holder) { return holder?.label || { desktop: t('桌面端'), cli: t('终端'), web: 'WebUI', }[holder?.type] || t('其他入口') } function OrbControlIcon({ type, muted = false, collapsed = false }) { if (type === 'microphone') { return } if (type === 'settings') { return } if (type === 'conversation') { return } if (type === 'collapse') { return } if (type === 'tasks') { return } return } function upsertTask(items, taskId, update, fallback) { const index = items.findIndex(item => item.id === taskId) if (index < 0) return fallback ? [...items, fallback] : items const next = [...items] next[index] = update(next[index]) return next } export default function App() { const [desktopClientSettings, setDesktopClientSettings] = useState( () => initialDesktopClientSettings(window.location.search), ) const { orbSkinId, autoHideSeconds, wakeWordEnabled, } = desktopClientSettings // `t()` reads the module-level runtime language. Keeping a revision in // React state makes a language-only settings update repaint this surface // without replacing its Gateway WebSocket or Realtime Session. const [, setLanguageRevision] = useState(0) const [sessionId, setSessionId] = useState(getSessionId) const [voiceEnabled, setVoiceEnabled] = useState(() => initialVoiceEnabled({ desktopOrbMode, })) const [waitingForVoice, setWaitingForVoice] = useState(false) const [messages, setMessages] = useState([]) const [activity, setActivity] = useState(t('正在检查后台 Agent')) const [frontend, setFrontend] = useState({ label: 'Realtime Agent' }) const [realtimeProviders, setRealtimeProviders] = useState([]) const [realtimeProvider, setRealtimeProvider] = useState( () => localStorage.getItem('qwen-audio-agent.realtimeProvider') || '', ) const [modelStatus, setModelStatus] = useState(() => realtimeModelStatus()) const [providerNotice, setProviderNotice] = useState('') const [healthValidated, setHealthValidated] = useState(false) const [gatewayRuntime, setGatewayRuntime] = useState('connecting') const [backend, setBackend] = useState({ label: 'Agent', enabled: null, ready: false, status: 'starting', code: null, }) const [agentTasks, setAgentTasks] = useState([]) const [desktopTasksCollapsed, setDesktopTasksCollapsed] = useState(false) const [showDomainLibrary, setShowDomainLibrary] = useState(false) const [desktopTaskLayout, setDesktopTaskLayout] = useState({ placement: 'below', orbOffsetX: 0, }) const [orbDragging, setOrbDragging] = useState(false) const [orbDragDirection, setOrbDragDirection] = useState('') const [spriteAnimationCues, setSpriteAnimationCues] = useState([]) const [spriteOrbFailed, setSpriteOrbFailed] = useState(false) const [desktopLifecycle, setDesktopLifecycle] = useState('active') const [desktopSurfaceMode, setDesktopSurfaceMode] = useState( initialDesktopSurfaceMode, ) const [lastInteractionAt, setLastInteractionAt] = useState(Date.now) const activeVoiceResponse = useRef('') const currentTurnId = useRef('') const responseTurnMap = useRef(new Map()) const agentTurnIds = useRef(new Set()) const taskDismissTimers = useRef(new Map()) const messagesRef = useRef(null) const stickToBottom = useRef(true) const orbDrag = useRef(null) const spriteAnimationCueId = useRef(0) const runtimeReadyAnnounced = useRef(false) const previousTasksActive = useRef(false) const workSettledAtRef = useRef(Date.now()) const autoHideStateRef = useRef(null) const autoHideRequestedDeadlineRef = useRef(0) const lastWakeAtRef = useRef(0) const previousDesktopLifecycle = useRef('active') const gatewayCommandsRef = useRef(null) const sessionIdRef = useRef(sessionId) sessionIdRef.current = sessionId const spriteAnimationCue = spriteAnimationCues[0] || null useEffect(() => { if (!desktopOrbMode) return undefined const bridge = window.qwenAudioAgentDesktop if (typeof bridge?.onClientSettings !== 'function') return undefined return bridge.onClientSettings(settings => { setDesktopClientSettings(current => applyDesktopClientSettings( current, settings, )) if (settings.orbSkin) setSpriteOrbFailed(false) if (settings.language) { setRuntimeLanguage(settings.language) setLanguageRevision(value => value + 1) } }) }, []) useEffect(() => { const persistSession = window.qwenAudioAgentDesktop?.setConversationSession if (!desktopOrbMode || typeof persistSession !== 'function') return void persistSession(sessionId).catch(() => {}) }, [sessionId]) const noteInteraction = useCallback(() => { setLastInteractionAt(Date.now()) }, []) const changeDesktopSurface = useCallback(async mode => { const bridge = window.qwenAudioAgentDesktop if (!desktopOrbMode || !bridge?.setSurface) return try { const result = await bridge.setSurface(mode) setDesktopSurfaceMode(result?.mode === 'panel' ? 'panel' : 'orb') noteInteraction() } catch { // A rejected host transition leaves the current presentation intact. } }, [noteInteraction]) const triggerSpriteAnimation = useCallback((eventName, { priority = false } = {}) => { if (!desktopOrbMode || isBuiltinOrbSkin(orbSkinId)) return const name = spriteAnimationForEvent(eventName) if (!name) return spriteAnimationCueId.current += 1 const cue = { id: spriteAnimationCueId.current, name } setSpriteAnimationCues(current => ( priority ? [cue, ...current] : [...current, cue] )) }, [orbSkinId]) const completeSpriteAnimationCue = useCallback(id => { setSpriteAnimationCues(current => ( current[0]?.id === id ? current.slice(1) : current )) }, []) const respondToPermission = useCallback(async (taskId, permission, decision) => { if (!permission?.id || permission.submitting) return setAgentTasks(items => upsertTask( items, taskId, task => ({ ...task, authorization: { ...task.authorization, submitting: true, error: null, }, }), )) try { await gatewayCommandsRef.current?.respondPermission(permission.id, decision) } catch (error) { if (['permission_not_found', 'task_not_found'].includes(error.code)) { setAgentTasks(items => upsertTask( items, taskId, task => ({ ...task, authorization: null }), )) return } setAgentTasks(items => upsertTask( items, taskId, task => ({ ...task, authorization: task.authorization ? { ...task.authorization, submitting: false, error: t('没有提交成功:{message}', { message: error.message }), } : null, }), )) } }, []) useLayoutEffect(() => { const container = messagesRef.current if (container && stickToBottom.current) { container.scrollTop = container.scrollHeight } }, [messages, agentTasks]) useEffect(() => () => { taskDismissTimers.current.forEach(timer => clearTimeout(timer)) taskDismissTimers.current.clear() }, []) useEffect(() => { let cancelled = false let refreshTimer const refresh = () => fetch('api/health', { cache: 'no-store' }) .then(async response => ({ response, payload: await response.json() })) .then(({ response, payload }) => { if (cancelled) return const gatewayReady = response.ok && payload.ok !== false const backendPayload = payload.backend || {} const backendEnabled = backendPayload.enabled !== false && Boolean( backendPayload.kind || backendPayload.protocol, ) const label = payload.backend?.label || payload.backend?.kind || 'Agent' setFrontend({ label: payload.realtimeLabel || payload.realtimeProvider || 'Realtime Agent', }) setRealtimeProviders(payload.realtimeProviders || []) setModelStatus(realtimeModelStatus(payload)) // A front end persisted by an earlier visit may no longer exist on this // server (removed provider, different deployment). Sending it would be // refused on every connect, so the stale selection is dropped in favour // of the server default instead of leaving the client stuck. setRealtimeProvider(current => { const selection = realtimeProviderSelection(current, payload) setProviderNotice(selection.notice) if (selection.provider !== current) { localStorage.removeItem('qwen-audio-agent.realtimeProvider') } return selection.provider }) setGatewayRuntime(gatewayReady ? 'ready' : 'failed') setHealthValidated(gatewayReady) setBackend({ label, enabled: backendEnabled, ready: gatewayReady && ( backendEnabled ? backendPayload.ok === true : true ), status: backendPayload.status || ( backendEnabled ? 'starting' : 'not_configured' ), code: backendPayload.code || null, error: backendPayload.error || '', url: payload.backend?.uiPath || payload.backend?.baseUrl || '', }) setActivity(response.ok ? t('Gateway 已连接') : t('能力服务尚未连接')) if (desktopOrbMode) { const backendSettled = !backendEnabled || [ 'ready', 'failed', ].includes(backendPayload.status) refreshTimer = setTimeout(refresh, backendSettled ? 3000 : 500) } }) .catch(() => { if (cancelled) return setGatewayRuntime('failed') setHealthValidated(false) setActivity(t('qwen-audio-agent Gateway 尚未连接')) if (desktopOrbMode) refreshTimer = setTimeout(refresh, 1000) }) refresh() return () => { cancelled = true clearTimeout(refreshTimer) } }, []) const updateUserTranscript = useCallback((event, final = false) => { const id = event.turnId ? `user:${event.turnId}` : crypto.randomUUID() setMessages(items => upsertUserTranscript(items, { id, content: event.content, turnId: event.turnId, final, })) if (final) noteInteraction() }, [noteInteraction]) const updateVoiceMessage = useCallback((event, final = false) => { const responseId = event.responseId || activeVoiceResponse.current if (!responseId) return activeVoiceResponse.current = responseId const id = `voice:${responseId}` const trackedTurnId = responseTurnMap.current.get(responseId) || event.turnId || currentTurnId.current setMessages(items => upsertAssistantTranscript(items, { id, content: event.content, turnId: trackedTurnId, taskId: event.taskId, taskIds: event.taskIds, origin: event.origin, citations: event.citations, final, })) }, []) const onRealtimeEvent = useCallback(event => { const animationEvent = spriteAnimationEventForGatewayEvent(event) if (animationEvent) { triggerSpriteAnimation(animationEvent) } if (event.type === 'turn.started') { currentTurnId.current = event.turnId || '' activeVoiceResponse.current = '' stickToBottom.current = true setActivity(t('正在听你说')) } if (event.type === 'gateway.disconnected') { setActivity(t('qwen-audio-agent Gateway 已断开,正在重连')) setAgentTasks(items => items.map(task => ( [ 'queued', 'running', 'delegated', 'finalizing', 'cancelling', 'responding', ].includes(task.phase) ? { ...task, phase: 'disconnected' } : task ))) } void applyDesktopClientState(event, { desktop: desktopOrbMode, bridge: window.qwenAudioAgentDesktop, onLifecycle: setDesktopLifecycle, lastWakeAt: lastWakeAtRef.current, }).catch(() => {}) if ( event.type === 'voice.sleep' && event.state === 'detected' && desktopOrbMode ) { window.qwenAudioAgentDesktop?.wake() } if (event.type === 'session.recovered') { if (sessionIdRef.current !== sessionId) return setMessages(items => mergeConversationHistory(items, event.messages || [])) const serverTasks = event.tasks || [] const byId = new Map(serverTasks.map(task => [task.id, task])) setAgentTasks(items => { const known = new Set(items.map(task => task.id)) const reconciled = items.flatMap(task => { const current = byId.get(task.id) if (current && taskDeliverySettled(current)) return [] if (current) return [taskView(current, task)] if (task.phase !== 'disconnected') return [task] return [{ ...task, phase: 'failed', error: t('网关重连后未找到这次后台执行,请重新提交。'), }] }) serverTasks .filter(task => taskNeedsPresentation(task) && !known.has(task.id)) .reverse() .forEach(task => reconciled.push(taskView(task))) return reconciled }) } if (event.type === 'voice.deactivated') { setVoiceEnabled(false) setWaitingForVoice(false) setActivity(t('{holder}正在使用语音', { holder: frontendLabel(event.holder) })) } if ( event.type === 'voice.ownership' && event.state === 'busy' && voiceEnabled ) { setVoiceEnabled(false) setWaitingForVoice(false) setActivity(t('{holder}正在使用语音', { holder: frontendLabel(event.holder) })) } if (event.type === 'voice.ownership' && event.state === 'available') { if (shouldClaimReleasedVoice(event, waitingForVoice)) { setWaitingForVoice(false) setVoiceEnabled(true) setActivity(t('正在接入语音')) } else if (!voiceEnabled) { setActivity(t('待命')) } } if (event.type === 'voice.state') { if ( event.turnId && event.turnId !== currentTurnId.current && event.origin === 'model' ) return if (event.state === 'listening') setActivity(t('正在听你说')) if (event.state === 'processing' && !agentTurnIds.current.has(currentTurnId.current)) { setActivity(t('正在处理')) } if (event.state === 'idle' && !agentTurnIds.current.has(currentTurnId.current)) { setActivity(t('待命')) } } if (event.type === 'transcript.delta' && event.role === 'user') { updateUserTranscript(event) } if (event.type === 'transcript.final' && event.role === 'user') { updateUserTranscript(event, true) } if (event.type === 'transcript.discard' && event.role === 'user') { setMessages(items => discardUserTranscript(items, event.turnId)) } if (event.type === 'response.started') { activeVoiceResponse.current = event.responseId if (event.turnId) { responseTurnMap.current.set(event.responseId, event.turnId) if (responseTurnMap.current.size > 100) { responseTurnMap.current.delete(responseTurnMap.current.keys().next().value) } } if ( event.turnId === currentTurnId.current && !agentTurnIds.current.has(event.turnId) ) { setActivity(t('正在回复')) } } if (event.type === 'transcript.delta' && event.role === 'assistant') updateVoiceMessage(event) if (event.type === 'transcript.final' && event.role === 'assistant') updateVoiceMessage(event, true) if (event.type === 'response.interrupted') { const id = `voice:${event.responseId}` setMessages(items => items.map(message => ( message.id === id ? { ...message, interrupted: true, live: false } : message ))) } if (event.type === 'task.accepted') { const task = event.task if (task.turnId) agentTurnIds.current.add(task.turnId) if (!task.turnId || task.turnId === currentTurnId.current) { setActivity(t('正在处理')) } setAgentTasks(items => upsertTask( items, task.id, current => taskView(task, current), taskView(task), )) } if (event.type === 'task.running') { const task = event.task if (task.turnId) agentTurnIds.current.add(task.turnId) if (!task.turnId || task.turnId === currentTurnId.current) { setActivity(t('正在处理')) } setAgentTasks(items => upsertTask( items, task.id, current => ({ ...current, elapsedMs: task.elapsedMs || 0, phase: 'running', }), { id: task.id, kind: task.kind, objective: task.objective, createdAt: task.createdAt, startedAt: task.startedAt, elapsedMs: task.elapsedMs || 0, phase: 'running', turnId: task.turnId, }, )) } if (event.type === 'task.progress') { const progress = event.task if (!progress.turnId || progress.turnId === currentTurnId.current) { setActivity(t('正在处理 · {seconds} 秒', { seconds: Math.round(progress.elapsedMs / 1000) })) } setAgentTasks(items => upsertTask( items, progress.id, task => taskView(progress, task), taskView(progress), )) } if (event.type === 'task.updated') { const task = event.task setAgentTasks(items => upsertTask( items, task.id, current => taskView(task, current), taskView(task), )) } if (event.type === 'task.delegated') { const task = event.task if (!task.turnId || task.turnId === currentTurnId.current) { setActivity(t('进行中')) } setAgentTasks(items => upsertTask( items, task.id, current => taskView(task, current), taskView(task), )) } if ( event.type === 'task.finalizing' || event.type === 'task.cancelling' ) { const task = event.task if (!task.turnId || task.turnId === currentTurnId.current) { setActivity(event.type === 'task.finalizing' ? t('正在整理项目结果') : t('正在取消')) } setAgentTasks(items => upsertTask( items, task.id, current => taskView(task, current), taskView(task), )) } if ( event.type === 'task.permission.requested' || event.type === 'task.permission.resolved' ) { const task = event.task if (event.type === 'task.permission.requested') { setActivity(t('等待你的确认')) } else { setActivity(t('正在继续处理')) } setAgentTasks(items => upsertTask( items, task.id, current => taskView(task, current), taskView(task), )) } if (event.type === 'task.completed') { const completed = event.task if (completed.turnId) agentTurnIds.current.delete(completed.turnId) if (!completed.turnId || completed.turnId === currentTurnId.current) { setActivity(t('正在准备回复')) } setAgentTasks(items => upsertTask( items, completed.id, task => taskView(completed, task), taskView(completed), )) } if (event.type === 'task.notification.delivered') { const delivered = event.task // Delivery is acknowledged after playback ends. The assistant transcript // may already have removed this card, so never upsert it again here. setAgentTasks(items => removeDeliveredTask(items, delivered.id)) } if (event.type === 'task.failed') { const failed = event.task if (failed.turnId) agentTurnIds.current.delete(failed.turnId) if (!failed.turnId || failed.turnId === currentTurnId.current) { setActivity(t('后台失败:{error}', { error: failed.error })) } setAgentTasks(items => upsertTask( items, failed.id, task => ({ ...taskView(failed, task), phase: 'failed' }), { ...taskView(failed), phase: 'failed' }, )) } if (event.type === 'task.cancelled') { const cancelled = event.task if (cancelled.turnId) agentTurnIds.current.delete(cancelled.turnId) if (!cancelled.turnId || cancelled.turnId === currentTurnId.current) { setActivity(t('已取消')) } setAgentTasks(items => upsertTask( items, cancelled.id, task => ({ ...taskView(cancelled, task), phase: 'cancelled' }), { ...taskView(cancelled), phase: 'cancelled' }, )) clearTimeout(taskDismissTimers.current.get(cancelled.id)) taskDismissTimers.current.set(cancelled.id, setTimeout(() => { setAgentTasks(items => removeTaskInPhase( items, cancelled.id, 'cancelled', )) setActivity(current => current === t('已取消') ? t('待命') : current) taskDismissTimers.current.delete(cancelled.id) }, 3000)) } if (event.type === 'transcript.final' && event.role === 'assistant') { if (event.turnId === currentTurnId.current) setActivity(t('待命')) const presentedTaskIds = new Set( event.taskIds?.length ? event.taskIds : [event.taskId].filter(Boolean), ) setAgentTasks(items => items.filter(task => ( !presentedTaskIds.has(task.id) || !['responding', 'completed'].includes(task.phase) ))) } }, [ sessionId, updateUserTranscript, updateVoiceMessage, voiceEnabled, waitingForVoice, triggerSpriteAnimation, ]) // Keep the microphone alive while the desktop orb is hidden and the wake // word is enabled, even if the user has muted the realtime conversation. // Microphone mute leaves output playback active; wake-word detection still // needs a live input stream to resume on "你好千问" while hidden. const voiceEnabledForWakeWord = ( desktopOrbMode && desktopLifecycle === 'hidden' && wakeWordEnabled ) const voice = useRealtimeVoice({ sessionId, enabled: voiceEnabled || voiceEnabledForWakeWord, suspended: desktopOrbMode && desktopLifecycle === 'hidden' && !wakeWordEnabled, outputMuted: false, // WebUI and desktop share one control contract: the toggle only changes // microphone capture and never closes or interrupts the output stream. inputOnlyMute: true, wakeWordOnly: voiceEnabledForWakeWord, clientType: desktopOrbMode ? 'desktop' : 'web', clientLabel: desktopOrbMode ? t('桌面端') : 'WebUI', clientStates: desktopOrbMode ? ['sleeping'] : [], realtimeProvider: realtimeProviderForConnection( realtimeProvider, healthValidated, ), onEvent: onRealtimeEvent, onInputError: message => { setVoiceEnabled(false) setWaitingForVoice(false) setActivity(message) }, onClientAction: event => performDesktopClientAction(event, { desktop: desktopOrbMode, bridge: window.qwenAudioAgentDesktop, onLifecycle: setDesktopLifecycle, }), onWakeWordAudio: (audio, sampleRate) => { window.qwenAudioAgentDesktop?.acceptWakeWordAudio(audio, sampleRate) }, }) gatewayCommandsRef.current = voice const lifecycleTransition = ( desktopOrbMode && desktopLifecycle !== 'active' ) const voiceConnectionError = ( !lifecycleTransition && voice.connectionState === 'unavailable' ) const desktopRuntime = resolveDesktopRuntime({ gateway: gatewayRuntime, realtime: desktopRealtimeRuntime(voice.connectionState), backend: desktopBackendRuntime(backend), }) const desktopHasWorkingTasks = desktopOrbMode && desktopTasksWorking(agentTasks) // 统一视觉状态仲裁:生命周期 → 异常 → 对话态 → 后台态。 // 后台工作态仅在桌面悬浮球展示;等待授权由播报和任务卡片承载, // 不占用 Agent 动画状态。WebUI 也由任务卡片承载同类信息。 const orbVisualState = resolveOrbVisualState({ lifecycle: desktopLifecycle, runtimeState: desktopOrbMode ? desktopRuntime.overall : null, connectionError: !desktopOrbMode && voiceConnectionError, connecting: !desktopOrbMode && voiceEnabled && voice.connectionState === 'connecting', ownershipBusy: voice.ownership.state === 'busy', voiceState: voice.visualState || voice.state, tasksWorking: desktopHasWorkingTasks, }) const authorizationTask = agentTasks.find( task => task.authorization?.status === 'pending', ) useEffect(() => { if (!desktopOrbMode) return const current = desktopRuntime.overall const presentation = advanceDesktopRuntimePresentation({ current, readyAnnounced: runtimeReadyAnnounced.current, }) runtimeReadyAnnounced.current = presentation.readyAnnounced if (presentation.cue) triggerSpriteAnimation(presentation.cue) }, [desktopRuntime.overall, triggerSpriteAnimation]) const desktopCards = useMemo( () => desktopOrbMode ? desktopTaskCards(agentTasks) : [], [agentTasks], ) useEffect(() => { if (!desktopCards.length) setDesktopTasksCollapsed(false) }, [desktopCards.length]) useEffect(() => { if (!desktopOrbMode) return undefined window.qwenAudioAgentDesktop?.loadSurface?.() .then(result => setDesktopSurfaceMode( result?.mode === 'panel' ? 'panel' : 'orb', )) .catch(() => {}) return undefined }, []) useEffect(() => { if (!desktopOrbMode) return undefined return window.qwenAudioAgentDesktop?.onTaskCardPlacement?.( setDesktopTaskLayout, ) }, []) useEffect(() => { if (!desktopOrbMode) return undefined window.qwenAudioAgentDesktop?.setTaskCardCount( desktopSurfaceMode === 'panel' ? desktopCards.length : desktopTasksCollapsed ? 0 : desktopCards.length, ) return undefined }, [desktopCards.length, desktopSurfaceMode, desktopTasksCollapsed]) useEffect(() => { if (!desktopOrbMode) return undefined return () => window.qwenAudioAgentDesktop?.setTaskCardCount(0) }, []) const ownershipLabel = voice.ownership.holder ? frontendLabel(voice.ownership.holder) : '' const workSettled = desktopWorkSettled({ tasks: agentTasks, voiceState: voice.visualState || voice.state, }) const tasksActive = desktopTasksActive(agentTasks) useEffect(() => { if (!desktopOrbMode) return if (!tasksActive && previousTasksActive.current) { const settledAt = Date.now() workSettledAtRef.current = settledAt } previousTasksActive.current = tasksActive }, [tasksActive]) useEffect(() => { if (!desktopOrbMode) return undefined const bridge = window.qwenAudioAgentDesktop if (!bridge) return undefined const applyLifecycle = lifecycle => { if (!lifecycle?.state) return if ( lifecycle.state === 'waking' && previousDesktopLifecycle.current !== 'waking' ) { triggerSpriteAnimation('wake', { priority: true }) } previousDesktopLifecycle.current = lifecycle.state setDesktopLifecycle(lifecycle.state) if (lifecycle.state === 'waking') lastWakeAtRef.current = Date.now() if (lifecycle.reason === 'activity') noteInteraction() if (lifecycle.state === 'hidden') { // Main has already collapsed a visible conversation panel before an // explicit sleep. Mirror that authoritative surface transition so a // later wake cannot render the panel inside the compact orb window. setDesktopSurfaceMode('orb') setActivity(t('已隐藏')) } if (lifecycle.state === 'waking') setActivity(t('正在显示悬浮球')) if (lifecycle.state === 'active' && lifecycle.reason === 'ready') { setActivity(t('待命')) noteInteraction() } } const dispose = bridge.onLifecycle(applyLifecycle) bridge.loadLifecycle().then(applyLifecycle).catch(() => {}) const onInteraction = () => noteInteraction() window.addEventListener('pointerdown', onInteraction) window.addEventListener('keydown', onInteraction) return () => { dispose() window.removeEventListener('pointerdown', onInteraction) window.removeEventListener('keydown', onInteraction) } }, [noteInteraction, triggerSpriteAnimation]) useEffect(() => { if (!desktopOrbMode || desktopLifecycle !== 'waking') return // Presence readiness describes whether the desktop surface can finish // waking, not whether microphone capture has initialized. Keeping those // lifecycles separate prevents a slow/denied microphone from leaving the // orb permanently in `waking`, which would also disable inactivity sleep. if (desktopCanFinishWaking(voice.connectionState)) { window.qwenAudioAgentDesktop?.lifecycleReady() } }, [ desktopLifecycle, voice.connectionState, ]) // 快捷键/托盘唤起恢复 Gateway presence;Realtime 连接在休眠期间保持。 const wakeGateway = voice.wake const publishClientEvent = voice.publishClientEvent useEffect(() => { if (!desktopOrbMode || desktopLifecycle !== 'waking') return wakeGateway() }, [desktopLifecycle, wakeGateway]) autoHideStateRef.current = { desktopLifecycle, desktopSurfaceMode, lastInteractionAt, connectionState: voice.connectionState, visualError: voice.visualError, workSettled, } useEffect(() => { if (!desktopOrbMode || autoHideSeconds === 0) return undefined const check = () => { const current = autoHideStateRef.current if (!current || current.desktopSurfaceMode === 'panel') return if (!desktopCanHide({ settled: current.workSettled, connectionState: current.connectionState, visualError: current.visualError, lifecycle: current.desktopLifecycle, })) return const deadline = desktopHideDeadline({ lastInteractionAt: current.lastInteractionAt, workSettledAt: workSettledAtRef.current, timeoutSeconds: autoHideSeconds, }) if ( Date.now() < deadline || autoHideRequestedDeadlineRef.current === deadline ) return if (publishClientEvent('desktop.presence.sleep_requested', { idle_ms: autoHideSeconds * 1000, })) { autoHideRequestedDeadlineRef.current = deadline } } const timer = setInterval(check, 1_000) check() return () => clearInterval(timer) }, [autoHideSeconds, publishClientEvent]) // Switching the front end reconnects on its own: realtimeProvider is part of // the realtime effect's dependencies, so changing it tears the current socket // down and connects again with the newly selected provider. const selectRealtimeProvider = value => { const selection = realtimeProviderSelection(value, { realtimeModel: modelStatus.id, realtimeModelProfile: modelStatus.id ? { id: modelStatus.id } : null, realtimeProviders, }) setRealtimeProvider(selection.provider) setProviderNotice(selection.notice) if (selection.provider) { localStorage.setItem( 'qwen-audio-agent.realtimeProvider', selection.provider, ) } else { localStorage.removeItem('qwen-audio-agent.realtimeProvider') } } const inputModeLabels = { text: t('文字'), audio: t('语音'), image: t('图片'), video: t('视频'), observation: t('画面观察'), nativeVideo: t('原生视频'), } const modeList = modes => modes.map(mode => inputModeLabels[mode]).join(' / ') const resetSession = () => { taskDismissTimers.current.forEach(timer => clearTimeout(timer)) taskDismissTimers.current.clear() const next = crypto.randomUUID() localStorage.setItem('qwen-audio-agent.session', next) setSessionId(next) setMessages([]) setAgentTasks([]) currentTurnId.current = '' activeVoiceResponse.current = '' responseTurnMap.current.clear() agentTurnIds.current.clear() setActivity(t('已创建新会话')) } const enableVoice = () => { if (!voice.activateAudio()) return if (voice.ownership.state === 'busy') { setWaitingForVoice(true) setActivity(t('等待{holder}释放语音', { holder: ownershipLabel || t('其他入口') })) return } setWaitingForVoice(false) setVoiceEnabled(true) } const disableVoice = () => { setWaitingForVoice(false) setVoiceEnabled(false) setActivity(t('待命')) } const sendComposerInput = parts => { // Sending is a browser user gesture, so it is also the earliest reliable // point to unlock audio playback while the microphone remains muted. voice.activateAudio() return voice.sendInput(parts) } const turns = useMemo( () => buildConversationTurns(messages, agentTasks), [messages, agentTasks], ) const beginOrbDrag = event => { const bridge = window.qwenAudioAgentDesktop if (!desktopOrbMode || event.button !== 0 || !bridge) return event.currentTarget.setPointerCapture?.(event.pointerId) orbDrag.current = { pointerId: event.pointerId, lastX: event.screenX, } setOrbDragging(true) setOrbDragDirection('') bridge.dragStart(event.screenX, event.screenY) } const moveOrb = event => { const drag = orbDrag.current if (!drag || drag.pointerId !== event.pointerId) return const deltaX = event.screenX - drag.lastX if (Math.abs(deltaX) >= 2) { setOrbDragDirection(deltaX > 0 ? 'right' : 'left') drag.lastX = event.screenX } window.qwenAudioAgentDesktop?.dragMove(event.screenX, event.screenY) } const endOrbDrag = event => { const drag = orbDrag.current if (!drag || drag.pointerId !== event.pointerId) return orbDrag.current = null setOrbDragging(false) setOrbDragDirection('') window.qwenAudioAgentDesktop?.dragEnd() } const handleVoiceOrbClick = () => { if (voice.state === 'speaking') { voice.interrupt() return } enableVoice() } if (desktopOrbMode && desktopSurfaceMode === 'orb') { return
triggerSpriteAnimation('hover')} onPointerDown={beginOrbDrag} onPointerMove={moveOrb} onPointerUp={endOrbDrag} onPointerCancel={endOrbDrag} > {isBuiltinOrbSkin(orbSkinId) || spriteOrbFailed ? ( ) : ( setSpriteOrbFailed(true)} /> )}
{desktopCards.length > 0 && !desktopTasksCollapsed &&
{desktopCards.map(task => { const detail = taskDetail(task) const title = task.delegation?.title || task.objective || taskLabel(task) const progress = ['completed', 'failed', 'cancelled'].includes(task.phase) ? taskLabel(task) : detail && detail !== title ? detail : taskLabel(task) const plan = task.activity?.findLast(item => item.kind === 'plan') const progressRatio = ['completed', 'failed', 'cancelled'].includes(task.phase) ? 1 : plan?.total > 0 ? plan.completed / plan.total : null return
{title}