import time import os from typing import TYPE_CHECKING if TYPE_CHECKING: from avatars.base_avatar import BaseAvatar from utils.logger import logger # Built-in LLM providers. Each exposes an OpenAI-compatible chat completions # endpoint; the active one is selected with --llm_provider (default: dashscope). LLM_PROVIDERS = { "dashscope": { "api_key_env": "DASHSCOPE_API_KEY", "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1", "default_model": "qwen-plus", }, "orcarouter": { "api_key_env": "ORCAROUTER_API_KEY", "base_url": "https://api.orcarouter.ai/v1", "default_model": "orcarouter/auto", }, } def _llm_provider(opt) -> str: """Return the configured provider name, defaulting to dashscope.""" return getattr(opt, 'llm_provider', 'dashscope') or 'dashscope' def _llm_client(opt): """Create the OpenAI-compatible client for the configured provider.""" from openai import OpenAI cfg = LLM_PROVIDERS.get(_llm_provider(opt), LLM_PROVIDERS['dashscope']) return OpenAI( api_key=os.getenv(cfg['api_key_env']), base_url=cfg['base_url'], ) def _llm_model(opt) -> str: """Resolve the model name, falling back to the provider default.""" cfg = LLM_PROVIDERS.get(_llm_provider(opt), LLM_PROVIDERS['dashscope']) return getattr(opt, 'llm_model', '') or cfg['default_model'] def llm_response(message,avatar_session:'BaseAvatar',datainfo:dict={}): try: opt = avatar_session.opt start = time.perf_counter() client = _llm_client(opt) model = _llm_model(opt) end = time.perf_counter() logger.info(f"llm Time init: {end-start}s,{message}") completion = client.chat.completions.create( model=model, messages=[{'role': 'system', 'content': '你是一个知识助手,尽量以简短、口语化的方式输出'}, {'role': 'user', 'content': message}], stream=True, # Display token usage in the last line of the streamed response. stream_options={"include_usage": True} ) result="" first = True for chunk in completion: if len(chunk.choices)>0: #print(chunk.choices[0].delta.content) if first: end = time.perf_counter() logger.info(f"llm Time to first chunk: {end-start}s") first = False msg = chunk.choices[0].delta.content if msg is None: continue lastpos=0 #msglist = re.split('[,.!;:,。!?]',msg) for i, char in enumerate(msg): if char in ",.!;:,。!?:;" : result = result+msg[lastpos:i+1] lastpos = i+1 if len(result)>10: logger.info(result) avatar_session.put_msg_txt(result,datainfo) result="" result = result+msg[lastpos:] end = time.perf_counter() logger.info(f"llm Time to last chunk: {end-start}s") if result: avatar_session.put_msg_txt(result,datainfo) except Exception as e: logger.exception('llm exceptiopn:') return