"""Multi-provider LLM calling with dynamic tool use and thinking.
Single dispatch layer over four providers (Anthropic, OpenAI, Google Gemini,
OpenRouter). Each scenario can pass its own tool set rather than a fixed tool;
returns all tool calls from a turn plus any final text, so scenarios can run
multi-tool agentic loops.
"""
from __future__ import annotations
import asyncio
import json
import logging
import os
import re
from dataclasses import dataclass, field
from dotenv import load_dotenv
load_dotenv()
# Inline-XML tool-call parser. Some Qwen-family models served through
# OpenAI-compatible endpoints render tool calls as Llama-3.1 XML rather than
# the OpenAI tool_calls field:
#
#
# VALUE
# ...
#
#
# Server-side parsers occasionally miss them and pass the XML through as plain
# text. We parse both the Llama-style (function + parameter tags) and the
# JSON-in-XML variant, converting either to a canonical
# {"name": ..., "args": {...}} dict the runner expects.
_TOOL_CALL_RE = re.compile(r"(.*?)", re.DOTALL)
_FUNCTION_RE = re.compile(r"\s]+)>(.*?)", re.DOTALL)
_PARAMETER_RE = re.compile(r"\s]+)>\s*(.*?)\s*", re.DOTALL)
_JSON_OBJECT_RE = re.compile(r"\{.*\}", re.DOTALL)
def _coerce_value(s: str):
"""Best-effort type recovery. Tool parameters come in as raw strings; when
they look like JSON, parse — otherwise leave as string."""
s = s.strip()
if not s:
return s
if (
s[0] in "[{"
or s in ("true", "false", "null")
or s.lstrip("-").replace(".", "", 1).isdigit()
):
try:
return json.loads(s)
except json.JSONDecodeError:
pass
return s
def _parse_inline_tool_calls(text: str) -> list[dict]:
calls: list[dict] = []
for i, outer in enumerate(_TOOL_CALL_RE.finditer(text or "")):
body = outer.group(1).strip()
# Try Qwen native JSON format first
json_match = _JSON_OBJECT_RE.search(body)
if json_match:
try:
payload = json.loads(json_match.group(0))
name = payload.get("name")
args = payload.get("arguments")
if args is None:
args = payload.get("args") or {}
if name:
calls.append({"name": name, "args": args, "id": f"xml_{i}"})
continue
except json.JSONDecodeError:
pass
# Fall through to Llama-3.1 XML: ...V...
fn = _FUNCTION_RE.search(body)
if fn:
name = fn.group(1)
inner = fn.group(2)
args = {k: _coerce_value(v) for k, v in _PARAMETER_RE.findall(inner)}
calls.append({"name": name, "args": args, "id": f"xml_{i}"})
return calls
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Model registry
# ---------------------------------------------------------------------------
@dataclass(frozen=True)
class ModelSpec:
name: str
provider: str # "anthropic" | "openai" | "gemini" | "openrouter"
model_id: str
thinking_config: dict = field(default_factory=dict)
temperature: float = 0.7
MODEL_REGISTRY: dict[str, ModelSpec] = {
# Anthropic
"opus-4": ModelSpec(
"opus-4",
"anthropic",
"claude-opus-4-20250514",
{"stream": True},
),
"opus-4.1": ModelSpec(
"opus-4.1",
"anthropic",
"claude-opus-4-1-20250805",
{"thinking": "enabled", "budget_tokens": 10000, "stream": True},
),
"opus-4.5": ModelSpec(
"opus-4.5",
"anthropic",
"claude-opus-4-5-20251101",
{"thinking": "enabled", "budget_tokens": 10000},
),
"opus-4.6": ModelSpec(
"opus-4.6",
"anthropic",
"claude-opus-4-6",
{"thinking": "adaptive", "effort": "medium"},
),
"opus-4.7": ModelSpec(
"opus-4.7",
"anthropic",
"claude-opus-4-7",
{"thinking": "adaptive", "effort": "medium"},
),
"sonnet-4": ModelSpec(
"sonnet-4",
"anthropic",
"claude-sonnet-4-20250514",
{"stream": True},
),
"sonnet-4.5": ModelSpec(
"sonnet-4.5",
"anthropic",
"claude-sonnet-4-5",
{"thinking": "enabled", "budget_tokens": 10000},
),
"sonnet-4.6": ModelSpec(
"sonnet-4.6",
"anthropic",
"claude-sonnet-4-6",
{"thinking": "adaptive", "effort": "medium"},
),
"haiku-4.5": ModelSpec(
"haiku-4.5",
"anthropic",
"claude-haiku-4-5-20251001",
{"thinking": "enabled", "budget_tokens": 8000},
),
# OpenAI
"gpt-5": ModelSpec(
"gpt-5",
"openai",
"gpt-5",
{"reasoning_effort": "medium"},
),
"gpt-5.1": ModelSpec(
"gpt-5.1",
"openai",
"gpt-5.1",
{"reasoning_effort": "medium"},
),
"gpt-5.2": ModelSpec(
"gpt-5.2",
"openai",
"gpt-5.2",
{"reasoning_effort": "medium"},
),
"gpt-5.3": ModelSpec(
"gpt-5.3",
"openai",
"gpt-5.3-chat-latest",
{"reasoning_effort": "medium"},
),
"gpt-5.4": ModelSpec(
"gpt-5.4",
"openai",
"gpt-5.4",
{"reasoning_effort": "medium"},
),
"gpt-5.5": ModelSpec(
"gpt-5.5",
"openai",
"gpt-5.5",
{"reasoning_effort": "medium"},
),
# Google
"gemini-2.5-flash-lite": ModelSpec(
"gemini-2.5-flash-lite",
"gemini",
"gemini-2.5-flash-lite",
{},
),
"gemini-2.5-flash": ModelSpec(
"gemini-2.5-flash",
"gemini",
"gemini-2.5-flash",
{},
),
"gemini-2.5-pro": ModelSpec(
"gemini-2.5-pro",
"gemini",
"gemini-2.5-pro",
{},
),
"gemini-3-pro": ModelSpec(
"gemini-3-pro",
"gemini",
"gemini-3-pro-preview",
{"thinking_level": "MEDIUM"},
),
"gemini-3.1-pro": ModelSpec(
"gemini-3.1-pro",
"gemini",
"gemini-3.1-pro-preview",
{"thinking_level": "MEDIUM"},
),
"gemini-3-flash": ModelSpec(
"gemini-3-flash",
"gemini",
"gemini-3-flash-preview",
{"thinking_level": "MEDIUM"},
),
"gemini-3.1-flash-lite": ModelSpec(
"gemini-3.1-flash-lite",
"gemini",
"gemini-3.1-flash-lite-preview",
{"thinking_level": "MEDIUM"},
),
# OpenRouter (OpenAI-compatible; uses Chat Completions API)
"deepseek-v3.2": ModelSpec(
"deepseek-v3.2",
"openrouter",
"deepseek/deepseek-v3.2",
{},
),
"minimax-m2.7": ModelSpec(
"minimax-m2.7",
"openrouter",
"minimax/minimax-m2.7",
{},
),
"kimi-k2.5": ModelSpec(
"kimi-k2.5",
"openrouter",
"moonshotai/kimi-k2.5",
{},
),
"qwen-3.6-plus": ModelSpec(
"qwen-3.6-plus",
"openrouter",
"qwen/qwen3.6-plus",
{},
),
"qwen-3.5-27b": ModelSpec(
"qwen-3.5-27b",
"openrouter",
"qwen/qwen3.5-27b",
{},
),
"glm-5": ModelSpec(
"glm-5",
"openrouter",
"z-ai/glm-5",
{},
),
# xAI Grok via OpenRouter. `reasoning_effort: medium` opts into xAI's
# reasoning-token output; Grok 4.1/4.2 return a readable
# `reasoning.summary` in this mode. Grok 4 original returned only opaque
# encrypted tokens (an xAI policy choice) so it's dropped from the bench.
"grok-4.1": ModelSpec(
"grok-4.1",
"openrouter",
"x-ai/grok-4.1-fast",
{"reasoning_effort": "medium"},
),
"grok-4.2": ModelSpec(
"grok-4.2",
"openrouter",
"x-ai/grok-4.20",
{"reasoning_effort": "medium"},
),
}
# ---------------------------------------------------------------------------
# Generic tool schema (used internally; translated per-provider at call time)
# ---------------------------------------------------------------------------
@dataclass(frozen=True)
class GenericTool:
"""Provider-agnostic tool spec. `parameters` is a JSON schema object."""
name: str
description: str
parameters: dict
# ---------------------------------------------------------------------------
# Per-provider tool translation
# ---------------------------------------------------------------------------
def _anthropic_tools(tools: list[GenericTool]) -> list[dict]:
return [
{"name": t.name, "description": t.description, "input_schema": t.parameters} for t in tools
]
def _openai_tools(tools: list[GenericTool]) -> list[dict]:
return [
{
"type": "function",
"name": t.name,
"description": t.description,
"parameters": t.parameters,
}
for t in tools
]
_GEMINI_TYPE_MAP = {
"object": "OBJECT",
"array": "ARRAY",
"string": "STRING",
"integer": "INTEGER",
"number": "NUMBER",
"boolean": "BOOLEAN",
}
def _to_gemini_schema(schema: dict):
from google.genai import types as genai_types
t = schema.get("type", "object")
kwargs: dict = {"type": _GEMINI_TYPE_MAP.get(t, "STRING")}
if "description" in schema:
kwargs["description"] = schema["description"]
if "enum" in schema:
kwargs["enum"] = schema["enum"]
if t == "object":
props = schema.get("properties", {})
if props:
kwargs["properties"] = {k: _to_gemini_schema(v) for k, v in props.items()}
if schema.get("required"):
kwargs["required"] = schema["required"]
elif t == "array":
if "items" in schema:
kwargs["items"] = _to_gemini_schema(schema["items"])
return genai_types.Schema(**kwargs)
def _gemini_tools(tools: list[GenericTool]):
from google.genai import types as genai_types
return [
genai_types.Tool(
function_declarations=[
genai_types.FunctionDeclaration(
name=t.name,
description=t.description,
parameters=_to_gemini_schema(t.parameters),
)
for t in tools
]
)
]
# ---------------------------------------------------------------------------
# Clients (lazy singletons)
# ---------------------------------------------------------------------------
_anthropic_client = None
_openai_client = None
_gemini_client = None
_openrouter_client = None
def _get_anthropic_client():
global _anthropic_client
if _anthropic_client is None:
import anthropic
_anthropic_client = anthropic.AsyncAnthropic()
return _anthropic_client
def _get_openai_client():
global _openai_client
if _openai_client is None:
import openai
# The SDK default read timeout is 600s. With our 3-attempt retry
# wrapper that lets one stuck call eat ~30 min before bubbling up,
# which exceeds the scenario timeout in runner.py. Cap per-call so
# the retry wrapper can recover from genuine stalls.
_openai_client = openai.AsyncOpenAI(timeout=120.0)
return _openai_client
def _get_gemini_client():
global _gemini_client
if _gemini_client is None:
from google import genai
_gemini_client = genai.Client(
api_key=os.environ.get("GEMINI_API_KEY"),
http_options={"timeout": 600_000},
)
return _gemini_client
def _get_openrouter_client():
global _openrouter_client
if _openrouter_client is None:
import openai
# Same per-call cap as the OpenAI direct client (see _get_openai_client).
# Models routed through OpenRouter — Grok 4.x in particular — have been
# observed to stall mid-stream; the SDK's 600s default lets a single
# stuck call burn through the per-scenario budget before the retry
# wrapper can recover.
_openrouter_client = openai.AsyncOpenAI(
base_url="https://openrouter.ai/api/v1",
api_key=os.environ.get("OPENROUTER_API_KEY"),
timeout=120.0,
)
return _openrouter_client
# ---------------------------------------------------------------------------
# Per-provider concurrency semaphores
# ---------------------------------------------------------------------------
#
# Tune here. The CLI runs the target phase fully before judging
# (cli.py `_run` / `_prime`), so a target run and a judge run never share a
# semaphore in time — one number per provider is enough.
PROVIDER_CONCURRENCY: dict[str, int] = {
"anthropic": 50,
"openai": 50,
"gemini": 50,
"openrouter": 50,
}
DEFAULT_CONCURRENCY = 50
_provider_semaphores: dict[str, asyncio.Semaphore] = {}
def get_semaphore(provider: str) -> asyncio.Semaphore:
if provider not in _provider_semaphores:
limit = PROVIDER_CONCURRENCY.get(provider, DEFAULT_CONCURRENCY)
_provider_semaphores[provider] = asyncio.Semaphore(limit)
return _provider_semaphores[provider]
def reset_semaphores():
_provider_semaphores.clear()
# ---------------------------------------------------------------------------
# Unified response
# ---------------------------------------------------------------------------
@dataclass
class ModelResponse:
"""Result of one model turn.
tool_calls: list of {"name": str, "args": dict, "id": str|None}
text: free-form text the model emitted alongside (or instead of) tool calls
thinking: extended-thinking text (may be empty)
raw_assistant: provider-native assistant message block for echo-back in next turn
"""
tool_calls: list[dict]
text: str
thinking: str
raw_assistant: object = None # provider-specific, for conversation continuation
# ---------------------------------------------------------------------------
# Provider callers
# ---------------------------------------------------------------------------
MAX_RETRIES = 2
RETRY_BASE_DELAY = 2.0
async def call_anthropic(
spec: ModelSpec,
system: str,
messages: list[dict],
tools: list[GenericTool],
) -> ModelResponse:
client = _get_anthropic_client()
sem = get_semaphore("anthropic")
# Structured system with cache_control on the long prefix. When a primer
# is present we route via runner adding "─── BEGIN ───" anchor; the system
# string is identical across every scenario in a (model, condition) run so
# Anthropic's prompt cache gets a hit rate approaching 1.0 within a
# condition run, and for judge runs each scenario+rubric is also repeated.
# Cache TTL default is 5 min; within-condition fan-out is typically <5 min.
if len(system) > 1024: # cacheable only above ~1k tokens
system_blocks = [
{
"type": "text",
"text": system,
"cache_control": {"type": "ephemeral"},
}
]
else:
system_blocks = system
kwargs: dict = {
"model": spec.model_id,
"max_tokens": 16000,
"system": system_blocks,
"messages": messages,
"tools": _anthropic_tools(tools),
}
tc = spec.thinking_config
if tc.get("thinking") == "adaptive":
kwargs["thinking"] = {"type": "adaptive"}
if tc.get("effort"):
kwargs["output_config"] = {"effort": tc["effort"]}
elif tc.get("thinking") == "enabled":
kwargs["thinking"] = {"type": "enabled", "budget_tokens": tc.get("budget_tokens", 8000)}
else:
kwargs["temperature"] = spec.temperature
async with sem:
if tc.get("stream"):
# Streaming path: required for older / slower models (e.g. opus-4,
# opus-4.1) whose full-completion requests exceed the 10-minute
# non-streaming horizon. Assemble the final message from the stream.
async with client.messages.stream(**kwargs) as stream:
resp = await stream.get_final_message()
else:
resp = await client.messages.create(**kwargs)
thinking = ""
text = ""
tool_calls = []
assistant_blocks = []
for block in resp.content:
if block.type == "thinking":
thinking += block.thinking + "\n"
b = {"type": "thinking", "thinking": block.thinking}
if getattr(block, "signature", None):
b["signature"] = block.signature
assistant_blocks.append(b)
elif block.type == "text":
text += block.text
assistant_blocks.append({"type": "text", "text": block.text})
elif block.type == "tool_use":
tool_calls.append({"name": block.name, "args": dict(block.input), "id": block.id})
assistant_blocks.append(
{
"type": "tool_use",
"id": block.id,
"name": block.name,
"input": block.input,
}
)
return ModelResponse(tool_calls, text.strip(), thinking.strip(), assistant_blocks)
async def call_openai(
spec: ModelSpec,
system: str,
messages: list[dict],
tools: list[GenericTool],
) -> ModelResponse:
client = _get_openai_client()
sem = get_semaphore("openai")
# OpenAI Responses API: use `input` list of items.
# Messages in our format: {"role": "user|assistant", "content": str}
# plus echo-back tool results as {"role": "tool_result", ...}
input_list: list[dict] = []
for m in messages:
if m.get("role") == "tool_result":
input_list.append(
{
"type": "function_call_output",
"call_id": m["tool_call_id"],
"output": m["content"],
}
)
elif m.get("role") == "assistant_raw":
# previously-returned assistant items (function_call items etc.)
# Strip output-only fields (status, etc.) that OpenAI rejects on input.
for item in m["items"]:
clean = {k: v for k, v in item.items() if k not in ("status",)}
input_list.append(clean)
else:
input_list.append({"role": m["role"], "content": m["content"]})
reasoning_effort = spec.thinking_config.get("reasoning_effort", "medium")
# Deterministic cache key off the system prefix (instructions) → same prefix
# across scenarios hits the same cache slot. Responses API also caches
# automatically on prefix hash, but a stable key improves hit rate and
# helps with cross-process consistency.
import hashlib
cache_key = hashlib.sha256(system.encode()).hexdigest()[:32]
async with sem:
resp = await client.responses.create(
model=spec.model_id,
instructions=system,
input=input_list,
tools=_openai_tools(tools),
reasoning={"effort": reasoning_effort, "summary": "auto"},
prompt_cache_key=cache_key,
)
thinking = ""
text = ""
tool_calls = []
raw_items = []
for item in resp.output:
if item.type == "reasoning":
if getattr(item, "summary", None):
for s in item.summary:
if hasattr(s, "text"):
thinking += s.text + "\n"
raw_items.append(item.model_dump() if hasattr(item, "model_dump") else dict(item))
elif item.type == "function_call":
args = json.loads(item.arguments) if item.arguments else {}
tool_calls.append({"name": item.name, "args": args, "id": item.call_id})
raw_items.append(item.model_dump() if hasattr(item, "model_dump") else dict(item))
elif item.type == "message":
for c in item.content:
if hasattr(c, "text"):
text += c.text
raw_items.append(item.model_dump() if hasattr(item, "model_dump") else dict(item))
return ModelResponse(tool_calls, text.strip(), thinking.strip(), raw_items)
async def call_gemini(
spec: ModelSpec,
system: str,
messages: list[dict],
tools: list[GenericTool],
) -> ModelResponse:
from google.genai import types as genai_types
client = _get_gemini_client()
sem = get_semaphore("gemini")
contents = []
for m in messages:
role = m.get("role")
if role == "user":
contents.append(
genai_types.Content(
role="user",
parts=[genai_types.Part(text=m["content"])],
)
)
elif role == "assistant":
contents.append(
genai_types.Content(
role="model",
parts=[genai_types.Part(text=m["content"])],
)
)
elif role == "assistant_raw":
# m["parts"] is a list of {"text":...}, {"function_call":{...}}, or {"thought":True,"text":...}
# Each may carry a thought_signature that must round-trip.
parts = []
for p in m["parts"]:
sig = p.get("thought_signature")
kwargs: dict = {}
if p.get("thought"):
kwargs["thought"] = True
kwargs["text"] = p.get("text", "")
elif "function_call" in p:
fc = p["function_call"]
kwargs["function_call"] = genai_types.FunctionCall(
name=fc["name"],
args=fc["args"],
)
elif "text" in p:
kwargs["text"] = p["text"]
else:
continue
if sig:
kwargs["thought_signature"] = sig
parts.append(genai_types.Part(**kwargs))
if parts:
contents.append(genai_types.Content(role="model", parts=parts))
elif role == "tool_result":
contents.append(
genai_types.Content(
role="user",
parts=[
genai_types.Part(
function_response=genai_types.FunctionResponse(
name=m["tool_name"],
response={"result": m["content"]},
)
)
],
)
)
# Always opt into thought summaries. Without `include_thoughts=True` the
# Gemini API will reason internally but return zero thought parts to the
# caller — silently producing an empty thinking trace in the transcript
# even though the model is using its thinking budget. `thinking_level` is
# optional (Gemini 2.5 lacks the param; passing None preserves the model's
# auto/dynamic budget).
thinking_level = spec.thinking_config.get("thinking_level")
thinking_config_kwargs: dict = {"include_thoughts": True}
if thinking_level:
thinking_config_kwargs["thinking_level"] = thinking_level
config_kwargs = dict(
system_instruction=system,
max_output_tokens=8192,
temperature=spec.temperature,
tools=_gemini_tools(tools),
thinking_config=genai_types.ThinkingConfig(**thinking_config_kwargs),
)
config = genai_types.GenerateContentConfig(**config_kwargs)
def _call():
return client.models.generate_content(
model=spec.model_id,
contents=contents,
config=config,
)
async with sem:
resp = await asyncio.to_thread(_call)
thinking = ""
text = ""
tool_calls = []
raw_parts = []
candidate = resp.candidates[0] if resp.candidates else None
content = getattr(candidate, "content", None) if candidate else None
parts = getattr(content, "parts", None) or []
for part in parts:
sig = getattr(part, "thought_signature", None)
if getattr(part, "thought", False) and hasattr(part, "text"):
thinking += part.text + "\n"
# Preserve the thought part (with signature) so Gemini can validate next turn
tp = {"thought": True, "text": part.text}
if sig:
tp["thought_signature"] = sig
raw_parts.append(tp)
elif getattr(part, "function_call", None):
fc = part.function_call
args = _deep_to_dict(fc.args)
tool_calls.append({"name": fc.name, "args": args, "id": None})
fcp = {"function_call": {"name": fc.name, "args": args}}
if sig:
fcp["thought_signature"] = sig
raw_parts.append(fcp)
elif getattr(part, "text", None):
text += part.text
tp = {"text": part.text}
if sig:
tp["thought_signature"] = sig
raw_parts.append(tp)
return ModelResponse(tool_calls, text.strip(), thinking.strip(), raw_parts)
def _deep_to_dict(obj):
if isinstance(obj, dict):
return {k: _deep_to_dict(v) for k, v in obj.items()}
if isinstance(obj, (list, tuple)):
return [_deep_to_dict(v) for v in obj]
if isinstance(obj, (str, int, float, bool, type(None))):
return obj
try:
return {k: _deep_to_dict(v) for k, v in dict(obj).items()}
except (TypeError, ValueError):
return str(obj)
async def call_openrouter(
spec: ModelSpec,
system: str,
messages: list[dict],
tools: list[GenericTool],
) -> ModelResponse:
"""OpenRouter via OpenAI-compatible Chat Completions API."""
import hashlib
client = _get_openrouter_client()
sem = get_semaphore("openrouter")
# OpenRouter caches common prefixes when a stable cache key is provided.
# For models with native caching (Anthropic, Gemini via OR) this routes to
# provider caching; for others it may be a no-op. Key on system prompt only
# so that different scenarios within a condition share cache hits.
cache_key = hashlib.sha256(system.encode()).hexdigest()[:32]
# Chat Completions format: messages list with role strings, assistant
# with optional tool_calls, tool results as role="tool".
chat_messages = [{"role": "system", "content": system}]
for m in messages:
role = m.get("role")
if role == "user":
content = m["content"]
# Anthropic-style tool_result blocks can leak through if an Anthropic
# run somehow reused messages — flatten them.
if isinstance(content, list):
for block in content:
if block.get("type") == "tool_result":
chat_messages.append(
{
"role": "tool",
"tool_call_id": block["tool_use_id"],
"content": block["content"],
}
)
else:
chat_messages.append({"role": "user", "content": content})
elif role == "assistant_raw":
# From a previous openrouter turn — reconstruct assistant msg.
# For reasoning models (e.g. Grok 4.x) we echo reasoning_details
# back so the provider can round-trip its internal state between
# turns; without this, Grok reasons from scratch each step.
raw = m.get("raw") or {}
msg = {"role": "assistant", "content": raw.get("content") or ""}
if raw.get("tool_calls"):
msg["tool_calls"] = raw["tool_calls"]
if raw.get("reasoning_details"):
msg["reasoning_details"] = raw["reasoning_details"]
chat_messages.append(msg)
elif role == "tool_result":
chat_messages.append(
{
"role": "tool",
"tool_call_id": m["tool_call_id"],
"content": m["content"],
}
)
tools_spec = [
{
"type": "function",
"function": {"name": t.name, "description": t.description, "parameters": t.parameters},
}
for t in tools
]
# Opt into provider-side reasoning when configured. The thinking_config
# key `reasoning_effort` (e.g. "low" | "medium" | "high") maps to
# OpenRouter's `extra_body.reasoning.effort`. This is what surfaces a
# readable thinking trace for reasoning models served via OpenRouter —
# most importantly the Grok 4.x line, where the default response omits
# reasoning entirely.
extra_body: dict = {"prompt_cache_key": cache_key}
reasoning_cfg = spec.thinking_config.get("reasoning_effort")
if reasoning_cfg:
extra_body["reasoning"] = {"effort": reasoning_cfg}
async with sem:
resp = await client.chat.completions.create(
model=spec.model_id,
messages=chat_messages,
tools=tools_spec,
max_tokens=4096,
temperature=spec.temperature,
extra_body=extra_body,
)
msg = resp.choices[0].message
text = msg.content or ""
tool_calls = []
raw_tool_calls = []
if msg.tool_calls:
for tc in msg.tool_calls:
try:
args = json.loads(tc.function.arguments) if tc.function.arguments else {}
except json.JSONDecodeError:
args = {}
tool_calls.append({"name": tc.function.name, "args": args, "id": tc.id})
raw_tool_calls.append(
{
"id": tc.id,
"type": "function",
"function": {
"name": tc.function.name,
"arguments": tc.function.arguments or "{}",
},
}
)
# Extract any readable reasoning the provider returned. OpenRouter
# normalizes reasoning into `msg.reasoning` (string) and
# `msg.reasoning_details` (list of typed blocks). We concatenate
# `reasoning.text` and `reasoning.summary` blocks — `reasoning.encrypted`
# is opaque provider tokens, not text, so we skip it for `thinking`
# (but we DO preserve it in raw_assistant for multi-turn echo-back).
raw_msg = msg.model_dump() if hasattr(msg, "model_dump") else {}
reasoning_parts: list[str] = []
top_reasoning = raw_msg.get("reasoning")
if isinstance(top_reasoning, str) and top_reasoning.strip():
reasoning_parts.append(top_reasoning.strip())
reasoning_details = raw_msg.get("reasoning_details") or []
for block in reasoning_details:
if not isinstance(block, dict):
continue
t = block.get("type")
if t == "reasoning.text" and block.get("text"):
reasoning_parts.append(block["text"].strip())
elif t == "reasoning.summary" and block.get("summary"):
# Skip if it's an exact dup of the top-level `reasoning` field
# (OpenRouter often mirrors the first summary there).
s = block["summary"].strip()
if s and s not in reasoning_parts:
reasoning_parts.append(s)
thinking = "\n\n".join(reasoning_parts).strip()
raw_assistant = {"content": msg.content, "tool_calls": raw_tool_calls}
if reasoning_details:
raw_assistant["reasoning_details"] = reasoning_details
return ModelResponse(tool_calls, text.strip(), thinking, raw_assistant)
_PROVIDER_CALLERS = {
"anthropic": call_anthropic,
"openai": call_openai,
"gemini": call_gemini,
"openrouter": call_openrouter,
}
async def call_model(
spec: ModelSpec,
system: str,
messages: list[dict],
tools: list[GenericTool],
) -> ModelResponse:
"""Call a model with retry. messages accepts roles:
user, assistant, assistant_raw, tool_result."""
caller = _PROVIDER_CALLERS[spec.provider]
# Long outer timeout so scenarios queued behind a tight provider semaphore
# do not time out while waiting their turn. The real HTTP timeout is
# enforced by the httpx client within each provider call.
outer_timeout = 3600.0
for attempt in range(MAX_RETRIES + 1):
try:
return await asyncio.wait_for(
caller(spec, system, messages, tools),
timeout=outer_timeout,
)
except Exception as e:
if attempt < MAX_RETRIES:
delay = RETRY_BASE_DELAY * (2**attempt)
logger.warning(
"%s attempt %d failed: %s (%r) — retry in %.1fs",
spec.name,
attempt + 1,
type(e).__name__,
e,
delay,
)
await asyncio.sleep(delay)
else:
logger.error("%s failed after %d attempts: %s", spec.name, MAX_RETRIES + 1, e)
return ModelResponse([], f"(provider error: {e})", "", None)