""" Agentic sampling loop that calls the Claude API and local implementation of anthropic-defined computer use tools. """ import platform from collections.abc import Callable from datetime import datetime from enum import StrEnum from typing import Any, Literal, cast import httpx2 as httpx from anthropic import ( Anthropic, AnthropicBedrock, AnthropicVertex, APIError, APIResponseValidationError, APIStatusError, omit, ) from anthropic.types.beta import ( BetaCacheControlEphemeralParam, BetaContentBlockParam, BetaImageBlockParam, BetaMessage, BetaMessageParam, BetaTextBlock, BetaTextBlockParam, BetaToolResultBlockParam, BetaToolUseBlockParam, ) from .tools import ( TOOL_GROUPS_BY_VERSION, ToolCollection, ToolResult, ToolVersion, ) from .tools.base import ToolFailure # The tool_result content a toolset member call receives when an earlier # member call in the same turn failed (the exact text is part of the computer # toolset contract). NOT_EXECUTED_ERROR = "Not executed: an earlier computer action in this turn failed." PROMPT_CACHING_BETA_FLAG = "prompt-caching-2024-07-31" # How the model's reasoning is requested. Which modes a given model supports is # declared per-model in streamlit.py (MODEL_TO_MODEL_CONF): # - "adaptive": the model decides how much to think, steered by an effort level # (Claude Opus 4.6 / Sonnet 4.6 and newer). Sent as # thinking={"type": "adaptive"} plus output_config={"effort": ...}. # - "extended": a fixed thinking token budget (Claude Opus 4.5 / Sonnet 4.5 and # older). Sent as thinking={"type": "enabled", "budget_tokens": ...}. # - "off": no thinking parameters are sent. ThinkingMode = Literal["adaptive", "extended", "off"] ThinkingEffort = Literal["low", "medium", "high", "max"] class APIProvider(StrEnum): ANTHROPIC = "anthropic" BEDROCK = "bedrock" VERTEX = "vertex" # This system prompt is optimized for the Docker environment in this repository and # specific tool combinations enabled. # We encourage modifying this system prompt to ensure the model has context for the # environment it is running in, and to provide any additional information that may be # helpful for the task at hand. SYSTEM_PROMPT = f""" * You are utilising an Ubuntu virtual machine using {platform.machine()} architecture with internet access. * You can feel free to install Ubuntu applications with your bash tool. Use curl instead of wget. * To open firefox, please just click on the firefox icon. Note, firefox-esr is what is installed on your system. * Using bash tool you can start GUI applications, but you need to set export DISPLAY=:1 and use a subshell. For example "(DISPLAY=:1 xterm &)". GUI apps run with bash tool will appear within your desktop environment, but they may take some time to appear. Take a screenshot to confirm it did. * When using your bash tool with commands that are expected to output very large quantities of text, redirect into a tmp file and use str_replace_based_edit_tool or `grep -n -B -A ` to confirm output. * When viewing a page it can be helpful to zoom out so that you can see everything on the page. Either that, or make sure you scroll down to see everything before deciding something isn't available. * When using your computer function calls, they take a while to run and send back to you. Where possible/feasible, try to chain multiple of these calls all into one function calls request. * The current date is {datetime.today().strftime("%A, %B %-d, %Y")}. * When using Firefox, if a startup wizard appears, IGNORE IT. Do not even click "skip this step". Instead, click on the address bar where it says "Search or enter address", and enter the appropriate search term or URL there. * If the item you are looking at is a pdf, if after taking a single screenshot of the pdf it seems that you want to read the entire document instead of trying to continue to read the pdf from your screenshots + navigation, determine the URL, use curl to download the pdf, install and use pdftotext to convert it to a text file, and then read that text file directly with your str_replace_based_edit_tool. """ async def sampling_loop( *, model: str, provider: APIProvider, system_prompt_suffix: str, messages: list[BetaMessageParam], output_callback: Callable[[BetaContentBlockParam], None], tool_output_callback: Callable[[ToolResult, str], None], api_response_callback: Callable[ [httpx.Request, httpx.Response | object | None, Exception | None], None ], api_key: str, only_n_most_recent_images: int | None = None, max_tokens: int = 4096, tool_version: ToolVersion, thinking_mode: ThinkingMode = "off", thinking_effort: ThinkingEffort = "medium", thinking_budget: int | None = None, token_efficient_tools_beta: bool = False, ): """ Agentic sampling loop for the assistant/tool interaction of computer use. """ tool_group = TOOL_GROUPS_BY_VERSION[tool_version] tool_collection = ToolCollection(*(ToolCls() for ToolCls in tool_group.tools)) system = BetaTextBlockParam( type="text", text=f"{SYSTEM_PROMPT}{' ' + system_prompt_suffix if system_prompt_suffix else ''}", ) while True: enable_prompt_caching = False betas = [tool_group.beta_flag] if tool_group.beta_flag else [] if token_efficient_tools_beta: betas.append("token-efficient-tools-2025-02-19") image_truncation_threshold = only_n_most_recent_images or 0 if provider == APIProvider.ANTHROPIC: client = Anthropic(api_key=api_key, max_retries=4) enable_prompt_caching = True elif provider == APIProvider.VERTEX: client = AnthropicVertex() elif provider == APIProvider.BEDROCK: client = AnthropicBedrock() if enable_prompt_caching: betas.append(PROMPT_CACHING_BETA_FLAG) _inject_prompt_caching(messages) # Because cached reads are 10% of the price, we don't think it's # ever sensible to break the cache by truncating images only_n_most_recent_images = 0 # Use type ignore to bypass TypedDict check until SDK types are updated system["cache_control"] = {"type": "ephemeral"} # type: ignore if only_n_most_recent_images: _maybe_filter_to_n_most_recent_images( messages, only_n_most_recent_images, min_removal_threshold=image_truncation_threshold, ) extra_body = {} if thinking_mode == "adaptive": # Newer models (Claude Opus 4.6 / Sonnet 4.6 and later) reason # adaptively: the model decides how much to think and the effort # level steers it. These models do not accept a manual token budget. extra_body = { "thinking": {"type": "adaptive"}, "output_config": {"effort": thinking_effort}, } elif thinking_mode == "extended" and thinking_budget: # Older models use extended thinking with an explicit token budget. # Ensure we only send the required fields for thinking. extra_body = { "thinking": {"type": "enabled", "budget_tokens": thinking_budget} } # Call the API # we use raw_response to provide debug information to streamlit. Your # implementation may be able call the SDK directly with: # `response = client.messages.create(...)` instead. try: raw_response = client.beta.messages.with_raw_response.create( max_tokens=max_tokens, messages=messages, model=model, system=[system], tools=tool_collection.to_params(), # The toolset group has no beta flag, so on Vertex/Bedrock the # list can be empty; omit the header rather than send it blank. betas=betas or omit, extra_body=extra_body, ) except (APIStatusError, APIResponseValidationError) as e: api_response_callback(e.request, e.response, e) return messages except APIError as e: api_response_callback(e.request, e.body, e) return messages api_response_callback( raw_response.http_response.request, raw_response.http_response, None ) response = raw_response.parse() response_params = _response_to_params(response) messages.append( { "role": "assistant", "content": response_params, } ) tool_result_content: list[BetaToolResultBlockParam] = [] member_call_failed = False for content_block in response_params: output_callback(content_block) if ( isinstance(content_block, dict) and content_block.get("type") == "tool_use" ): # Type narrowing for tool use blocks tool_use_block = cast(BetaToolUseBlockParam, content_block) # A toolset member call carries toolset_name naming its # family; its tool_result must carry the same toolset_name. toolset_name = cast( str | None, cast(dict[str, Any], content_block).get("toolset_name"), ) if toolset_name is not None and member_call_failed: # Toolset member calls run sequentially in block order and # stop on the first failure; the remaining member calls # are answered without being executed. result: ToolResult = ToolFailure(error=NOT_EXECUTED_ERROR) else: result = await tool_collection.run( name=tool_use_block["name"], tool_input=cast( dict[str, Any], tool_use_block.get("input", {}) ), toolset_name=toolset_name, ) if toolset_name is not None and result.error: member_call_failed = True tool_result_content.append( _make_api_tool_result( result, tool_use_block["id"], toolset_name=toolset_name ) ) tool_output_callback(result, tool_use_block["id"]) if not tool_result_content: return messages messages.append({"content": tool_result_content, "role": "user"}) def _maybe_filter_to_n_most_recent_images( messages: list[BetaMessageParam], images_to_keep: int, min_removal_threshold: int, ): """ With the assumption that images are screenshots that are of diminishing value as the conversation progresses, remove all but the final `images_to_keep` tool_result images in place, with a chunk of min_removal_threshold to reduce the amount we break the implicit prompt cache. """ if images_to_keep is None: return messages tool_result_blocks = cast( list[BetaToolResultBlockParam], [ item for message in messages # BetaMessageParam is a union per role and only the system variant # has optional content, so rule it out before indexing "content". if message["role"] != "system" and isinstance(message["content"], list) for item in message["content"] if isinstance(item, dict) and item.get("type") == "tool_result" ], ) total_images = sum( 1 for tool_result in tool_result_blocks for content in tool_result.get("content", []) if isinstance(content, dict) and content.get("type") == "image" ) images_to_remove = total_images - images_to_keep # for better cache behavior, we want to remove in chunks images_to_remove -= images_to_remove % min_removal_threshold for tool_result in tool_result_blocks: if isinstance(tool_result.get("content"), list): new_content = [] for content in tool_result.get("content", []): if isinstance(content, dict) and content.get("type") == "image": if images_to_remove > 0: images_to_remove -= 1 continue new_content.append(content) tool_result["content"] = new_content def _response_to_params( response: BetaMessage, ) -> list[BetaContentBlockParam]: res: list[BetaContentBlockParam] = [] for block in response.content: if isinstance(block, BetaTextBlock): if block.text: res.append(BetaTextBlockParam(type="text", text=block.text)) elif getattr(block, "type", None) == "thinking": # Handle thinking blocks - include signature field thinking_block = { "type": "thinking", "thinking": getattr(block, "thinking", None), } if hasattr(block, "signature"): thinking_block["signature"] = getattr(block, "signature", None) res.append(cast(BetaContentBlockParam, thinking_block)) else: # Handle tool use blocks normally res.append(cast(BetaToolUseBlockParam, block.model_dump())) return res def _inject_prompt_caching( messages: list[BetaMessageParam], ): """ Set cache breakpoints for the 3 most recent turns one cache breakpoint is left for tools/system prompt, to be shared across sessions """ breakpoints_remaining = 3 for message in reversed(messages): if message["role"] == "user" and isinstance( content := message["content"], list ): if breakpoints_remaining: breakpoints_remaining -= 1 # Use type ignore to bypass TypedDict check until SDK types are updated content[-1]["cache_control"] = BetaCacheControlEphemeralParam( # type: ignore {"type": "ephemeral"} ) else: if isinstance(content[-1], dict) and "cache_control" in content[-1]: del content[-1]["cache_control"] # type: ignore # we'll only every have one extra turn per loop break def _make_api_tool_result( result: ToolResult, tool_use_id: str, *, toolset_name: str | None = None ) -> BetaToolResultBlockParam: """Convert an agent ToolResult to an API ToolResultBlockParam. A tool_result answering a toolset member call must carry the same toolset_name as its paired tool_use block, so it is stamped here. """ tool_result_content: list[BetaTextBlockParam | BetaImageBlockParam] | str = [] is_error = False if result.error: is_error = True tool_result_content = _maybe_prepend_system_tool_result(result, result.error) else: if result.output: tool_result_content.append( { "type": "text", "text": _maybe_prepend_system_tool_result(result, result.output), } ) if result.base64_image: tool_result_content.append( { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": result.base64_image, }, } ) tool_result_block: dict[str, Any] = { "type": "tool_result", "content": tool_result_content, "tool_use_id": tool_use_id, "is_error": is_error, } if toolset_name is not None: tool_result_block["toolset_name"] = toolset_name return cast(BetaToolResultBlockParam, tool_result_block) def _maybe_prepend_system_tool_result(result: ToolResult, result_text: str): if result.system: result_text = f"{result.system}\n{result_text}" return result_text