{ "meta": { "name": "best-of-Agent-Harnesses", "description": "Hand-curated, ranked list of AI agent harnesses, orchestration frameworks, and harness techniques.", "url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses", "site_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/", "llms_txt_url": "https://raw.githubusercontent.com/RyanAlberts/best-of-Agent-Harnesses/main/llms.txt", "jsonld_url": "https://raw.githubusercontent.com/RyanAlberts/best-of-Agent-Harnesses/main/harnesses.jsonld", "feed_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/feed.json", "license": "CC-BY-SA-4.0", "stars_captured": "2026-08-23", "project_count": 160, "graveyard_count": 4, "tiers": [ "super simple", "mostly simple", "slightly complex", "complex" ], "tier_help": "Adoption surface area, least to most: tier_rank 1 = format-only/single concept, 4 = platform with its own runtime and ecosystem.", "autonomy_tiers": [ "step-gated", "checkpoint-gated", "bounded", "headless" ], "autonomy_help": "Designed autonomy regime, least to most: rank 1 = human approves each action, 4 = built for unattended runs and fleets. rank 0 / 'n/a' = doesn't own an agent loop.", "recovery_tiers": [ "none", "retry", "resumable", "durable" ], "recovery_help": "Behavior when a run dies mid-task, weakest to strongest: rank 1 = start over, 4 = persisted execution state survives restarts. rank 0 / 'n/a' = doesn't execute.", "deep_dive_vocab": { "tooling_sandboxing": [ "none", "basic", "strong" ], "context_memory": [ "none", "basic", "strong" ], "lifecycle_hooks": [ "none", "partial", "full" ], "prompt_optimization": [ "none", "configurable", "native" ], "build_vs_buy_labels": { "1": "build", "2": "blueprint", "3": "managed" } }, "deep_dive_help": "Researched attributes per attributes/RUBRIC.md: tooling_sandboxing (how it acts), context_memory (how it remembers), lifecycle_hooks (before/after interception), prompt_optimization (refinement support) — rank 3 strongest, 0 unknown — plus build_vs_buy tier (1 build / 2 blueprint / 3 managed, categorical). deep_dive: null = not researched (outside runtime-harness scope)." }, "categories": [ { "id": "progressive-disclosure", "title": "Progressive disclosure harnesses", "subtitle": "Formats, runtimes, and patterns that reveal context, tools, or instructions in layers—index first, details on demand—to control tokens and improve agent focus (the \"map, not encyclopedia\" principle)." }, { "id": "coding-agent-products", "title": "Coding agent products (IDEs, CLIs, full suites)", "subtitle": "Turnkey coding agents you install and run: IDE extensions, terminal CLIs, Dockerized workspaces. Each entry notes which part is the harness (the agent loop, tool wiring, approval model) versus the UI shell (VS Code extension, TUI, browser client)." }, { "id": "coding-harness-configs", "title": "Coding harness configs and SDKs", "subtitle": "Skill packs, slash-command libraries, meta-prompting frameworks, and official SDKs that give you the harness (the agent loop, planning, memory, hooks) without bundling a specific IDE or CLI shell." }, { "id": "personal-agent-runtimes", "title": "Personal agent runtimes", "subtitle": "Always-on, self-hosted agents you run as a daemon and talk to from chat apps: gateway runtimes, second brains, and self-improving assistants. The agent as a product you operate, not a library you build with." }, { "id": "frameworks", "title": "Frameworks", "subtitle": "General-purpose agent and LLM application frameworks (the app layer, not harnesses per se)." }, { "id": "multi-agent", "title": "Multi-agent and orchestration", "subtitle": "Harnesses and patterns for multi-agent coordination and handoffs." }, { "id": "plugins-mcp-cli", "title": "Plugins, MCPs, CLI tools", "subtitle": "IDE plugins, concrete MCP servers, and CLI tools that give agents tools and context." }, { "id": "memory", "title": "Memory and state", "subtitle": "Persistent memory layers that give agents recall across turns and sessions: knowledge graphs, vector stores, and session-capture tools that survive a restart. The state a harness needs but rarely ships with." }, { "id": "evaluation", "title": "Evaluation and benchmarking harnesses", "subtitle": "Agentic eval systems, reasoning benchmarks, and open agent benchmarks." }, { "id": "observability", "title": "Observability and eval-ops", "subtitle": "Tracing, monitoring, and production evaluation for live agent runs: capture every step, tool call, and token, then score and debug in the loop. Distinct from the fixed-task benchmarks above—this is what you run against your own traffic." }, { "id": "research-task", "title": "Research and task-specific harnesses", "subtitle": "Deep research, document QA, and domain-specific agent loops." }, { "id": "libraries-sdks", "title": "Libraries and SDKs", "subtitle": "Lightweight runtimes, tool loops, and provider-agnostic harness primitives." } ], "use_cases": [ { "intent": "I want a turnkey coding agent today", "picks": [ "anomalyco/opencode", "cline/cline", "openai/codex", "google-gemini/gemini-cli", "OpenHands/OpenHands", "charmbracelet/crush", "RooCodeInc/Roo-Code" ], "category_title": "Coding agent products (IDEs, CLIs, full suites)" }, { "intent": "I want an always-on personal agent that lives in my chat apps", "picks": [ "openclaw/openclaw", "NousResearch/hermes-agent", "khoj-ai/khoj", "agent0ai/agent-zero", "HKUDS/OpenHarness" ], "category_title": "Personal agent runtimes" }, { "intent": "I want to extend Claude Code, Codex, or OpenCode with skills and slash commands", "picks": [ "anthropics/skills", "wshobson/agents", "obra/superpowers", "garrytan/gstack", "RyanAlberts/pmstack" ], "category_title": "Coding harness configs and SDKs" }, { "intent": "I want to build my own coding harness from scratch", "picks": [ "anthropics/claude-agent-sdk-python", "google/adk-python", "aiming-lab/AutoHarness", "SWE-agent/SWE-agent", "QuantaAlpha/RepoMaster", "HarnessLab/claw-code-agent" ], "category_title": "Coding harness configs and SDKs" }, { "intent": "I want a drop-in memory layer for agents", "picks": [ "mem0ai/mem0", "getzep/graphiti", "thedotmack/claude-mem", "RyanAlberts/agentlog", "letta-ai/letta" ], "category_title": "Plugins, MCPs, CLI tools" }, { "intent": "I want to plug hundreds to thousands of tools without context bloat", "picks": [ "xfey/MCP-Zero", "Reason-Wang/ToolGen", "antl3x/ToolRAG", "langchain-ai/langgraph-bigtool" ], "category_title": "Progressive disclosure harnesses" }, { "intent": "I want multi-agent orchestration", "picks": [ "openai/openai-agents-python", "crewAIInc/crewAI", "microsoft/autogen", "microsoft/agent-framework", "MervinPraison/PraisonAI", "2FastLabs/agent-squad" ], "category_title": "Multi-agent and orchestration" }, { "intent": "I want a general LLM app framework", "picks": [ "langchain-ai/langgraph", "langchain-ai/langchain", "run-llama/llama_index", "pydantic/pydantic-ai", "agno-agi/agno" ], "category_title": "Frameworks" }, { "intent": "I want low-code / visual workflows", "picks": [ "langflow-ai/langflow", "langgenius/dify", "n8n-io/n8n" ], "category_title": "Frameworks" }, { "intent": "I want browser-using agents", "picks": [ "browser-use/browser-use", "browserbase/stagehand", "MinorJerry/WebVoyager", "withLinda/puppeteer-real-browser-mcp-server" ], "category_title": "Plugins, MCPs, CLI tools" }, { "intent": "I want sandboxed code execution for agent-generated code", "picks": [ "e2b-dev/E2B", "kubernetes-sigs/agent-sandbox", "daytonaio/daytona", "huggingface/smolagents", "OpenHands/OpenHands" ], "category_title": "Libraries and SDKs" }, { "intent": "I want to evaluate or benchmark agents", "picks": [ "SWE-bench/SWE-bench", "harbor-framework/terminal-bench", "GAIR-NLP/AgencyBench", "UKGovernmentBEIS/inspect_ai", "web-arena-x/webarena", "meituan-longcat/vitabench" ], "category_title": "Evaluation and benchmarking harnesses" }, { "intent": "I want a deep research / autonomous research agent", "picks": [ "langchain-ai/deepagents", "assafelovic/gpt-researcher", "OpenAgentsInc/openagents" ], "category_title": "Research and task-specific harnesses" }, { "intent": "I want a provider-agnostic LLM pipe (not a framework)", "picks": [ "BerriAI/litellm", "vercel/ai" ], "category_title": "Libraries and SDKs" } ], "faq": [ { "kind": "use-case", "q": "What is the best agent harness if I want a turnkey coding agent today?", "a": "Top picks: opencode, Cline, Codex. See the “Coding agent products (IDEs, CLIs, full suites)” category for the full ranked list.", "slug": "what-is-the-best-agent-harness-if-i-want-a-turnkey-coding-agent-today" }, { "kind": "use-case", "q": "What is the best agent harness if I want an always-on personal agent that lives in my chat apps?", "a": "Top picks: OpenClaw, Hermes, Khoj. See the “Personal agent runtimes” category for the full ranked list.", "slug": "what-is-the-best-agent-harness-if-i-want-an-always-on-personal-agent-that-lives-in-my-chat-apps" }, { "kind": "use-case", "q": "What is the best agent harness if I want to extend Claude Code, Codex, or OpenCode with skills and slash commands?", "a": "Top picks: Anthropic Skills, wshobson/agents, superpowers. See the “Coding harness configs and SDKs” category for the full ranked list.", "slug": "what-is-the-best-agent-harness-if-i-want-to-extend-claude-code-codex-or-opencode-with-skills-and-slash-commands" }, { "kind": "use-case", "q": "What is the best agent harness if I want to build my own coding harness from scratch?", "a": "Top picks: Claude Agent SDK, Google ADK, AutoHarness. See the “Coding harness configs and SDKs” category for the full ranked list.", "slug": "what-is-the-best-agent-harness-if-i-want-to-build-my-own-coding-harness-from-scratch" }, { "kind": "use-case", "q": "What is the best agent harness if I want a drop-in memory layer for agents?", "a": "Top picks: Mem0, Graphiti (Zep), claude-mem. See the “Plugins, MCPs, CLI tools” category for the full ranked list.", "slug": "what-is-the-best-agent-harness-if-i-want-a-drop-in-memory-layer-for-agents" }, { "kind": "use-case", "q": "What is the best agent harness if I want to plug hundreds to thousands of tools without context bloat?", "a": "Top picks: MCP-Zero, ToolGen, ToolRAG. See the “Progressive disclosure harnesses” category for the full ranked list.", "slug": "what-is-the-best-agent-harness-if-i-want-to-plug-hundreds-to-thousands-of-tools-without-context-bloat" }, { "kind": "use-case", "q": "What is the best agent harness if I want multi-agent orchestration?", "a": "Top picks: openai-agents-python, crewAI, autogen. See the “Multi-agent and orchestration” category for the full ranked list.", "slug": "what-is-the-best-agent-harness-if-i-want-multi-agent-orchestration" }, { "kind": "use-case", "q": "What is the best agent harness if I want a general LLM app framework?", "a": "Top picks: langgraph, langchain, llama-index. See the “Frameworks” category for the full ranked list.", "slug": "what-is-the-best-agent-harness-if-i-want-a-general-llm-app-framework" }, { "kind": "use-case", "q": "What is the best agent harness if I want low-code / visual workflows?", "a": "Top picks: langflow, Dify, n8n. See the “Frameworks” category for the full ranked list.", "slug": "what-is-the-best-agent-harness-if-i-want-low-code-visual-workflows" }, { "kind": "use-case", "q": "What is the best agent harness if I want browser-using agents?", "a": "Top picks: browser-use, Stagehand, WebVoyager. See the “Plugins, MCPs, CLI tools” category for the full ranked list.", "slug": "what-is-the-best-agent-harness-if-i-want-browser-using-agents" }, { "kind": "use-case", "q": "What is the best agent harness if I want sandboxed code execution for agent-generated code?", "a": "Top picks: E2B, Agent Sandbox, Daytona. See the “Libraries and SDKs” category for the full ranked list.", "slug": "what-is-the-best-agent-harness-if-i-want-sandboxed-code-execution-for-agent-generated-code" }, { "kind": "use-case", "q": "What is the best agent harness if I want to evaluate or benchmark agents?", "a": "Top picks: SWE-bench, Terminal-Bench, AgencyBench. See the “Evaluation and benchmarking harnesses” category for the full ranked list.", "slug": "what-is-the-best-agent-harness-if-i-want-to-evaluate-or-benchmark-agents" }, { "kind": "use-case", "q": "What is the best agent harness if I want a deep research / autonomous research agent?", "a": "Top picks: deepagents, gpt-researcher, openagents. See the “Research and task-specific harnesses” category for the full ranked list.", "slug": "what-is-the-best-agent-harness-if-i-want-a-deep-research-autonomous-research-agent" }, { "kind": "use-case", "q": "What is the best agent harness if I want a provider-agnostic LLM pipe (not a framework)?", "a": "Top picks: LiteLLM, vercel/ai. See the “Libraries and SDKs” category for the full ranked list.", "slug": "what-is-the-best-agent-harness-if-i-want-a-provider-agnostic-llm-pipe-not-a-framework" }, { "kind": "derived", "q": "Which agent harnesses can run unattended (headless)?", "a": "Harnesses designed for unattended runs, batches, and fleets: opencode, OpenHands, goose, Symphony, SWE-agent, Claude Agent SDK, RepoMaster, OpenClaw.", "slug": "which-agent-harnesses-can-run-unattended-headless" }, { "kind": "derived", "q": "Which agent harnesses survive a crash mid-task (durable)?", "a": "Harnesses whose execution state persists across restarts: langgraph-bigtool, n8n, langgraph, mastra, letta, deepagents, pydantic-ai, Cloudflare Agents.", "slug": "which-agent-harnesses-survive-a-crash-mid-task-durable" }, { "kind": "derived", "q": "How many of these agent harnesses are open source?", "a": "118 of 160 carry a standard open-source license; the rest are source-available or unclear, and flagged per row.", "slug": "how-many-of-these-agent-harnesses-are-open-source" }, { "kind": "concept", "q": "What is an agent harness?", "a": "The runtime that turns a model into an agent: it decides what the model's reasoning is allowed to touch, and supplies the orchestration, tool wiring, memory, error recovery, and guardrails around per-turn inference.", "slug": "what-is-an-agent-harness" }, { "kind": "concept", "q": "How is this list ranked?", "a": "By relevance to harness concerns (environment, orchestration, lifecycle, guardrails) and by GitHub stars (captured 2026-08-23); each project also carries an adoption-surface tier and autonomy/recovery scores.", "slug": "how-is-this-list-ranked" }, { "kind": "concept", "q": "How can an AI agent use this list directly?", "a": "Three machine-readable surfaces: harnesses.json (structured), llms.txt (one file), and an MCP server (uvx agent-harnesses-mcp) exposing recommend, compare, pick_harness, and search_harnesses.", "slug": "how-can-an-ai-agent-use-this-list-directly" } ], "comparisons": [ { "slug": "agent-eval-harnesses", "title": "Agent evals: SWE-bench vs inspect_ai vs Terminal-Bench", "summary": "You changed your agent: new model, new prompt, new tools. Did it get better or worse? An eval is how you answer that with a number instead of a feeling. The word covers two different products, and knowing which one you need is most of the decision. A **benchmark** is a fixed public exam with a leade", "url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses/blob/main/comparisons/agent-eval-harnesses.md", "raw_url": "https://raw.githubusercontent.com/RyanAlberts/best-of-Agent-Harnesses/main/comparisons/agent-eval-harnesses.md" }, { "slug": "browser-agents", "title": "Browser agents: browser-use vs Stagehand vs Playwright MCP vs chrome-devtools-mcp", "summary": "\"Browser agent\" covers three different kinds of product, and most bad picks here come from comparing across the lanes instead of within one. **Agent libraries** own the whole job: you hand them a goal in plain language (\"find the cheapest flight, fill the form\") and they decide every click. browser-", "url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses/blob/main/comparisons/browser-agents.md", "raw_url": "https://raw.githubusercontent.com/RyanAlberts/best-of-Agent-Harnesses/main/comparisons/browser-agents.md" }, { "slug": "browser-infrastructure", "title": "Browser infrastructure for agents: Browserbase vs Steel vs Hyperbrowser", "summary": "Agent libraries like browser-use and Stagehand decide what to click; something still has to run the browsers they click in. At small scale that's Chrome on your own machine. At production scale it becomes its own operations problem: hundreds of concurrent sessions, sites that block automation, CAPTC", "url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses/blob/main/comparisons/browser-infrastructure.md", "raw_url": "https://raw.githubusercontent.com/RyanAlberts/best-of-Agent-Harnesses/main/comparisons/browser-infrastructure.md" }, { "slug": "claude-code-skill-packs", "title": "Claude Code skill packs: superpowers vs GStack vs get-shit-done vs Anthropic Skills", "summary": "A skill is a folder of instructions (a SKILL.md file, plus any scripts it needs) that a coding agent loads only when the task matches, instead of carrying every instruction all the time. A skill pack is a curated bundle of them: someone else's working habits for your agent, installed as files. Nothi", "url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses/blob/main/comparisons/claude-code-skill-packs.md", "raw_url": "https://raw.githubusercontent.com/RyanAlberts/best-of-Agent-Harnesses/main/comparisons/claude-code-skill-packs.md" }, { "slug": "eval-platforms", "title": "Eval and observability platforms: Langfuse vs LangSmith vs Braintrust vs Phoenix", "summary": "Benchmarks tell you how a model ranks; your production agent still fails in ways no public exam covers. An eval and observability platform is where teams watch what their agent actually did (tracing: recording every step, tool call, and token of a run) and score it continuously (evals: checks that r", "url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses/blob/main/comparisons/eval-platforms.md", "raw_url": "https://raw.githubusercontent.com/RyanAlberts/best-of-Agent-Harnesses/main/comparisons/eval-platforms.md" }, { "slug": "how-to-pick-a-harness", "title": "How to pick a harness", "summary": "This is the decision guide for [best-of-Agent-Harnesses](../README.md), a curated, ranked list of the runtimes that turn an AI model into a working agent: the software that decides what the model's thinking is allowed to touch. Six questions, in order. Each one eliminates most of the list; by the en", "url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses/blob/main/comparisons/how-to-pick-a-harness.md", "raw_url": "https://raw.githubusercontent.com/RyanAlberts/best-of-Agent-Harnesses/main/comparisons/how-to-pick-a-harness.md" }, { "slug": "how-to-test-drive-a-harness", "title": "How to test-drive a harness", "summary": "Spec sheets cannot answer \"which harness should I use,\" because an agent's performance is a property of the *pairing* between harness and model, not of either alone: the same model passes [46% of tasks in one harness and 80% in another](https://www.mindstudio.ai/blog/agent-harness-scaffolding-matter", "url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses/blob/main/comparisons/how-to-test-drive-a-harness.md", "raw_url": "https://raw.githubusercontent.com/RyanAlberts/best-of-Agent-Harnesses/main/comparisons/how-to-test-drive-a-harness.md" }, { "slug": "memory-layers", "title": "Agent memory layers: Mem0 vs Zep vs Letta vs claude-mem", "summary": "Agents forget. A model keeps nothing between sessions, so anything your agent should still know tomorrow (who the user is, what was decided, what failed last time) has to live in a memory system outside the model. \"Add memory to my agent\" then hides genuinely different products, and picking by star ", "url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses/blob/main/comparisons/memory-layers.md", "raw_url": "https://raw.githubusercontent.com/RyanAlberts/best-of-Agent-Harnesses/main/comparisons/memory-layers.md" }, { "slug": "multi-agent-orchestration", "title": "Multi-agent orchestration: OpenAI Agents SDK vs CrewAI vs AutoGen vs Agent Framework vs LangGraph", "summary": "Orchestration is the layer that coordinates several AI agents working on one job: who acts next, what they share, and what happens when a step fails. The five frameworks here answer that with genuinely different architectures, and the pick is expensive to get wrong because the coordination model sha", "url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses/blob/main/comparisons/multi-agent-orchestration.md", "raw_url": "https://raw.githubusercontent.com/RyanAlberts/best-of-Agent-Harnesses/main/comparisons/multi-agent-orchestration.md" }, { "slug": "openclaw-vs-hermes", "title": "OpenClaw vs Hermes: the always-on personal-agent debate", "summary": "An always-on personal agent is a program that runs all day on your own machine, talks to you through the chat apps you already use (WhatsApp, Telegram, Slack, Discord), and keeps acting between your messages: watching inboxes, running scheduled jobs, remembering what you told it last month. OpenClaw", "url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses/blob/main/comparisons/openclaw-vs-hermes.md", "raw_url": "https://raw.githubusercontent.com/RyanAlberts/best-of-Agent-Harnesses/main/comparisons/openclaw-vs-hermes.md" }, { "slug": "progressive-disclosure", "title": "Context files for agents: AGENTS.md vs CLAUDE.md vs skills vs MCP tool search", "summary": "A model has a context window: a fixed amount of text it can consider at once. Everything competes for that space: your instructions, the definitions of every tool the agent could call, and the output of every tool it already called. Context bloat is what happens when the \"might need it\" pile crowds ", "url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses/blob/main/comparisons/progressive-disclosure.md", "raw_url": "https://raw.githubusercontent.com/RyanAlberts/best-of-Agent-Harnesses/main/comparisons/progressive-disclosure.md" }, { "slug": "sandboxed-code-execution", "title": "Agent sandboxing: what it is and how to pick", "summary": "An AI agent does not just suggest code. It runs code, opens web pages, and edits files on a real computer. Agent sandboxing means making that computer a disposable one, so that when the agent makes a mistake, or a malicious web page talks it into one, the damage lands in a box you can throw away ins", "url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses/blob/main/comparisons/sandboxed-code-execution.md", "raw_url": "https://raw.githubusercontent.com/RyanAlberts/best-of-Agent-Harnesses/main/comparisons/sandboxed-code-execution.md" }, { "slug": "terminal-coding-agents", "title": "Terminal coding agents: opencode vs Codex vs Gemini CLI vs crush vs goose", "summary": "The most-asked pick in this list: *\"I want a turnkey coding agent in my terminal today.\"* A terminal coding agent is a program you run in your shell that takes a plain-language request, then works in a loop: the model proposes an action (edit this file, run this command, search the repo), the tool e", "url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses/blob/main/comparisons/terminal-coding-agents.md", "raw_url": "https://raw.githubusercontent.com/RyanAlberts/best-of-Agent-Harnesses/main/comparisons/terminal-coding-agents.md" } ], "projects": [ { "name": "Headroom", "github_id": "headroomlabs-ai/headroom", "url": "https://github.com/headroomlabs-ai/headroom", "slug": "headroom", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#headroom", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/headroom/", "description": "Compresses tool outputs, logs, files, and RAG chunks with content-aware compressors before they reach the model—claimed 20% fewer tokens for coding agents and 60–95% fewer for JSON, same answers. Ships as a library, HTTP proxy, or MCP server, so it drops in front of whatever harness you already run.", "category": "progressive-disclosure", "category_title": "Progressive disclosure harnesses", "stars": 67250, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (compression library/proxy/MCP server)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "mcp", "rag" ], "example": { "label": "Project README", "url": "https://github.com/headroomlabs-ai/headroom#readme" }, "deep_dive": null }, { "name": "awesome-cursorrules", "github_id": "PatrickJS/awesome-cursorrules", "url": "https://github.com/PatrickJS/awesome-cursorrules", "slug": "awesome-cursorrules", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#awesome-cursorrules", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/awesome-cursorrules/", "description": "Curated .cursorrules and skills that leverage Cursor's index-then-load model; the canonical collection for rules-as-progressive-disclosure in the IDE.", "category": "progressive-disclosure", "category_title": "Progressive disclosure harnesses", "stars": 40646, "tier": "super simple", "tier_rank": 1, "axis": "super simple (content bundle)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "ide" ], "example": { "label": "PyTorch cursorrules", "url": "https://github.com/PatrickJS/awesome-cursorrules/blob/main/rules/pytorch-scikit-learn-cursorrules-prompt-file.mdc" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "none", "rank": 1, "detail": "The repo is a curated set of .mdc rule files loaded by Cursor; it contains no tool-execution or sandboxing layer of its own.", "evidence": "https://raw.githubusercontent.com/PatrickJS/awesome-cursorrules/HEAD/README.md" }, "context_memory": { "rating": "none", "rank": 1, "detail": "Rules are stateless per-project configuration files scoped by glob/alwaysApply toggles, with no session or cross-session persistence documented.", "evidence": "https://raw.githubusercontent.com/PatrickJS/awesome-cursorrules/HEAD/README.md" }, "lifecycle_hooks": { "rating": "none", "rank": 1, "detail": "The .mdc format only exposes glob/alwaysApply scoping metadata for when a rule loads, not before/after tool-call interception.", "evidence": "https://raw.githubusercontent.com/PatrickJS/awesome-cursorrules/HEAD/README.md" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "The entire project is a library of drop-in .mdc rule files that inject project-specific system-prompt guidance into Cursor, the canonical rules-file customization pattern.", "evidence": "https://raw.githubusercontent.com/PatrickJS/awesome-cursorrules/HEAD/README.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Users copy config-driven .mdc rule blueprints into .cursor/rules/ rather than importing an SDK or running a hosted service; the runtime is Cursor itself.", "evidence": "https://raw.githubusercontent.com/PatrickJS/awesome-cursorrules/HEAD/README.md" } } }, { "name": "agents.md", "github_id": "agentsmd/agents.md", "url": "https://github.com/agentsmd/agents.md", "slug": "agentsmd", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#agentsmd", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/agentsmd/", "description": "Open format for repo-scoped agent briefings; nested AGENTS.md files scope instructions per directory, so agents get a map of what exists and load only what's relevant. Read by 20+ tools including Codex, Cursor, and Copilot.", "category": "progressive-disclosure", "category_title": "Progressive disclosure harnesses", "stars": 23804, "tier": "super simple", "tier_rank": 1, "axis": "super simple (format only)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "ide", "typescript" ], "example": { "label": "Self-hosting AGENTS.md", "url": "https://github.com/agentsmd/agents.md/blob/main/AGENTS.md" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "none", "rank": 1, "detail": "AGENTS.md is a markdown format spec with no execution or isolation mechanism; running test commands it lists is left entirely to the host agent.", "evidence": "https://agents.md/" }, "context_memory": { "rating": "none", "rank": 1, "detail": "Nested AGENTS.md files are resolved by directory-proximity precedence at read time (closest file wins) with no session state or cross-session persistence.", "evidence": "https://agents.md/" }, "lifecycle_hooks": { "rating": "none", "rank": 1, "detail": "The spec defines no before/after interception points; it only documents static markdown sections an agent reads once per task.", "evidence": "https://agents.md/" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "AGENTS.md is exactly the rules-file/system-prompt-layering pattern: repo- and directory-scoped instruction files an agent reads for dev/testing/PR guidance.", "evidence": "https://raw.githubusercontent.com/agentsmd/agents.md/HEAD/README.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "It is an open format/config spec (with a reference Next.js site) that projects adopt, not an SDK to import or a hosted runtime.", "evidence": "https://raw.githubusercontent.com/agentsmd/agents.md/HEAD/README.md" } } }, { "name": "context-mode", "github_id": "mksglu/context-mode", "url": "https://github.com/mksglu/context-mode", "slug": "context-mode", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#context-mode", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/context-mode/", "description": "Context-window optimization layer that sandboxes tool output before it reaches the model (claimed 98% reduction) and persists session memory across 17 agent platforms via MCP and hooks—progressive disclosure applied to tool results, not just instructions.", "category": "progressive-disclosure", "category_title": "Progressive disclosure harnesses", "stars": 20102, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (output sandboxing, cross-platform)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "restricted (Elastic-2.0)", "tags": [ "mcp", "memory", "sandbox" ], "example": { "label": "Project README", "url": "https://github.com/mksglu/context-mode#readme" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "strong", "rank": 3, "detail": "ctx_execute runs tools at the subprocess level so raw output never enters context, and a PreToolUse hook enforces sandbox routing that blocks dangerous commands (e.g. sudo, rm -rf) before execution.", "evidence": "https://raw.githubusercontent.com/mksglu/context-mode/HEAD/README.md" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "A per-project SQLite database logs every file edit, git operation, task, error, and decision, with PreCompact/SessionStart hooks building an XML snapshot that restores full working state after context compaction on supported platforms.", "evidence": "https://raw.githubusercontent.com/mksglu/context-mode/HEAD/README.md" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "Registers PreToolUse (blocking, can deny execution), PostToolUse, UserPromptSubmit, PreCompact, and SessionStart hooks across supported platforms (Claude Code, Gemini CLI, VS Code Copilot).", "evidence": "https://raw.githubusercontent.com/mksglu/context-mode/HEAD/README.md" }, "prompt_optimization": { "rating": "none", "rank": 1, "detail": "The project explicitly states it enforces no prose style and never dictates how the model writes its answer, only filtering tool output, not tuning prompts.", "evidence": "https://raw.githubusercontent.com/mksglu/context-mode/HEAD/README.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Ships as a self-hosted, config/hook-driven layer installed into existing agent platforms (nothing leaves the machine); an optional hosted Insight dashboard at context-mode.com/insight is a secondary add-on.", "evidence": "https://raw.githubusercontent.com/mksglu/context-mode/HEAD/README.md" } } }, { "name": "langgraph-bigtool", "github_id": "langchain-ai/langgraph-bigtool", "url": "https://github.com/langchain-ai/langgraph-bigtool", "slug": "langgraph-bigtool", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#langgraph-bigtool", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/langgraph-bigtool/", "description": "Build LangGraph agents with large tool sets; retrieval and on-demand tool loading so agents scale beyond context without stuffing every schema upfront.", "category": "progressive-disclosure", "category_title": "Progressive disclosure harnesses", "stars": 556, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (large tool sets)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "durable", "recovery_rank": 4, "license_signal": "open-source", "tags": [ "tool-discovery", "python" ], "example": { "label": "Math-library tool agent", "url": "https://github.com/langchain-ai/langgraph-bigtool#quickstart" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "none", "rank": 1, "detail": "The library only handles tool retrieval; it documents no isolated execution environment, and users must implement their own execution constraints or safety measures.", "evidence": "https://raw.githubusercontent.com/langchain-ai/langgraph-bigtool/HEAD/README.md" }, "context_memory": { "rating": "basic", "rank": 2, "detail": "Uses LangGraph's long-term memory store purely to index and semantically search for relevant TOOL DEFINITIONS (in-memory or Postgres backend), not to persist conversation/task context.", "evidence": "https://raw.githubusercontent.com/langchain-ai/langgraph-bigtool/HEAD/README.md" }, "lifecycle_hooks": { "rating": "none", "rank": 1, "detail": "No pre-execution interception or blocking mechanism is documented; the library only wires retrieval into the agent's tool-selection step.", "evidence": "https://raw.githubusercontent.com/langchain-ai/langgraph-bigtool/HEAD/README.md" }, "prompt_optimization": { "rating": "none", "rank": 1, "detail": "No prompt tuning or system-prompt engineering feature is described in the project docs.", "evidence": "https://raw.githubusercontent.com/langchain-ai/langgraph-bigtool/HEAD/README.md" }, "build_vs_buy": { "tier": 1, "label": "build", "detail": "Installed via pip install langgraph-bigtool as a Python library composed into a custom LangGraph agent loop, with no opinionated hosted runtime.", "evidence": "https://raw.githubusercontent.com/langchain-ai/langgraph-bigtool/HEAD/README.md" } } }, { "name": "MCP-Zero", "github_id": "xfey/MCP-Zero", "url": "https://github.com/xfey/MCP-Zero", "slug": "mcp-zero", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#mcp-zero", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/mcp-zero/", "description": "Active tool discovery for autonomous agents: model requests tools by requirement; hierarchical semantic routing over 308 servers / 2,797 tools with ~98% token reduction (APIBank).", "category": "progressive-disclosure", "category_title": "Progressive disclosure harnesses", "stars": 511, "tier": "complex", "tier_rank": 4, "axis": "complex (3k tools, full routing)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "none", "recovery_rank": 1, "license_signal": "open-source", "tags": [ "tool-discovery" ], "example": { "label": "APIBank experiment", "url": "https://github.com/xfey/MCP-Zero/blob/master/MCP-zero/experiment_apibank.py" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "unknown", "rank": 0, "detail": "Searched the README, arxiv paper, and repo structure (matcher.py/sampler.py/reformatter.py); MCP-Zero is a tool-discovery/retrieval library with no agent execution loop, so there is nothing to rate for sandboxing.", "evidence": "" }, "context_memory": { "rating": "basic", "rank": 2, "detail": "Hierarchical semantic routing retrieves top-k relevant tool descriptions on demand (~98% token cut on APIBank) - context scoping for tool definitions only, with no session persistence or conversation-context management.", "evidence": "https://arxiv.org/html/2506.01056v2" }, "lifecycle_hooks": { "rating": "unknown", "rank": 0, "detail": "Searched the README, arxiv paper, and repo structure for hooks/callbacks/middleware; found none because the project is a retrieval library rather than an agent loop with before/after interception points.", "evidence": "" }, "prompt_optimization": { "rating": "none", "rank": 1, "detail": "Uses one fixed, hardcoded prompt template (plus a single in-context example) to trigger tool retrieval; this is not a user-configurable rules layer and there is no automatic prompt-tuning loop.", "evidence": "https://arxiv.org/html/2506.01056v2" }, "build_vs_buy": { "tier": 1, "label": "build", "detail": "Ships as an MIT-licensed Python research library (matcher/sampler/reformatter modules plus a precomputed tool-embedding dataset) meant to be imported into someone else's agent loop, not a runnable agent product or opinionated runtime.", "evidence": "https://github.com/xfey/MCP-Zero" } } }, { "name": "ToolGen", "github_id": "Reason-Wang/ToolGen", "url": "https://github.com/Reason-Wang/ToolGen", "slug": "toolgen", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#toolgen", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/toolgen/", "description": "ICLR 2025: unified tool retrieval and calling via generation; 47k+ tools without context stuffing—retrieval and invocation in one generative step.", "category": "progressive-disclosure", "category_title": "Progressive disclosure harnesses", "stars": 184, "tier": "complex", "tier_rank": 4, "axis": "complex (47k+ tools)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "unknown", "tags": [ "tool-discovery", "python" ], "example": { "label": "Full eval pipeline", "url": "https://github.com/Reason-Wang/ToolGen/blob/master/scripts/eval_full_pipeline.sh" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "none", "rank": 1, "detail": "No isolation layer for tool execution is described; the project is research training code, not an execution harness.", "evidence": "https://raw.githubusercontent.com/Reason-Wang/ToolGen/HEAD/README.md" }, "context_memory": { "rating": "basic", "rank": 2, "detail": "Tools are virtualized as unique generation tokens so the model retrieves and invokes them directly rather than stuffing 47k tool schemas into context, i.e. retrieval scoped to TOOL DEFINITIONS only.", "evidence": "https://raw.githubusercontent.com/Reason-Wang/ToolGen/HEAD/README.md" }, "lifecycle_hooks": { "rating": "none", "rank": 1, "detail": "No middleware, callback, or before/after interception system is mentioned anywhere in the project docs.", "evidence": "https://raw.githubusercontent.com/Reason-Wang/ToolGen/HEAD/README.md" }, "prompt_optimization": { "rating": "none", "rank": 1, "detail": "The three-stage pipeline (memorization, retrieval, end-to-end tuning) trains model weights on tool tokens; it is not a system-prompt refinement or prompt-tuning mechanism.", "evidence": "https://raw.githubusercontent.com/Reason-Wang/ToolGen/HEAD/README.md" }, "build_vs_buy": { "tier": 1, "label": "build", "detail": "Delivered as open-source research code/model weights on Hugging Face that users must integrate and deploy themselves, with no packaged runtime.", "evidence": "https://raw.githubusercontent.com/Reason-Wang/ToolGen/HEAD/README.md" } } }, { "name": "ToolRAG", "github_id": "antl3x/ToolRAG", "url": "https://github.com/antl3x/ToolRAG", "slug": "toolrag", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#toolrag", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/toolrag/", "description": "Semantic tool retrieval for LLMs; serves only the tools the user query demands (MCP-compatible), unlimited tool sets with zero context penalty.", "category": "progressive-disclosure", "category_title": "Progressive disclosure harnesses", "stars": 33, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (query-driven retrieval)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "mcp", "tool-discovery" ], "example": { "label": "MCP server retrieval", "url": "https://github.com/antl3x/ToolRAG/blob/main/packages/%40antl3x-toolrag/README.md" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "none", "rank": 1, "detail": "Tool execution isolation is not addressed anywhere in the documentation; the library only handles which tool schemas get shown to the model.", "evidence": "https://raw.githubusercontent.com/antl3x/ToolRAG/HEAD/README.md" }, "context_memory": { "rating": "basic", "rank": 2, "detail": "A LibSQL database stores tool definitions and their vector embeddings for semantic tool search, limited to TOOL DEFINITION retrieval rather than conversation or task memory.", "evidence": "https://raw.githubusercontent.com/antl3x/ToolRAG/HEAD/README.md" }, "lifecycle_hooks": { "rating": "none", "rank": 1, "detail": "No interceptor pattern, middleware, or before/after lifecycle hook is described in the project docs.", "evidence": "https://raw.githubusercontent.com/antl3x/ToolRAG/HEAD/README.md" }, "prompt_optimization": { "rating": "none", "rank": 1, "detail": "The project focuses solely on semantic tool selection via vector embeddings, with no prompt engineering or tuning feature documented.", "evidence": "https://raw.githubusercontent.com/antl3x/ToolRAG/HEAD/README.md" }, "build_vs_buy": { "tier": 1, "label": "build", "detail": "Distributed as the @antl3x/toolrag npm package that developers import into their own MCP/agent code, not a self-hosted runtime or managed service.", "evidence": "https://raw.githubusercontent.com/antl3x/ToolRAG/HEAD/README.md" } } }, { "name": "opencode", "github_id": "anomalyco/opencode", "url": "https://github.com/anomalyco/opencode", "slug": "opencode", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#opencode", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/opencode/", "description": "Open-source terminal coding agent (formerly `sst/opencode`; transferred to anomalyco). The **harness** is a multi-provider tool-call loop (Claude, OpenAI, Gemini, local) with strong plugin and MCP support; the TUI is the shell. 100% OSS, very actively shipped.", "category": "coding-agent-products", "category_title": "Coding agent products (IDEs, CLIs, full suites)", "stars": 200557, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (multi-provider, plugins, MCP)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "mcp", "provider-agnostic", "cli", "tui", "typescript" ], "example": { "label": "Agent system page", "url": "https://opencode.ai/docs/agents/" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "Tool execution runs on the host by default; a `permission` config resolves each action to allow/ask/deny with interactive approval prompts, with no container/VM/enforced allowlist isolation the agent can't bypass.", "evidence": "https://opencode.ai/docs/permissions/" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Automatic token-based compaction generates an LLM-written checkpoint summary before the context window overflows and keeps a serialized tail of recent messages, while earlier session messages remain durable rather than deleted.", "evidence": "https://v2.opencode.ai/compaction" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "The plugin API exposes tool.execute.before/after hooks where a before-hook can throw to block a call (e.g. denying reads of .env files) or mutate output.args to rewrite it, plus session/message/permission event hooks.", "evidence": "https://opencode.ai/docs/plugins/" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Project- and user-level AGENTS.md files (plus legacy CLAUDE.md fallback and glob/URL `instructions` entries in opencode.json) are layered as static, user-authored context; there is no automatic prompt-refinement or tuning loop.", "evidence": "https://opencode.ai/docs/rules/" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Installs as a self-hosted, config-driven CLI/TUI coding agent (build/plan agent modes, plugin system, any-LLM-provider config) that you run yourself; OpenCode Zen is an optional hosted curated-model gateway, not a required execution backend.", "evidence": "https://opencode.ai/docs/" } } }, { "name": "Codex", "github_id": "openai/codex", "url": "https://github.com/openai/codex", "slug": "codex", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#codex", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/codex/", "description": "OpenAI's terminal coding agent. The **harness** is the sandboxed tool-call loop with multi-provider support; the CLI is the shell. Reference implementation for \"official CLI that ships code.\"", "category": "coding-agent-products", "category_title": "Coding agent products (IDEs, CLIs, full suites)", "stars": 114837, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (reference CLI, sandboxed)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "sandbox", "provider-agnostic", "cli" ], "example": { "label": "Sandboxing concept", "url": "https://developers.openai.com/codex/concepts/sandboxing" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "strong", "rank": 3, "detail": "Sandboxing is on by default via platform-native enforcement (macOS Seatbelt, Linux/WSL2 bubblewrap user-namespace isolation), confining writes to a workspace-write boundary and blocking network access unless explicitly approved.", "evidence": "https://learn.chatgpt.com/codex/sandboxing" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Built-in PreCompact/PostCompact hooks handle context compaction and a SessionStart hook (matcher \"startup|resume\") restores state when a prior session is resumed.", "evidence": "https://learn.chatgpt.com/docs/hooks" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "PreToolUse hooks intercept Bash, apply_patch file edits, and MCP tool calls and can return permissionDecision: \"deny\" to block them (or rewrite inputs) before they run, with PostToolUse hooks logging after execution.", "evidence": "https://learn.chatgpt.com/docs/hooks" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "AGENTS.md files provide custom, repo-scoped instructions layered into the system prompt, without any automatic tuning of those instructions.", "evidence": "https://developers.openai.com/codex/guides/agents-md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Primary artifact is the self-hosted Codex CLI configured via config.toml/hooks.json that you run locally; OpenAI also offers a hosted ChatGPT Codex cloud environment as a managed alternative.", "evidence": "https://raw.githubusercontent.com/openai/codex/HEAD/README.md" } } }, { "name": "Gemini CLI", "github_id": "google-gemini/gemini-cli", "url": "https://github.com/google-gemini/gemini-cli", "slug": "gemini-cli", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#gemini-cli", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/gemini-cli/", "description": "Google's first-party terminal agent for Gemini. The **harness** is the plugin/MCP tool-call loop; the terminal is the shell—Google's parallel to Claude Code / Codex, not just an API.", "category": "coding-agent-products", "category_title": "Coding agent products (IDEs, CLIs, full suites)", "stars": 106626, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (official CLI, plugins, MCP)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "mcp", "cli", "typescript" ], "example": { "label": "MCP server setup", "url": "https://github.com/google-gemini/gemini-cli/blob/main/docs/tools/mcp-server.md" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "Ships real sandboxing (macOS Seatbelt, or Docker/Podman/gVisor/LXC on Linux, confining writes to the project dir) but it is disabled by default and only activates via -s flag, GEMINI_SANDBOX env var, or config, so host execution is the default.", "evidence": "https://raw.githubusercontent.com/google-gemini/gemini-cli/main/docs/cli/sandbox.md" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Supports conversation checkpointing to save and resume complex sessions plus token caching, alongside GEMINI.md context files for project-specific memory.", "evidence": "https://raw.githubusercontent.com/google-gemini/gemini-cli/HEAD/README.md" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "BeforeTool hooks run synchronously before a tool call and can deny it (exit code 2 or a JSON {\"decision\":\"deny\"} response), with matching AfterTool hooks for post-execution logic, configured in settings.json.", "evidence": "https://geminicli.com/docs/hooks/reference/" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Custom context files (GEMINI.md) let projects inject persistent, hierarchical system-prompt guidance without modifying the CLI's core prompt.", "evidence": "https://raw.githubusercontent.com/google-gemini/gemini-cli/HEAD/README.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Distributed as a self-hosted, config-driven CLI runtime (settings.json, hooks, GEMINI.md) that you run yourself against the Gemini API/Vertex AI backend, not an SDK primitive set or a fully managed execution platform.", "evidence": "https://raw.githubusercontent.com/google-gemini/gemini-cli/HEAD/README.md" } } }, { "name": "pi", "github_id": "earendil-works/pi", "url": "https://github.com/earendil-works/pi", "slug": "pi", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#pi", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/pi/", "description": "The upstream AI agent toolkit behind this list's oh-my-pi fork: a unified multi-provider LLM API, agent loop, and TUI shell providing the **harness** that oh-my-pi's Rust rewrite builds on.", "category": "coding-agent-products", "category_title": "Coding agent products (IDEs, CLIs, full suites)", "stars": 95747, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (multi-provider agent loop, TUI)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "resumable", "recovery_rank": 3, "license_signal": "unknown", "tags": [ "provider-agnostic", "tui", "rust" ], "example": { "label": "Project README", "url": "https://github.com/earendil-works/pi#readme" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "Pi ships no built-in permission system for restricting filesystem/process/network/credential access, and instead documents optional external containerization patterns (Gondolin micro-VM, Docker, OpenShell).", "evidence": "https://raw.githubusercontent.com/earendil-works/pi/HEAD/README.md" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Sessions persist as JSONL files with a branching tree structure (message entries, compactions, branch summaries), with session_before_compact/session_compact hooks handling conversation compaction and /resume restoring prior sessions.", "evidence": "https://raw.githubusercontent.com/earendil-works/pi/main/packages/coding-agent/docs/extensions.md" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "The tool_call hook fires after a tool starts but before execution and can block it with a reason (or mutate its input), while tool_execution_end/tool_result hooks trace and can modify results after the fact.", "evidence": "https://raw.githubusercontent.com/earendil-works/pi/main/packages/coding-agent/docs/extensions.md" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "The before_agent_start hook exposes event.systemPrompt and systemPromptOptions so extensions can chain modifications to the system prompt per turn, and tools can declare promptGuidelines appended to it.", "evidence": "https://raw.githubusercontent.com/earendil-works/pi/main/packages/coding-agent/docs/extensions.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Ships as a self-hosted coding-agent CLI/TUI configured through an extensive extension/hook system rather than a bare SDK primitive; a separate lower-level agent-loop package is also importable for tier-1 style use.", "evidence": "https://raw.githubusercontent.com/earendil-works/pi/HEAD/README.md" } } }, { "name": "OpenHands", "github_id": "OpenHands/OpenHands", "url": "https://github.com/OpenHands/OpenHands", "slug": "openhands", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#openhands", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/openhands/", "description": "Dockerized software-engineering agent. The **harness** is the bash/editor/browser toolset with micro-agents and event-stream session bridging; Docker is the sandbox. Main OSS choice for teams self-hosting autonomous repo work.", "category": "coding-agent-products", "category_title": "Coding agent products (IDEs, CLIs, full suites)", "stars": 84844, "tier": "complex", "tier_rank": 4, "axis": "complex (Docker runtime, multi-surface agent — product suite)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "resumable", "recovery_rank": 3, "license_signal": "restricted ((multi-license))", "tags": [ "memory", "browser", "sandbox", "python" ], "example": { "label": "Repository microagents", "url": "https://docs.all-hands.dev/usage/prompting/microagents-repo" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "strong", "rank": 3, "detail": "RUNTIME=docker is the default sandbox provider, running the agent server in an isolated Docker container described as providing 'complete isolation from the host system'; a Process (no-container) mode exists but is opt-in and documented as unsafe.", "evidence": "https://docs.openhands.dev/openhands/usage/sandboxes/overview.md" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "A Condenser auto-compresses event history past a configurable threshold (default 120 events) via keep-first-N/last-M plus LLM summarization, and full conversation state (event log, working directory, tool results) is persisted to disk and restorable across sessions or after a crash.", "evidence": "https://docs.openhands.dev/sdk/guides/convo-persistence.md" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "PreToolUse, UserPromptSubmit, and Stop hooks run before the corresponding action and can block it by exiting with code 2 (e.g. blocking a dangerous rm -rf command), alongside non-blocking PostToolUse/SessionStart/SessionEnd hooks.", "evidence": "https://docs.openhands.dev/sdk/guides/hooks.md" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "A repo-level AGENTS.md plus on-demand 'skills' (markdown files under .agents/skills/, triggered by keyword or by agent decision) statically inject instructions into the system prompt; no automatic prompt-tuning mechanism is documented.", "evidence": "https://docs.openhands.dev/overview/skills.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Primary artifact is the MIT-licensed, self-hosted Agent Server/SDK you run locally, in Docker, or in your own Kubernetes/VPC deployment; OpenHands Cloud is a separate vendor-managed offering for teams that don't want to self-host.", "evidence": "https://docs.openhands.dev/overview/introduction" } } }, { "name": "Open Interpreter", "github_id": "openinterpreter/openinterpreter", "url": "https://github.com/openinterpreter/openinterpreter", "slug": "openinterpreter", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#openinterpreter", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/openinterpreter/", "description": "Lightweight terminal coding agent oriented to open models (DeepSeek, Kimi, Qwen). The **harness** is a code-execution loop — the model writes code, the harness executes it with confirmation gates; the CLI is the shell. The original \"let the LLM run code on my machine\" project, reborn for open weights.", "category": "coding-agent-products", "category_title": "Coding agent products (IDEs, CLIs, full suites)", "stars": 68121, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (lean code-exec loop)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "cli", "python" ], "example": { "label": "Quick start", "url": "https://github.com/openinterpreter/openinterpreter#readme" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "strong", "rank": 3, "detail": "OS-level sandboxing (macOS Seatbelt profiles, Linux/WSL Bubblewrap+seccomp) restricts execution to read-only/workspace-write/danger-full-access modes with network disabled by default in workspace-write, and the project fails closed rather than running unsandboxed when a policy can't be enforced.", "evidence": "https://www.openinterpreter.com/docs/terminal/sandbox" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "PreCompact/PostCompact hooks manage context compaction and SessionStart plus `interpreter resume --last` restore a prior session's state.", "evidence": "https://www.openinterpreter.com/docs/terminal/hooks" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "PreToolUse fires before a tool call and can return permissionDecision: \"deny\" to block it (e.g. deny a command), with PostToolUse tracing after execution.", "evidence": "https://www.openinterpreter.com/docs/terminal/hooks" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "AGENTS.md gives the agent project-specific guidance it always reads, layered into the system prompt without automatic tuning.", "evidence": "https://www.openinterpreter.com/docs/terminal" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "A self-hosted terminal CLI (fork of Codex) with selectable model harnesses, configured via hooks/permissions/AGENTS.md and local state under ~/.openinterpreter, not an SDK primitive or managed platform.", "evidence": "https://raw.githubusercontent.com/openinterpreter/openinterpreter/HEAD/README.md" } } }, { "name": "Cline", "github_id": "cline/cline", "url": "https://github.com/cline/cline", "slug": "cline", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#cline", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/cline/", "description": "VS Code extension whose **harness** is a plan-then-act loop with per-step human approval and cost transparency; the VS Code integration is the UI shell. Open-source counterweight to Cursor.", "category": "coding-agent-products", "category_title": "Coding agent products (IDEs, CLIs, full suites)", "stars": 66707, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (plan-then-act, approval gates)", "autonomy": "step-gated", "autonomy_rank": 1, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "ide", "typescript" ], "example": { "label": "Plan & Act mode", "url": "https://docs.cline.bot/features/plan-and-act" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "Execution is on the host by default with an Auto Approve/YOLO-mode permission system gating file edits, terminal commands, browser actions, and MCP tools; the docs recommend an external sandboxed VM/container for YOLO mode rather than shipping isolation.", "evidence": "https://docs.cline.bot/features/auto-approve" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Team/session state persists across restarts (\"pick up where you left off\", scheduled runs persist across restarts) beyond a single live process.", "evidence": "https://raw.githubusercontent.com/cline/cline/HEAD/README.md" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "The tool_call_before hook fires before a tool call and can audit or prevent the invocation (policy enforcement, fail_closed mode for unsafe bypass), with tool_call_after logging results afterward.", "evidence": "https://docs.cline.bot/sdk/hooks" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": ".clinerules files inject project-specific coding standards, architecture conventions, and deployment procedures into how Cline behaves, without automatic tuning.", "evidence": "https://raw.githubusercontent.com/cline/cline/HEAD/README.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Primary published artifact is the self-hosted VS Code/JetBrains extension and CLI configured via .clinerules and the plugin/hook system; a lower-level programmatic SDK is also offered for tier-1 style composition.", "evidence": "https://raw.githubusercontent.com/cline/cline/HEAD/README.md" } } }, { "name": "goose", "github_id": "aaif-goose/goose", "url": "https://github.com/aaif-goose/goose", "slug": "goose", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#goose", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/goose/", "description": "Block-originated Rust agent, now stewarded by the Linux Foundation's Agentic AI Foundation (`aaif-goose/goose`). The **harness** is the MCP/ACP extension model with recipes and provider choice; there's no fixed UI slot—you bolt it into whatever shell you use.", "category": "coding-agent-products", "category_title": "Coding agent products (IDEs, CLIs, full suites)", "stars": 53295, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (extensions, MCP/ACP)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "mcp", "rust" ], "example": { "label": "Goose recipes guide", "url": "https://block.github.io/goose/docs/guides/recipes/" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "Tool execution is gated by a three-tier permission.yaml policy (Always Allow / Ask Before / Never Allow) with Chat/Auto/Approve/Smart-Approval modes, but there is no container/VM isolation shipped (an open \"Sandbox Support?\" feature request confirms it runs with host permissions today).", "evidence": "https://goose-docs.ai/docs/guides/managing-tools/tool-permissions/" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Each session's Agent maintains conversation state and automatic context management compacts history when approaching token limits while preserving recent/important context.", "evidence": "https://block-goose.mintlify.app/concepts/agents" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "The Never Allow/Ask Before permission levels can pre-emptively block a tool call, but this is a fixed built-in policy config (not user-definable hook code) with no documented after-the-fact tracing/audit log, short of a full hooks/middleware system.", "evidence": "https://goose-docs.ai/docs/guides/managing-tools/tool-permissions/" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": ".goosehints/AGENTS.md files are scanned from the working directory and injected into the system prompt at session start, and Recipe YAML files carry their own instructions field plus session-level system_prompt overrides.", "evidence": "https://block-goose.mintlify.app/concepts/agents" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Ships as a self-hosted Rust desktop app/CLI/API configured via extensions, permission.yaml, and recipe YAML files that you run yourself, not a managed hosted platform.", "evidence": "https://raw.githubusercontent.com/aaif-goose/goose/HEAD/README.md" } } }, { "name": "DeepSeek-Reasonix", "github_id": "esengine/DeepSeek-Reasonix", "url": "https://github.com/esengine/DeepSeek-Reasonix", "slug": "deepseek-reasonix", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#deepseek-reasonix", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/deepseek-reasonix/", "description": "DeepSeek-native terminal coding agent. The **harness** is engineered around prefix-cache stability for long-running sessions; the TUI is the shell.", "category": "coding-agent-products", "category_title": "Coding agent products (IDEs, CLIs, full suites)", "stars": 35065, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (terminal agent, prefix-cache tuned)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "unknown", "tags": [ "memory", "cli", "tui", "typescript" ], "example": { "label": "Project README", "url": "https://github.com/esengine/DeepSeek-Reasonix#readme" }, "deep_dive": null }, { "name": "vibe-kanban", "github_id": "BloopAI/vibe-kanban", "url": "https://github.com/BloopAI/vibe-kanban", "slug": "vibe-kanban", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#vibe-kanban", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/vibe-kanban/", "description": "Kanban-style fleet manager for running Claude Code, Codex, or any coding agent across many tasks at once. The **harness** contribution is the task-queue/review layer on top of whichever agent executes; not an agent loop itself.", "category": "coding-agent-products", "category_title": "Coding agent products (IDEs, CLIs, full suites)", "stars": 27893, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (task-fleet manager)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "unknown", "tags": [], "example": { "label": "Project README", "url": "https://github.com/BloopAI/vibe-kanban#readme" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "none", "rank": 1, "detail": "Isolation is limited to Git worktree branching so concurrent agent tasks don't collide on files; there is no execution/security sandbox of its own, since actual tool execution and any permission model belong to whichever orchestrated coding agent (Claude Code, Codex, etc.) runs the task.", "evidence": "https://deepwiki.com/BloopAI/vibe-kanban/6-git-and-github-integration" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "A local-first SQLite database (via SQLx) persists workflow state per task attempt/session — conversation and file-change history a user can reopen from the board — independent of the Git-managed code state.", "evidence": "https://raw.githubusercontent.com/BloopAI/vibe-kanban/HEAD/README.md" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "The board offers inline diff review to approve and merge a completed task attempt, an after-the-fact human review/merge gate rather than a hook that blocks an individual tool call before it executes.", "evidence": "https://deepwiki.com/BloopAI/vibe-kanban/5.6-kanban-board-and-issue-management" }, "prompt_optimization": { "rating": "none", "rank": 1, "detail": "No system-prompt customization or prompt tuning feature owned by vibe-kanban itself is documented; each orchestrated coding agent retains its own prompt configuration.", "evidence": "https://raw.githubusercontent.com/BloopAI/vibe-kanban/HEAD/README.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Primary artifact is the self-hosted local app (npx vibe-kanban, Rust/SQLite backend) that orchestrates external agents; a self-hosted \"Vibe Kanban Cloud\" instance option is also documented. The project has since announced it is sunsetting.", "evidence": "https://raw.githubusercontent.com/BloopAI/vibe-kanban/HEAD/README.md" } } }, { "name": "crush", "github_id": "charmbracelet/crush", "url": "https://github.com/charmbracelet/crush", "slug": "crush", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#crush", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/crush/", "description": "Charm's terminal coding agent (Charm's fork of the original OpenCode). The **harness** is the tool-calling loop with session persistence; the Bubble Tea TUI is the shell.", "category": "coding-agent-products", "category_title": "Coding agent products (IDEs, CLIs, full suites)", "stars": 27601, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (terminal agent, TUI)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "resumable", "recovery_rank": 3, "license_signal": "restricted (FSL-1.1-MIT)", "tags": [ "memory", "cli", "tui" ], "example": { "label": "Crush launch post", "url": "https://charm.land/blog/crush-comes-home/" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "Tool calls run on the host and require an interactive permission prompt by default, with a per-tool allowlist config or a blanket --yolo flag to skip prompts; there is no container/VM/enforced-filesystem isolation.", "evidence": "https://github.com/charmbracelet/crush/blob/main/README.md" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Sessions and messages persist to a local SQLite database (internal/db) so conversations resume across restarts, and Crush auto-summarizes into an IsSummaryMessage checkpoint (session.SummaryMessageID) once a conversation nears the model's context limit.", "evidence": "https://deepwiki.com/charmbracelet/crush/4.4-message-history-and-summarization" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "The PreToolUse hook event runs user-defined shell commands before a tool executes and can deny the call (exit code 2, blocking it before it runs), rewrite its input via updated_input, or auto-allow it, bypassing the permission prompt.", "evidence": "https://github.com/charmbracelet/crush/blob/main/docs/hooks/README.md" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Crush layers project instructions from AGENTS.md, CRUSH.md, CLAUDE.md, GEMINI.md (and .local variants) in the working directory into Go-template system prompts (internal/agent/templates/*.md.tpl); there is no automatic prompt-tuning loop.", "evidence": "https://github.com/charmbracelet/crush/blob/main/AGENTS.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Crush installs as a self-hosted terminal CLI you run and configure yourself (providers, permissions, hooks, context files); Charm's Hyper is an optional hosted model-provider subscription, not a required execution backend.", "evidence": "https://github.com/charmbracelet/crush/blob/main/README.md" } } }, { "name": "qwen-code", "github_id": "QwenLM/qwen-code", "url": "https://github.com/QwenLM/qwen-code", "slug": "qwen-code", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#qwen-code", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/qwen-code/", "description": "Alibaba's official terminal coding agent, forked from Gemini CLI's agent loop and retuned for Qwen models. The **harness** is the same sandboxed tool-call loop as its upstream; the terminal is the shell.", "category": "coding-agent-products", "category_title": "Coding agent products (IDEs, CLIs, full suites)", "stars": 27310, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (official CLI, Gemini-CLI fork)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "resumable", "recovery_rank": 3, "license_signal": "unknown", "tags": [ "sandbox", "cli", "typescript" ], "example": { "label": "Project README", "url": "https://github.com/QwenLM/qwen-code#readme" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "A native sandbox mode (QWEN_SANDBOX=true|docker|podman|sandbox-exec, or -s) runs tool execution inside macOS Seatbelt or a Docker/Podman container, but it is opt-in - default execution is on the host.", "evidence": "https://qwenlm.github.io/qwen-code-docs/en/users/features/sandbox/" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Auto-memory automatically writes learned preferences, corrections, and project context to ~/.qwen/projects//memory/ after each session, with a daily background 'dream' pass that deduplicates and cleans stale entries, plus dedicated PreCompact/PostCompact hooks around conversation compaction.", "evidence": "https://qwenlm.github.io/qwen-code-docs/en/users/features/memory/" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "The PreToolUse hook returns a permissionDecision of allow/deny/ask (or exit code 2) to block a tool call before it executes, alongside PostToolUse, SessionStart/End, PreCompact/PostCompact, and other lifecycle events.", "evidence": "https://qwenlm.github.io/qwen-code-docs/en/users/features/hooks/" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "QWEN.md/GEMINI.md context files are discovered hierarchically (global, project root, nested directories) and concatenated verbatim into the system prompt; there is no automatic tuning of the written instructions themselves.", "evidence": "https://qwenlm.github.io/qwen-code-docs/en/users/features/memory/" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Qwen Code installs as a self-hosted terminal CLI (with an optional qwen serve daemon mode for shared HTTP+SSE sessions) that you configure and run yourself against any Qwen-compatible model endpoint; there is no required hosted execution backend.", "evidence": "https://raw.githubusercontent.com/QwenLM/qwen-code/main/README.md" } } }, { "name": "Kilo Code", "github_id": "Kilo-Org/kilocode", "url": "https://github.com/Kilo-Org/kilocode", "slug": "kilocode", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#kilocode", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/kilocode/", "description": "VS Code extension and CLI in the Cline/Roo-Code lineage — a natural pick now that Roo-Code is archived upstream. The **harness** is an approval-gated autonomous-mode loop with a provider/tool marketplace; the IDE is the shell.", "category": "coding-agent-products", "category_title": "Coding agent products (IDEs, CLIs, full suites)", "stars": 26978, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (IDE extension + CLI, MCP)", "autonomy": "step-gated", "autonomy_rank": 1, "recovery": "resumable", "recovery_rank": 3, "license_signal": "unknown", "tags": [ "mcp", "cli", "ide", "typescript" ], "example": { "label": "Project README", "url": "https://github.com/Kilo-Org/kilocode#readme" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "The extension/CLI runs tool calls with the full privileges of the host process and relies on an allow/ask/deny permission-prompt system for confirmation, not isolation; per a third-party security analysis, \"the extension runs with the full privileges of the VS Code extension host process\" with no built-in sandboxing.", "evidence": "https://agent-safehouse.dev/docs/agent-investigations/kilo-code" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Auto-compaction triggers on configured token-usage or safety-buffer thresholds, condenses older history into a running summary while keeping the most recent turns verbatim within a configurable token budget, and separately prunes stale tool outputs beyond a 40,000-token window between turns.", "evidence": "https://kilo.ai/docs/customize/context/context-condensing" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "The plugin API's tool.execute.before hook fires before a tool runs and can block it by throwing an error (e.g. the documented example denies reads of .env files), applying to both the CLI and VS Code extension.", "evidence": "https://kilo.ai/docs/automate/extending/plugins" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Custom modes/agents are authored as Markdown files with YAML frontmatter (or kilo.jsonc entries) whose body becomes the system prompt verbatim, plus AGENTS.md project instructions; there is no automatic tuning or optimization of the written prompt text.", "evidence": "https://kilo.ai/docs/customize/custom-modes" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "The primary published artifact is a self-hosted VS Code/JetBrains extension and CLI you configure with modes, rules, and permissions; Kilo Cloud Agents (isolated per-user Linux containers, remote execution) are an optional hosted tier layered on top, not required for the core loop.", "evidence": "https://kilo.ai/docs/code-with-ai/platforms/cloud-agent" } } }, { "name": "Symphony", "github_id": "openai/symphony", "url": "https://github.com/openai/symphony", "slug": "symphony", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#symphony", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/symphony/", "description": "OpenAI's harness for fanning a task out into many isolated, autonomous coding-agent implementation runs and surfacing the ones that pass, so a team manages outcomes instead of supervising each session.", "category": "coding-agent-products", "category_title": "Coding agent products (IDEs, CLIs, full suites)", "stars": 26812, "tier": "complex", "tier_rank": 4, "axis": "complex (parallel isolated runs — product suite)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "resumable", "recovery_rank": 3, "license_signal": "unknown", "tags": [ "sandbox" ], "example": { "label": "Project README", "url": "https://github.com/openai/symphony#readme" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "The spec explicitly makes \"mandating strong sandbox controls beyond what the coding agent and host OS provide\" a non-goal, so isolation is whatever the wrapped coding agent and host OS already offer; each implementation defines its own trust boundary rather than Symphony enforcing one.", "evidence": "https://raw.githubusercontent.com/openai/symphony/main/SPEC.md" }, "context_memory": { "rating": "none", "rank": 1, "detail": "The orchestrator is explicitly stateless beyond the filesystem: \"no persistent database is mandated,\" scheduler state recovers by re-reading the issue tracker, and per-issue workspace directories persist only as git/file state, not conversational context or memory.", "evidence": "https://raw.githubusercontent.com/openai/symphony/main/SPEC.md" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "before_run/after_run/after_create/before_remove are run-level shell-script callbacks around a whole implementation attempt (before_run can abort the attempt before it starts) rather than hooks that intercept individual tool calls inside the agent's own loop.", "evidence": "https://raw.githubusercontent.com/openai/symphony/main/SPEC.md" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "WORKFLOW.md's Markdown body is rendered as the system prompt with strict variable substitution (issue fields, attempt metadata); rendering fails the run rather than falling back, and there is no automatic prompt tuning.", "evidence": "https://raw.githubusercontent.com/openai/symphony/main/SPEC.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Symphony is a self-hosted orchestrator daemon (polls an issue tracker, spawns workers as local or SSH subprocesses, optional observability HTTP API) that you configure via WORKFLOW.md and hooks; \"rich web UI or multi-tenant control plane\" is explicitly not a goal, so there is no managed backend.", "evidence": "https://raw.githubusercontent.com/openai/symphony/main/SPEC.md" } } }, { "name": "oh-my-pi", "github_id": "can1357/oh-my-pi", "url": "https://github.com/can1357/oh-my-pi", "slug": "oh-my-pi", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#oh-my-pi", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/oh-my-pi/", "description": "Terminal coding agent (fork of Pi) that wires the IDE into the **harness**: hash-anchored edits, a 32-tool loop tuned per-model, LSP rename/references/diagnostics on every write, a real DAP debugger (lldb/dlv/debugpy), long-lived Python + Bun execution kernels that call back into the agent's tools, browser control, and 40+ providers (Claude/OpenAI/Gemini/local). ~55k-line Rust core.", "category": "coding-agent-products", "category_title": "Coding agent products (IDEs, CLIs, full suites)", "stars": 26658, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (terminal agent, LSP/DAP, multi-provider)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "browser", "provider-agnostic", "cli", "ide", "rust", "python" ], "example": { "label": "LSP wired into edits", "url": "https://github.com/can1357/oh-my-pi/blob/main/docs/lsp-config.md" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "Tools declare an approval tier (read/write/exec) checked against tools.approval. policy and an approvalMode (always-ask/write/yolo), but the default mode is yolo which auto-approves everything on the host; pi-iso's APFS/btrfs/overlayfs worktree cloning is copy-on-write workspace isolation for concurrent subagents, not a security sandbox, which is why third-party Docker wrappers (pi-docker-sandbox, pi-less-yolo) exist to add real containment.", "evidence": "https://github.com/can1357/oh-my-pi/blob/main/docs/approval-mode.md" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "The Hindsight subsystem lets the agent write durable facts mid-run (retain), search them (recall), and synthesize over them (reflect) against a local or cloud memory bank, and compresses each session into a project-scoped 'mental model' that auto-loads on the first turn of the next session.", "evidence": "https://hindsight.vectorize.io/blog/2026/06/08/oh-my-pi-hindsight-memory" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "The tool_call hook event fires pre-execution and any handler returning { block: true, reason } stops the call before it runs (a thrown error also fails closed and blocks), alongside session_before_compact, turn_start/end, and tool_result post-execution hooks.", "evidence": "https://github.com/can1357/oh-my-pi/blob/main/docs/hooks.md" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "System prompts are assembled at session start from Handlebars templates plus discovered project context files (AGENTS.md, SYSTEM.md, and third-party formats like Cursor MDC or Cline .clinerules); composition is config/template-driven with no automatic tuning loop.", "evidence": "https://github.com/can1357/oh-my-pi/blob/main/AGENTS.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "omp is a self-hosted terminal binary (macOS/Linux/Windows) you configure with extensions, hooks, and approval policy; the optional Hindsight memory backend can point at a hosted Cloud endpoint or a self-hosted Docker server, but the core agent runs locally with no required managed control plane.", "evidence": "https://github.com/can1357/oh-my-pi/blob/main/README.md" } } }, { "name": "Roo Code", "github_id": "RooCodeInc/Roo-Code", "url": "https://github.com/RooCodeInc/Roo-Code", "slug": "roo-code", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#roo-code", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/roo-code/", "description": "VS Code/Cursor extension in the Cline lineage. The **harness** is the approval-gated agent with custom modes and a strong MCP story; the IDE is the UI. Popular community fork when you want that workflow without the upstream extension.", "category": "coding-agent-products", "category_title": "Coding agent products (IDEs, CLIs, full suites)", "stars": 24326, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (IDE extension, MCP-first)", "autonomy": "step-gated", "autonomy_rank": 1, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "mcp", "workflow", "ide", "typescript" ], "example": { "label": "Custom modes guide", "url": "https://docs.roocode.com/features/custom-modes" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "Actions run on the host with the user's privileges behind a per-category (reads/writes/execute/MCP) approval-prompt system; disabling prompts via auto-approve removes the confirmation step but adds no containment layer.", "evidence": "https://roocodeinc.github.io/Roo-Code/features/auto-approving-actions" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Intelligent Context Condensing automatically summarizes earlier conversation once usage crosses a configured context-window percentage, and original (pre-condensed) messages are preserved and restorable by rewinding to a Checkpoint.", "evidence": "https://roocodeinc.github.io/Roo-Code/features/intelligent-context-condensing" }, "lifecycle_hooks": { "rating": "none", "rank": 1, "detail": "There is no user-definable pre/post tool-call hook surface; open feature requests (e.g. issue #11504, #12025) explicitly ask for Claude Code-style pre-tool-use/post-tool-use hooks that do not yet exist, so interception is limited to the built-in approval-prompt flow.", "evidence": "https://github.com/RooCodeInc/Roo-Code/issues/11504" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Custom Modes compose the system prompt from a manually authored Role Definition (prepended) and Custom Instructions (appended), plus mode-specific rule files under .roo/rules-{mode-slug}/; there is no automatic prompt tuning.", "evidence": "https://roocodeinc.github.io/Roo-Code/features/custom-modes" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "The published artifact was a self-hosted VS Code/JetBrains extension configured via modes and permission settings, with an optional paid Roo Code Cloud/Roomote layer for remote agents and task sharing; note this repository was archived by the owner on 2026-05-15 and the extension itself shut down, with the team pivoting fully to the hosted Roomote cloud agent.", "evidence": "https://github.com/RooCodeInc/Roo-Code" } } }, { "name": "jcode", "github_id": "1jehuang/jcode", "url": "https://github.com/1jehuang/jcode", "slug": "jcode", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#jcode", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/jcode/", "description": "Rust terminal coding agent pitched as the most RAM-efficient **harness** in its class; MCP support, multi-provider (Claude/OpenAI).", "category": "coding-agent-products", "category_title": "Coding agent products (IDEs, CLIs, full suites)", "stars": 18308, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (terminal agent, low-memory)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "unknown", "tags": [ "mcp", "memory", "provider-agnostic", "cli", "rust" ], "example": { "label": "Project README", "url": "https://github.com/1jehuang/jcode#readme" }, "deep_dive": null }, { "name": "eigent", "github_id": "eigent-ai/eigent", "url": "https://github.com/eigent-ai/eigent", "slug": "eigent", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#eigent", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/eigent/", "description": "Open-source desktop **harness** positioned as a local, free alternative to Claude Cowork and Codex: multi-agent workspace orchestration in a self-hosted app rather than a hosted product.", "category": "coding-agent-products", "category_title": "Coding agent products (IDEs, CLIs, full suites)", "stars": 15083, "tier": "complex", "tier_rank": 4, "axis": "complex (desktop multi-agent workspace — product suite)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "unknown", "tags": [ "multi-agent", "local" ], "example": { "label": "Project README", "url": "https://github.com/eigent-ai/eigent#readme" }, "deep_dive": null }, { "name": "cc-haha", "github_id": "NanmiCoder/cc-haha", "url": "https://github.com/NanmiCoder/cc-haha", "slug": "cc-haha", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#cc-haha", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/cc-haha/", "description": "Local-first desktop workspace **harness** for Claude Code and other agents: multi-agent sessions, Git worktrees, code diffs, a skill marketplace, and chat-app access (WeChat, Telegram, WhatsApp).", "category": "coding-agent-products", "category_title": "Coding agent products (IDEs, CLIs, full suites)", "stars": 14189, "tier": "complex", "tier_rank": 4, "axis": "complex (desktop workspace, multi-agent — product suite)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "unknown", "tags": [ "memory", "multi-agent", "typescript" ], "example": { "label": "Project README", "url": "https://github.com/NanmiCoder/cc-haha#readme" }, "deep_dive": null }, { "name": "claw-code-agent", "github_id": "HarnessLab/claw-code-agent", "url": "https://github.com/HarnessLab/claw-code-agent", "slug": "claw-code-agent", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#claw-code-agent", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/claw-code-agent/", "description": "Python reimplementation of the Claude Code agent architecture with zero external dependencies; interactive chat, streaming, plugin runtime, nested agent delegation, cost tracking, MCP transport—portable harness without the Rust/TS toolchain.", "category": "coding-agent-products", "category_title": "Coding agent products (IDEs, CLIs, full suites)", "stars": 543, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (pure Python, plugin runtime)", "autonomy": "checkpoint-gated", "autonomy_rank": 2, "recovery": "none", "recovery_rank": 1, "license_signal": "unknown", "tags": [ "mcp", "rust", "python", "typescript" ], "example": { "label": "Quick Start guide", "url": "https://github.com/HarnessLab/claw-code-agent#-quick-start" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "Access is gated by CLI-selected permission tiers (read-only default, --allow-write, --allow-shell, --unsafe) that filter which tools the model can see; this is a host-level permission gate with no container/VM/restricted-filesystem isolation documented for the agent itself.", "evidence": "https://raw.githubusercontent.com/HarnessLab/claw-code-agent/main/README.md" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Auto-compact summarizes conversation history while preserving recent turns when the tracked token count nears the limit, and sessions auto-save with file-history journaling and snapshot IDs so a session resumes mid-task across restarts.", "evidence": "https://raw.githubusercontent.com/HarnessLab/claw-code-agent/main/README.md" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "The six documented plugin hooks (beforePrompt, afterTurn, onResume, beforePersist, beforeDelegate, afterDelegate) intercept at the turn/session/delegation level, not individual tool calls; separately, .claw-policy.json's HookPolicyRuntime.denied_tool_message() can preemptively deny a tool by exact name or prefix match before execution, but only as a static allow/deny list rather than a programmable before/after hook with call-level tracing.", "evidence": "https://raw.githubusercontent.com/HarnessLab/claw-code-agent/main/src/hook_policy.py" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "--system-prompt, --append-system-prompt, and --override-system-prompt CLI flags (plus a live GUI prompt editor) let users replace or extend the rendered system prompt; there is no automatic prompt-tuning loop.", "evidence": "https://raw.githubusercontent.com/HarnessLab/claw-code-agent/main/README.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Claw Code Agent is a self-hosted Python CLI/library with a bundled localhost-only FastAPI GUI; it requires only a local Python runtime and an OpenAI-compatible model endpoint (vLLM, Ollama, LiteLLM, OpenRouter), with no hosted execution backend of its own.", "evidence": "https://raw.githubusercontent.com/HarnessLab/claw-code-agent/main/README.md" } } }, { "name": "AgentBox", "github_id": "madarco/agentbox", "url": "https://github.com/madarco/agentbox", "slug": "agentbox", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#agentbox", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/agentbox/", "description": "Runs multiple coding agents in parallel, each in its own sandboxed VM, locally or in the cloud, from one command. The **harness** contribution is the VM-per-agent isolation and fleet fan-out layer; whichever agent runs inside owns the loop.", "category": "coding-agent-products", "category_title": "Coding agent products (IDEs, CLIs, full suites)", "stars": 374, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (VM-per-agent sandbox, parallel fan-out)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "sandbox", "typescript" ], "example": { "label": "Parallel agents quick start", "url": "https://github.com/madarco/agentbox#readme" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "strong", "rank": 3, "detail": "Each agent runs in its own sandboxed box: locally a Docker container with a FUSE overlay filesystem, or a cloud VM (Hetzner, Vercel, Daytona, E2B, DigitalOcean); isolation is the project's core shipped mechanism, not delegated to the user.", "evidence": "https://github.com/madarco/agentbox" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "agentbox pause/unpause and agentbox checkpoint freeze and resume a box's full environment state sub-second from a previous checkpoint, giving session resume of the whole sandboxed workspace (including whatever the inner agent had persisted) across restarts.", "evidence": "https://github.com/madarco/agentbox" }, "lifecycle_hooks": { "rating": "none", "rank": 1, "detail": "There is no documented hooks/callback system for intercepting agent actions; AgentBox's extensibility is limited to a provider-plugin SDK for adding new infrastructure backends, not for observing or blocking tool calls made by the agent running inside a box.", "evidence": "https://github.com/madarco/agentbox" }, "prompt_optimization": { "rating": "none", "rank": 1, "detail": "AgentBox has no system-prompt layer of its own; it explicitly passes through whichever wrapped agent's native configuration (\"bring all your skills, plugins, and settings for Claude Code, Codex, Open Code\") unchanged into the box.", "evidence": "https://github.com/madarco/agentbox" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "AgentBox is a self-hosted CLI you run against your own local Docker or your own cloud provider credentials; it orchestrates fleet fan-out and isolation as an opinionated runtime you configure, with no AgentBox-operated managed backend.", "evidence": "https://github.com/madarco/agentbox" } } }, { "name": "Proliferate", "github_id": "proliferate-ai/proliferate", "url": "https://github.com/proliferate-ai/proliferate", "slug": "proliferate", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#proliferate", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/proliferate/", "description": "Open-source AI IDE for Claude Code, Codex, OpenCode, and more. The **harness** contribution is the workspace/session orchestration layer: run multiple coding agents in parallel, locally or in the cloud, with isolated workspaces, reusable workflows, and shared team context.", "category": "coding-agent-products", "category_title": "Coding agent products (IDEs, CLIs, full suites)", "stars": 310, "tier": "complex", "tier_rank": 4, "axis": "complex (multi-agent workspace orchestration — product suite)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "multi-agent", "sandbox", "ide", "typescript" ], "example": { "label": "Product README", "url": "https://github.com/proliferate-ai/proliferate#readme" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "strong", "rank": 3, "detail": "Cloud runs execute inside isolated cloud sandbox environments provisioned per task, with a credential gateway that keeps real API keys/subscriptions off the sandbox and issues only short-lived tokens; local runs use a lighter git-worktree-per-task isolation.", "evidence": "https://raw.githubusercontent.com/proliferate-ai/proliferate/main/README.md" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "A running workspace (agent session state, git history, and in-progress changes) can be moved between local machine and cloud mid-task with history intact, i.e. native session resume/mobility across environments rather than a stateless per-run invocation.", "evidence": "https://raw.githubusercontent.com/proliferate-ai/proliferate/main/README.md" }, "lifecycle_hooks": { "rating": "unknown", "rank": 0, "detail": "Looked for a pre-tool-call interception hook in the README, the docs/product/agents page, and the changelog; found only a reference to admin-configurable 'guardrails' and 'admin policies' with no documented mechanism for how or where they intercept an agent's actions.", "evidence": "" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "MCP servers, skills, and custom tools are configured once and shared by every agent/workspace, and dedicated plan-review and code-review agents (reusable reviewer prompts) check work before it lands; there is no automatic prompt tuning.", "evidence": "https://raw.githubusercontent.com/proliferate-ai/proliferate/main/README.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "The published GitHub artifact is an AGPL-3.0 self-hosted control plane (Caddy + Postgres + API, deployable via Docker Compose or AWS CloudFormation) you run yourself; Proliferate Cloud (beta) is an optional managed hosted alternative, not the primary installed artifact.", "evidence": "https://raw.githubusercontent.com/proliferate-ai/proliferate/main/README.md" } } }, { "name": "superpowers", "github_id": "obra/superpowers", "url": "https://github.com/obra/superpowers", "slug": "superpowers", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#superpowers", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/superpowers/", "description": "Performance-oriented harness pack for Claude Code and 13 other harnesses (Codex, Cursor, OpenCode, Gemini CLI, more): skills, instincts, memory, security, research-first workflows. Treats harness engineering itself as the performance lever.", "category": "coding-harness-configs", "category_title": "Coding harness configs and SDKs", "stars": 276518, "tier": "complex", "tier_rank": 4, "axis": "complex (multi-IDE skill stack — product suite)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "memory", "cli", "ide" ], "example": { "label": "TDD skill", "url": "https://github.com/obra/superpowers/blob/main/skills/test-driven-development/SKILL.md" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "none", "rank": 1, "detail": "Superpowers' own isolation mechanism is git-worktree-based (\"isolated workspace on new branch\") for change management, not execution sandboxing; it ships no container/VM/allowlist of its own and entirely inherits whatever sandboxing (or lack of it) the underlying harness (Claude Code, Codex, OpenCode, Cursor) already provides.", "evidence": "https://raw.githubusercontent.com/obra/superpowers/main/README.md" }, "context_memory": { "rating": "basic", "rank": 2, "detail": "hooks/hooks.json registers only a SessionStart hook that re-injects the using-superpowers bootstrap instructions at session startup and again after the host harness compacts, which manages live-session continuity but implements no persistent memory or compaction of its own; a separate companion plugin (obra/episodic-memory, installable via the superpowers marketplace) adds vector-indexed cross-session recall but is a distinct optional install, not part of this repo.", "evidence": "https://raw.githubusercontent.com/obra/superpowers/main/hooks/hooks.json" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "The only hook event registered is SessionStart, a run-level kickoff callback that injects text at session start/compact; it cannot intercept, block, or rewrite an individual tool call, and skill activation itself is a soft model instruction (\"you do not have a choice, you MUST use it\") rather than a technical enforcement gate.", "evidence": "https://github.com/obra/superpowers/blob/main/skills/using-superpowers/SKILL.md" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Composable SKILL.md files are loaded/injected into the host harness's context on a conditional, task-matched basis (progressive disclosure), letting users add or author skills that steer agent behavior; there is no automatic tuning of the skill text itself.", "evidence": "https://github.com/obra/superpowers/blob/main/skills/using-superpowers/SKILL.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Superpowers installs as a self-hosted skill/plugin pack layered onto an existing harness (Claude Code marketplace plugin, Codex/OpenCode/Cursor equivalents) rather than a standalone runtime you buy into; there is no hosted execution backend.", "evidence": "https://raw.githubusercontent.com/obra/superpowers/main/README.md" } } }, { "name": "Anthropic Skills", "github_id": "anthropics/skills", "url": "https://github.com/anthropics/skills", "slug": "skills", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#skills", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/skills/", "description": "Anthropic's official Agent Skills repository: SKILL.md-based folders (instructions, scripts, resources) Claude dynamically loads on Claude Code, Claude.ai, and the API. The reference for progressive-disclosure skill packs in 2026.", "category": "coding-harness-configs", "category_title": "Coding harness configs and SDKs", "stars": 171127, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (official skills format)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "restricted (Anthropic terms)", "tags": [], "example": { "label": "docx skill", "url": "https://github.com/anthropics/skills/blob/main/skills/docx/SKILL.md" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "none", "rank": 1, "detail": "This is a pure content repository of SKILL.md folders with no runtime code; execution isolation, if any, happens entirely at whichever platform loads the skill (Claude Code, Claude.ai, the API), not within this repo.", "evidence": "https://raw.githubusercontent.com/anthropics/skills/main/README.md" }, "context_memory": { "rating": "none", "rank": 1, "detail": "No context or memory management mechanism is defined in the repository; skills are loaded on demand into whatever conversation context the host platform already manages.", "evidence": "https://raw.githubusercontent.com/anthropics/skills/main/README.md" }, "lifecycle_hooks": { "rating": "none", "rank": 1, "detail": "No hooks, callbacks, or interception points are defined; skill content only shapes behavior by being present in context when Claude decides a skill applies, with no programmatic before/after execution surface.", "evidence": "https://raw.githubusercontent.com/anthropics/skills/main/README.md" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Skills are SKILL.md files (YAML frontmatter name/description plus markdown instructions, scripts, and resources) that Claude discovers and loads dynamically into its prompt/context on a progressive-disclosure basis; authoring is manual and there is no automatic tuning of skill text.", "evidence": "https://raw.githubusercontent.com/anthropics/skills/main/README.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "The repo is the reference format for a config-driven skill/plugin system (folders of instructions/scripts/resources you register with a harness) rather than an SDK you import to write your own loop or a hosted service; it has no runtime or managed backend of its own and is consumed by Claude Code, Claude.ai, or the API.", "evidence": "https://raw.githubusercontent.com/anthropics/skills/main/README.md" } } }, { "name": "GStack", "github_id": "garrytan/gstack", "url": "https://github.com/garrytan/gstack", "slug": "gstack", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#gstack", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/gstack/", "description": "Garry Tan's Claude Code skill stack: 23 slash-command modes (CEO/eng/design review, QA, ship, browse, retro, …) that structure one assistant as a virtual engineering team. Daily driver while running YC.", "category": "coding-harness-configs", "category_title": "Coding harness configs and SDKs", "stars": 129281, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (multi-role slash-command harness)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "typescript" ], "example": { "label": "/ship SKILL.md", "url": "https://github.com/garrytan/gstack/blob/main/ship/SKILL.md" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "General tool execution (edits, shell) is host-level with configurable guardrails only: /freeze restricts edits to one directory and /careful warns before destructive commands; the one enforced piece is scoped to the browser skill, whose Chromium sessions run behind a deny-default Chrome DevTools Protocol allowlist and per-tab token scoping.", "evidence": "https://raw.githubusercontent.com/garrytan/gstack/main/README.md" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "A three-layer system persists beyond the session: /learn writes per-repo project learnings to $GSTACK_STATE_ROOT/projects/$SLUG/ that auto-integrate into downstream skills, and optional GBrain integration (local PGLite or Supabase) gives indexed, cross-session code search and shared team memory.", "evidence": "https://raw.githubusercontent.com/garrytan/gstack/main/README.md" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "Continuous checkpoint mode auto-commits work with a WIP: prefix after each step (a post-hoc, observable checkpoint rather than a pre-execution gate), and /context-restore replays state from those commits; there is no hook that blocks or rewrites a tool call before it runs.", "evidence": "https://raw.githubusercontent.com/garrytan/gstack/main/README.md" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "A project CLAUDE.md declares available skills and routing rules, gstack-config holds global settings (telemetry, checkpoint_mode, command prefix), and each SKILL.md is a manually authored template Claude loads to decide when to invoke it; there is no automatic prompt tuning.", "evidence": "https://raw.githubusercontent.com/garrytan/gstack/main/README.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "GStack is explicitly stated to have \"no runtime of its own\": it is an MIT-licensed skill/slash-command pack (Markdown + shell scripts) layered onto an existing Claude Code install, with an optional on-demand browser daemon and optional GBrain backend; no vendor-hosted execution stack is required.", "evidence": "https://raw.githubusercontent.com/garrytan/gstack/main/README.md" } } }, { "name": "addyosmani/agent-skills", "github_id": "addyosmani/agent-skills", "url": "https://github.com/addyosmani/agent-skills", "slug": "agent-skills", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#agent-skills", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/agent-skills/", "description": "Addy Osmani's production-grade skill pack: 24 engineering skills and 4 specialist agent personas that encode senior-dev workflows (spec through deploy) across 70+ coding agents including Claude Code, Cursor, and Copilot. The **harness** contribution is the skill/workflow layer, not a new agent loop.", "category": "coding-harness-configs", "category_title": "Coding harness configs and SDKs", "stars": 89221, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (skills bundle, cross-agent)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "workflow", "ide" ], "example": { "label": "Project README", "url": "https://github.com/addyosmani/agent-skills#readme" }, "deep_dive": null }, { "name": "awesome-claude-code", "github_id": "hesreallyhim/awesome-claude-code", "url": "https://github.com/hesreallyhim/awesome-claude-code", "slug": "awesome-claude-code", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#awesome-claude-code", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/awesome-claude-code/", "description": "Large community-curated index of Claude Code skills, slash commands, status lines, and plugins—resources for extending the harness, not a harness itself, but the most-followed catalog of the genre.", "category": "coding-harness-configs", "category_title": "Coding harness configs and SDKs", "stars": 52857, "tier": "super simple", "tier_rank": 1, "axis": "super simple (curated resource index)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "unknown", "tags": [], "example": { "label": "Project README", "url": "https://github.com/hesreallyhim/awesome-claude-code#readme" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "none", "rank": 1, "detail": "The repo is a hand-picked Markdown index of third-party Claude Code resources (e.g. it links out to Cleat and Brood Box for Docker/microVM sandboxing); it ships no execution environment of its own to sandbox.", "evidence": "https://github.com/hesreallyhim/awesome-claude-code#readme" }, "context_memory": { "rating": "none", "rank": 1, "detail": "Static curated list of links with no running process, so there is no session, history, or retrieval state to manage — it only catalogs memory tools like Callimachus and Fable.", "evidence": "https://github.com/hesreallyhim/awesome-claude-code#readme" }, "lifecycle_hooks": { "rating": "none", "rank": 1, "detail": "No hook/callback system exists in the project itself; it links to community docs like \"Claude Code Hooks: Complete Guide\" without implementing interception of its own.", "evidence": "https://github.com/hesreallyhim/awesome-claude-code#readme" }, "prompt_optimization": { "rating": "none", "rank": 1, "detail": "The README is descriptive prose/links, not a system prompt, rules file, or tuning mechanism.", "evidence": "https://github.com/hesreallyhim/awesome-claude-code#readme" }, "build_vs_buy": { "tier": 1, "label": "build", "detail": "The repo installs nothing runnable — it's a bibliography of external tools/skills you'd assemble yourself, the closest fit among the three tiers to \"no opinionated runtime.\"", "evidence": "https://github.com/hesreallyhim/awesome-claude-code#readme" } } }, { "name": "wshobson/agents", "github_id": "wshobson/agents", "url": "https://github.com/wshobson/agents", "slug": "agents", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#agents", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/agents/", "description": "Cross-harness marketplace of drop-in subagents and skills for Claude Code, Codex CLI, Cursor, OpenCode, and Copilot; specialized, production-ready agent definitions you install rather than hand-write.", "category": "coding-harness-configs", "category_title": "Coding harness configs and SDKs", "stars": 39042, "tier": "super simple", "tier_rank": 1, "axis": "super simple (drop-in agent packs)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "multi-agent", "cli", "ide" ], "example": { "label": "Agent catalog", "url": "https://github.com/wshobson/agents#readme" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "none", "rank": 1, "detail": "A cross-harness marketplace of agent/skill/command markdown definitions installed into the host tool (Claude Code, Codex, Cursor, OpenCode, Copilot); it has no execution or isolation layer of its own — that's entirely delegated to whichever harness runs it.", "evidence": "https://github.com/wshobson/agents#readme" }, "context_memory": { "rating": "basic", "rank": 2, "detail": "Skills use a progressive-disclosure pattern (SKILL.md kept under 8KB for navigation, with references/ material loaded on demand) that scopes which skill/tool content enters context — this manages the tool surface, not conversation memory.", "evidence": "https://github.com/wshobson/agents/blob/main/docs/authoring.md" }, "lifecycle_hooks": { "rating": "none", "rank": 1, "detail": "No callback or interception system is documented; installation only wires markdown agent/skill/command files into the host harness's own discovery mechanism.", "evidence": "https://github.com/wshobson/agents#readme" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Each of the 203 agents and 175 skills is a markdown file with YAML frontmatter (name/description/model/tools) plus a freeform instructions body that users select, install, and can edit or fork locally — no automatic tuning loop.", "evidence": "https://github.com/wshobson/agents/blob/main/docs/authoring.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "A config-driven plugin/skill marketplace (94 plugins, 203 agents, 175 skills) you install into an existing harness rather than a library you import or a hosted service.", "evidence": "https://github.com/wshobson/agents#readme" } } }, { "name": "planning-with-files", "github_id": "OthmanAdi/planning-with-files", "url": "https://github.com/OthmanAdi/planning-with-files", "slug": "planning-with-files", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#planning-with-files", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/planning-with-files/", "description": "Skill for persistent, file-based planning across long-running coding-agent sessions: crash-proof markdown plans, session recovery after `/clear`/compaction, and a deterministic completion gate—Manus-style planning as a drop-in **harness** layer via the Agent Skills standard.", "category": "coding-harness-configs", "category_title": "Coding harness configs and SDKs", "stars": 26305, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (skill, file-based state)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "unknown", "tags": [ "memory" ], "example": { "label": "Project README", "url": "https://github.com/OthmanAdi/planning-with-files#readme" }, "deep_dive": null }, { "name": "SWE-agent", "github_id": "SWE-agent/SWE-agent", "url": "https://github.com/SWE-agent/SWE-agent", "slug": "swe-agent", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#swe-agent", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/swe-agent/", "description": "LM-driven harness built for SWE-bench: edit state, command execution, and issue-focused loop—the reference agent stack next to the benchmark itself.", "category": "coding-harness-configs", "category_title": "Coding harness configs and SDKs", "stars": 20112, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (SWE-bench pairing, stateful edits)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "memory", "evals", "python" ], "example": { "label": "Default agent config", "url": "https://github.com/SWE-agent/SWE-agent/blob/main/config/default.yaml" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "strong", "rank": 3, "detail": "Docker is the default execution backend for the agent's shell/edit commands, containerizing the issue-fixing environment out of the box.", "evidence": "https://swe-agent.com/latest/installation/source/" }, "context_memory": { "rating": "basic", "rank": 2, "detail": "History processors trim/manage the live run's (thought, action, observation) history and trajectories are logged to a .traj file, but nothing persists or resumes across separate runs.", "evidence": "https://swe-agent.com/latest/usage/trajectories/" }, "lifecycle_hooks": { "rating": "none", "rank": 1, "detail": "Tool behavior is controlled only through static YAML config (command blocklists, timeouts) with no documented callback/hook system that can intercept or veto an action before it runs.", "evidence": "https://swe-agent.com/latest/reference/tools_config/" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Prompts are Jinja-style templates (next_step_template, format_error_template, etc.) set per-agent in a single YAML config file, with no automatic tuning loop.", "evidence": "https://swe-agent.com/latest/config/templates/" }, "build_vs_buy": { "tier": 1, "label": "build", "detail": "SWE-agent installs as a pip/source Python CLI and library governed by a single YAML config that you run yourself (optionally against Docker); no vendor-hosted platform is required for core use.", "evidence": "https://raw.githubusercontent.com/SWE-agent/SWE-agent/main/README.md" } } }, { "name": "get-shit-done", "github_id": "open-gsd/gsd-core", "url": "https://github.com/open-gsd/gsd-core", "slug": "gsd-core", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#gsd-core", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/gsd-core/", "description": "Goal-backward planning and wave-based execution over fresh context windows; avoids context rot by design. Python/JS meta-prompting for Claude Code, OpenCode, Gemini CLI.", "category": "coding-harness-configs", "category_title": "Coding harness configs and SDKs", "stars": 8617, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (meta-prompting, you own stack)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "cli", "python" ], "example": { "label": "gsd:ship command", "url": "https://github.com/open-gsd/gsd-core/blob/next/commands/gsd/ship.md" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "The project's own documentation states there is explicitly no sandbox for third-party capabilities; installation is instead gated by a consent record, a SHA512 integrity pin on the fetched artifact, a reversibility path (gsd capability remove), and an optional registry allowlist (strict_known_registries) — approval/allowlist gates, not execution isolation.", "evidence": "https://github.com/open-gsd/gsd-core/blob/next/docs/CONFIGURATION.md" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Work is delegated to fresh-context subagents for research/planning/execution/verification so the orchestrator's own context stays lean, and every meaningful output is written to durable, human-readable .planning/ files (STATE.md, CONTEXT.md, RESEARCH.md, PLAN.md) that any workflow reads on start and writes back to on completion, surviving session boundaries.", "evidence": "https://github.com/open-gsd/gsd-core/blob/next/docs/explanation/context-engineering.md" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "gsd-read-guard.js is a PreToolUse hook that prevents Edit/Write on files the agent hasn't read (blocking before execution), alongside PostToolUse tracing/logging hooks (gsd-context-monitor.js injects context-budget warnings, gsd-session-state.sh and gsd-validate-commit.sh log state); a separate gsd-prompt-guard.js PreToolUse hook is advisory-only (logs, does not block).", "evidence": "https://www.opengsd.net/docs/v1/architecture" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "An agent_skills map in .planning/config.json injects project-specific SKILL.md files into individual subagent prompts (gsd-executor, gsd-planner, gsd-verifier) at spawn time, layered on top of CLAUDE.md; there is no automatic tuning of the injected instructions.", "evidence": "https://github.com/open-gsd/gsd-core/blob/next/docs/CONFIGURATION.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "GSD installs as a self-hosted npm script pack (slash commands, hooks, and .planning/ state files) layered onto an existing coding-agent CLI (Claude Code, Gemini CLI, OpenCode, Codex, and others) that you configure and run yourself; there is no hosted execution backend. Note: gsd-build/get-shit-done redirects upstream to the actively maintained open-gsd/gsd-core, which this research used as the current canonical source.", "evidence": "https://raw.githubusercontent.com/open-gsd/gsd-core/main/README.md" } } }, { "name": "Claude Agent SDK", "github_id": "anthropics/claude-agent-sdk-python", "url": "https://github.com/anthropics/claude-agent-sdk-python", "slug": "claude-agent-sdk-python", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#claude-agent-sdk-python", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/claude-agent-sdk-python/", "description": "Official Anthropic SDK (Python + [TypeScript](https://github.com/anthropics/claude-agent-sdk-typescript), [demos](https://github.com/anthropics/claude-agent-sdk-demos), [quickstarts](https://github.com/anthropics/claude-quickstarts)): built-in tools, MCP, long-running coding agents with session bridging.", "category": "coding-harness-configs", "category_title": "Coding harness configs and SDKs", "stars": 7957, "tier": "complex", "tier_rank": 4, "axis": "complex (full SDK, session bridging — product suite)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "mcp", "memory", "python", "typescript" ], "example": { "label": "Research agent demo", "url": "https://github.com/anthropics/claude-agent-sdk-demos/blob/main/research-agent/research_agent/agent.py" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "SDK apps can enable Claude Code's sandboxed Bash tool (Seatbelt on macOS, bubblewrap on Linux/WSL2) for kernel-enforced filesystem/network restrictions, but it is opt-in configuration - default Bash execution is on the host with permission prompts.", "evidence": "https://code.claude.com/docs/en/sandboxing" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Sessions are written to disk automatically and can be continued, resumed by ID, or forked, with PreCompact/PostCompact hooks around built-in conversation compaction.", "evidence": "https://code.claude.com/docs/en/agent-sdk/sessions" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "PreToolUse hook callbacks receive the tool call before it executes and can return permissionDecision: deny/allow/ask or an updatedInput to block or rewrite the action, alongside PostToolUse, Stop, and many other lifecycle events.", "evidence": "https://code.claude.com/docs/en/agent-sdk/hooks" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "System prompts are layered via the claude_code preset, append text, output-style files, or a fully custom string, plus CLAUDE.md project instructions injected into context; there is no automatic prompt tuning.", "evidence": "https://code.claude.com/docs/en/agent-sdk/modifying-system-prompts" }, "build_vs_buy": { "tier": 1, "label": "build", "detail": "The published artifact is the claude-agent-sdk Python/TypeScript package (query()/ClaudeSDKClient) that you import and compose into your own application/loop; Anthropic's hosted Claude Code and Managed Agents offerings are the tier-3 alternative for teams that don't want to own the runtime.", "evidence": "https://raw.githubusercontent.com/anthropics/claude-agent-sdk-python/main/README.md" } } }, { "name": "agents-cli", "github_id": "google/agents-cli", "url": "https://github.com/google/agents-cli", "slug": "agents-cli", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#agents-cli", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/agents-cli/", "description": "Google's official CLI and skill pack that layers agent-creation, evaluation, and deployment skills on top of whatever coding assistant you already run, rather than shipping its own agent loop—the **harness** as a config/skills add-on, not a new runtime.", "category": "coding-harness-configs", "category_title": "Coding harness configs and SDKs", "stars": 5708, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (skills/CLI layer, no new runtime)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "unknown", "tags": [ "evals", "cli" ], "example": { "label": "Project README", "url": "https://github.com/google/agents-cli#readme" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "unknown", "rank": 0, "detail": "Looked for Docker/container/subprocess isolation detail in the README; docs cover the dev/eval/deploy workflow and Google Cloud deployment targets (Agent Runtime, Cloud Run, GKE) without describing how code execution is isolated.", "evidence": "" }, "context_memory": { "rating": "unknown", "rank": 0, "detail": "Looked for session/conversation persistence or retrieval mechanisms; the README documents CLI commands for building, evaluating, and deploying ADK agents but says nothing about memory management.", "evidence": "" }, "lifecycle_hooks": { "rating": "unknown", "rank": 0, "detail": "Looked for pre/post tool-call interception; none documented — the tool's surface is eval/trace/deploy commands, not a runtime loop with hook points.", "evidence": "" }, "prompt_optimization": { "rating": "native", "rank": 3, "detail": "Ships `agents-cli eval optimize`, described as auto-tuning agent prompts using eval data — an automatic, eval-driven refinement loop rather than manual editing.", "evidence": "https://github.com/google/agents-cli#readme" }, "build_vs_buy": { "tier": 1, "label": "build", "detail": "Explicitly \"a tool for coding agents, not a coding agent itself\" — a CLI/skill pack layered on whatever coding assistant you already run, built on the ADK framework; it works from a local API key with no opinionated runtime of its own (Google Cloud Agent Runtime is an optional deployment target, not a requirement).", "evidence": "https://github.com/google/agents-cli#readme" } } }, { "name": "skillhub", "github_id": "iflytek/skillhub", "url": "https://github.com/iflytek/skillhub", "slug": "skillhub", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#skillhub", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/skillhub/", "description": "iFlytek's self-hosted registry for publishing, versioning, and governing agent skill packages—the **harness** config layer treated as an enterprise artifact store rather than a CLI or IDE shell.", "category": "coding-harness-configs", "category_title": "Coding harness configs and SDKs", "stars": 4891, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (skill registry/governance)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "unknown", "tags": [ "local", "cli", "ide" ], "example": { "label": "Project README", "url": "https://github.com/iflytek/skillhub#readme" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "none", "rank": 1, "detail": "SkillHub is a skill package registry/discovery platform (Spring Boot + Postgres + Redis), not an execution engine — it stores and versions skill packages but does not run them.", "evidence": "https://github.com/iflytek/skillhub#readme" }, "context_memory": { "rating": "none", "rank": 1, "detail": "As a registry (search, versioning, RBAC, audit logging) it has no agent conversation or task-context concept to manage.", "evidence": "https://github.com/iflytek/skillhub#readme" }, "lifecycle_hooks": { "rating": "none", "rank": 1, "detail": "No tool-call interception surface — the product's lifecycle is publish/discover/version, not agent execution.", "evidence": "https://github.com/iflytek/skillhub#readme" }, "prompt_optimization": { "rating": "none", "rank": 1, "detail": "No prompt tuning or system-prompt injection feature; it manages skill package metadata, not prompt content.", "evidence": "https://github.com/iflytek/skillhub#readme" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Self-hosted-only enterprise registry you deploy on your own infrastructure (Docker Compose/Kubernetes, \"full data sovereignty\") with RBAC and namespaces to configure — an opinionated runtime you run yourself.", "evidence": "https://github.com/iflytek/skillhub#readme" } } }, { "name": "Meta-Harness", "github_id": "stanford-iris-lab/meta-harness", "url": "https://github.com/stanford-iris-lab/meta-harness", "slug": "meta-harness", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#meta-harness", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/meta-harness/", "description": "Reference implementation from the Meta-Harness paper: an academic testbed for harness-engineering research, not a product—useful as a citation-grade baseline rather than something you'd run in production.", "category": "coding-harness-configs", "category_title": "Coding harness configs and SDKs", "stars": 1448, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (research reference implementation)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "unknown", "tags": [], "example": { "label": "Project README", "url": "https://github.com/stanford-iris-lab/meta-harness#readme" }, "deep_dive": null }, { "name": "RepoMaster", "github_id": "QuantaAlpha/RepoMaster", "url": "https://github.com/QuantaAlpha/RepoMaster", "slug": "repomaster", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#repomaster", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/repomaster/", "description": "Repo-scoped research harness: builds function-call and module-dependency graphs to explore only what's needed; large relative gains on MLE-bench and GitTaskBench with lower token use.", "category": "coding-harness-configs", "category_title": "Coding harness configs and SDKs", "stars": 543, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (graph-based exploration)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "none", "recovery_rank": 1, "license_signal": "unknown", "tags": [ "workflow", "python" ], "example": { "label": "PDF-parse case study", "url": "https://github.com/QuantaAlpha/RepoMaster/blob/main/example/pdf_parse.md" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "unknown", "rank": 0, "detail": "Looked in the README, USAGE.md, and the Mintlify docs site for Docker/subprocess/filesystem isolation detail around executing discovered-repo code; none of the three sources describe how execution is contained.", "evidence": "" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "The repository-exploration internals summarize and restart the chat history when a token-limit threshold (80000 tokens) is hit during a task, per token_limit_termination() — built-in compaction/summarization of the working context, not just budgeting.", "evidence": "https://mintlify.wiki/QuantaAlpha/RepoMaster/concepts/repository-exploration.md" }, "lifecycle_hooks": { "rating": "unknown", "rank": 0, "detail": "Looked in README/USAGE.md/docs site for pre/post tool-call interception or blocking; none found — documentation covers installation, model config, and the multi-agent dispatcher only.", "evidence": "" }, "prompt_optimization": { "rating": "unknown", "rank": 0, "detail": "Looked for prompt tuning or system-prompt customization across README, USAGE.md, and the docs site; none documented.", "evidence": "" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Self-hosted, MIT-licensed multi-agent framework you clone, configure with API keys, and run locally (CLI or web UI) — an opinionated orchestrator, not a library you import or a managed service.", "evidence": "https://github.com/QuantaAlpha/RepoMaster#readme" } } }, { "name": "AutoHarness", "github_id": "aiming-lab/AutoHarness", "url": "https://github.com/aiming-lab/AutoHarness", "slug": "autoharness", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#autoharness", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/autoharness/", "description": "Lightweight governance harness: wraps any LLM client in ~2 lines for automated harness engineering—6–14 step pipeline, YAML constitution, risk-pattern matching, session persistence with cost tracking, multi-agent profiles.", "category": "coding-harness-configs", "category_title": "Coding harness configs and SDKs", "stars": 368, "tier": "super simple", "tier_rank": 1, "axis": "super simple (2-line wrapper, YAML gov)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "none", "recovery_rank": 1, "license_signal": "open-source", "tags": [ "memory", "multi-agent", "provider-agnostic", "python" ], "example": { "label": "Full pipeline demo", "url": "https://github.com/aiming-lab/AutoHarness/blob/main/examples/full_pipeline_demo.py" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "Governance is enforced through YAML-configured risk-pattern matching and a denylist (path traversal, secret exposure, dangerous ops) at the Permission Check pipeline step, but nothing isolates execution itself (no container/VM) — it's a configurable restriction layer on top of host execution.", "evidence": "https://github.com/aiming-lab/AutoHarness#readme" }, "context_memory": { "rating": "basic", "rank": 2, "detail": "Provides \"multi-layer context management\" and token-budget/truncation to keep context under control within a run, plus session persistence for cost-tracking/audit logs, but no described cross-session semantic retrieval or intelligent compaction.", "evidence": "https://github.com/aiming-lab/AutoHarness#readme" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "Every tool call routes through a fixed Parse & Validate -> Risk Classify -> Permission Check -> Execute -> Output Sanitize -> Audit Log pipeline with pre-execution gating and audit traces, but the stages are built-in policy, not user-definable hook code.", "evidence": "https://github.com/aiming-lab/AutoHarness#readme" }, "prompt_optimization": { "rating": "unknown", "rank": 0, "detail": "Looked in the README for prompt tuning, rules files, or system-prompt customization; not mentioned — the project's documented scope is governance/validation, not prompt content.", "evidence": "" }, "build_vs_buy": { "tier": 1, "label": "build", "detail": "Explicitly \"wrap any LLM client (2 lines, instant governance)\" — a Python library you import and compose (AutoHarness.wrap(OpenAI())), not a runtime you deploy.", "evidence": "https://github.com/aiming-lab/AutoHarness#readme" } } }, { "name": "LoopTroop", "github_id": "looptroop-ai/LoopTroop", "url": "https://github.com/looptroop-ai/LoopTroop", "slug": "looptroop", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#looptroop", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/looptroop/", "description": "Config layer that chains LLM councils for planning, Ralph loops for iterative refinement, and OpenCode worktrees for shipping. The **harness** contribution is the council → loop → worktree pipeline; OpenCode underneath executes.", "category": "coding-harness-configs", "category_title": "Coding harness configs and SDKs", "stars": 123, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (config pipeline over OpenCode)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "retry", "recovery_rank": 2, "license_signal": "open-source", "tags": [ "typescript" ], "example": { "label": "Council → loop → worktree pipeline", "url": "https://github.com/looptroop-ai/LoopTroop#readme" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "none", "rank": 1, "detail": "The README states plainly that \"Git worktrees isolate your code changes, but they do not sandbox the command execution process itself. The agent runs with your local user privileges,\" and it runs OpenCode in dangerously-skip-permissions mode, explicitly recommending an external VM for real isolation.", "evidence": "https://github.com/looptroop-ai/LoopTroop#readme" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Deliberate context engineering per work unit: \"the agent only sees the specific active bead, its immediate file target, and the test file,\" and each retry \"discards the broken chat session entirely and retries the exact bead with a fresh context window (plus notes from previous failures)\" — scoped isolation plus carried-forward failure summaries.", "evidence": "https://github.com/looptroop-ai/LoopTroop#readme" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "Human approval checkpoints exist at major stage transitions (\"you actively review and sign off on planning specs, execution blueprints, and final pull request deliverables\"), but these are run-level gates between stages, not per-tool-call pre-execution blocking.", "evidence": "https://github.com/looptroop-ai/LoopTroop#readme" }, "prompt_optimization": { "rating": "none", "rank": 1, "detail": "No user-configurable system prompts or rules-file layering documented; the LLM Council's draft/vote/refine cycle operates on implementation blueprints (plan content), not on the orchestrator's own prompts, and context handling is described as fixed curation rather than editable instructions.", "evidence": "https://github.com/looptroop-ai/LoopTroop#readme" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "A self-hosted local GUI orchestrator (Node.js/npm) that runs OpenCode underneath and wires council-planning, Ralph-loop execution, and worktree-shipping into one opinionated pipeline you run locally — no managed service.", "evidence": "https://github.com/looptroop-ai/LoopTroop#readme" } } }, { "name": "pmstack", "github_id": "RyanAlberts/pmstack", "url": "https://github.com/RyanAlberts/pmstack", "slug": "pmstack", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#pmstack", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/pmstack/", "description": "Claude Code config for AI product managers: CLAUDE.md plus skills for competitive analysis, PRD-from-signal, metric frameworks, stakeholder briefs, and agent eval design. \"GStack for PMs.\"", "category": "coding-harness-configs", "category_title": "Coding harness configs and SDKs", "stars": 8, "tier": "super simple", "tier_rank": 1, "axis": "super simple (skills bundle, PM-focused)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "evals" ], "example": { "label": "PRD-from-signal skill", "url": "https://github.com/RyanAlberts/pmstack/blob/main/skills/prd-from-signal.md" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "none", "rank": 1, "detail": "A skills package that runs inside Claude Code/Claude.ai — stateless command dispatch where \"the skill is just markdown — Claude reads it as instructions\" — with no execution isolation layer of its own.", "evidence": "https://github.com/RyanAlberts/pmstack#readme" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Eval runs persist to disk across sessions for drift comparison — e.g. `eval-runs/code-review-eval-2026-05-06/summary.md` — and `/eval-drift` explicitly \"re-runs the suite weekly, diffs against last week, flags any regression\" using that file-based history.", "evidence": "https://github.com/RyanAlberts/pmstack#readme" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "The only gating mechanism is `/launch-readiness`, which aggregates evidence into a GO/NO-GO/CONDITIONAL verdict after work is produced — an output guardrail/validator, not a pre-execution blocking hook on individual tool calls.", "evidence": "https://github.com/RyanAlberts/pmstack#readme" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Each command is a markdown skill file containing instructions Claude reads on invocation (\"save and re-run; your changes apply immediately\") — first-class, user-editable instruction layering with no automatic tuning loop.", "evidence": "https://github.com/RyanAlberts/pmstack#readme" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "A config-driven skills/command package layered on Claude Code or Claude.ai Projects (markdown skills + slash commands you configure), billed against an existing Claude subscription rather than a separate managed service.", "evidence": "https://github.com/RyanAlberts/pmstack#readme" } } }, { "name": "OpenClaw", "github_id": "openclaw/openclaw", "url": "https://github.com/openclaw/openclaw", "slug": "openclaw", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#openclaw", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/openclaw/", "description": "Self-hosted, always-on personal agent (formerly Clawdbot/Moltbot): a gateway + event-loop runtime that treats messages, heartbeats, crons, and webhooks as one input queue, persists state to local files, and lives in your chat apps (WhatsApp, Telegram, Slack, Discord). 13,700+ community skills; the fastest-growing repo in GitHub history.", "category": "personal-agent-runtimes", "category_title": "Personal agent runtimes", "stars": 387220, "tier": "complex", "tier_rank": 4, "axis": "complex (always-on runtime, channels, skill ecosystem — product suite)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "typescript", "multi-agent" ], "example": { "label": "Agent runtime architecture", "url": "https://github.com/openclaw/openclaw/blob/main/docs/agent-runtime-architecture.md" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "A Docker-backed sandbox with tiered workspace mount access (none/ro/rw), blocked system/credential paths, and default-deny private-network access ships in the runtime but is opt-in via agents.defaults.sandbox.mode - default tool execution is on the host gateway.", "evidence": "https://docs.openclaw.ai/gateway/security" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Durable memory lives in on-disk MEMORY.md and dated memory/*.md files retrieved via a hybrid vector+keyword memory_search tool, with automatic pre-compaction reminders and an optional 'dreaming' consolidation pass into long-term memory.", "evidence": "https://docs.openclaw.ai/concepts/memory" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "The plugin hook before_tool_call can return block: true, rewritten params, or requireApproval to stop or alter a tool call before it runs, alongside observation-only hooks like after_tool_call.", "evidence": "https://docs.openclaw.ai/plugins/hooks" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "The system prompt is assembled from workspace bootstrap files (AGENTS.md, SOUL.md, TOOLS.md, etc.) with configurable per-file and total character budgets; composition is manual/config-driven with no automatic prompt tuning.", "evidence": "https://docs.openclaw.ai/concepts/system-prompt" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "OpenClaw installs as a self-hosted npm/daemon Gateway runtime you configure with workspace files, plugins, and sandbox settings; optional companion apps (Windows Hub, macOS, iOS/Android nodes) are additional surfaces, not a required hosted execution stack.", "evidence": "https://raw.githubusercontent.com/openclaw/openclaw/main/README.md" } } }, { "name": "Hermes", "github_id": "NousResearch/hermes-agent", "url": "https://github.com/NousResearch/hermes-agent", "slug": "hermes-agent", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#hermes-agent", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/hermes-agent/", "description": "Nous Research's self-improving agent: a learning loop turns experience into reusable skills, builds a persistent user model across sessions, and checkpoints state to disk with rollback; lean enough for a $5 VPS, driven from chat, and model-agnostic (Nous Portal, OpenRouter, OpenAI, or any endpoint).", "category": "personal-agent-runtimes", "category_title": "Personal agent runtimes", "stars": 234688, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (lean runtime, learning loop, disk-first memory)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "memory", "python", "provider-agnostic" ], "example": { "label": "Built-in skills", "url": "https://github.com/NousResearch/hermes-agent/tree/main/skills" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "Local backend runs tool commands directly on the host by default (approval prompts plus a dangerous-command blocklist), with container/VM isolation (Docker, Singularity, Modal, Daytona) only enforced when the user opts into those backends.", "evidence": "https://hermes-agent.nousresearch.com/docs/user-guide/security" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Persistent MEMORY.md and USER.md files on disk are re-injected into the system prompt every session, plus FTS5 full-text session search and Honcho-based dialectic user modeling for cross-session recall.", "evidence": "https://hermes-agent.nousresearch.com/docs/user-guide/features/memory" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "User-definable pre_tool_call shell/plugin hooks can return an action:block result to veto a tool call before it executes, alongside post_tool_call, pre_llm_call, and a dozen other lifecycle events.", "evidence": "https://hermes-agent.nousresearch.com/docs/user-guide/features/hooks" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Behavior is customized via user-editable MEMORY.md/USER.md instruction files and a skills system injected into the prompt, while automatic DSPy+GEPA prompt/skill evolution lives in a separate, unmerged experimental repo (hermes-agent-self-evolution) that generates PRs rather than being built into the core agent.", "evidence": "https://github.com/NousResearch/hermes-agent-self-evolution" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "The published artifact is a self-hostable, MIT-licensed agent runtime installed via curl script and configured through config.yaml/backends/skills/plugins that you run yourself (VPS, Docker, GPU box); Nous Portal's 'Hermes Cloud' is the managed hosted alternative.", "evidence": "https://github.com/NousResearch/hermes-agent/blob/main/README.md" } } }, { "name": "nanobot", "github_id": "HKUDS/nanobot", "url": "https://github.com/HKUDS/nanobot", "slug": "nanobot", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#nanobot", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/nanobot/", "description": "Ultra-lightweight, self-hosted personal agent framework: the **harness** is a Python daemon wiring tools, memory, and MCP into chat/webhook front ends (Telegram, Discord, web); minimal footprint alternative to heavier personal-runtime stacks.", "category": "personal-agent-runtimes", "category_title": "Personal agent runtimes", "stars": 47301, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (lightweight daemon, chat/MCP)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "unknown", "tags": [ "mcp", "memory", "local", "python" ], "example": { "label": "Project README", "url": "https://github.com/HKUDS/nanobot#readme" }, "deep_dive": null }, { "name": "CowAgent", "github_id": "zhayujie/CowAgent", "url": "https://github.com/zhayujie/CowAgent", "slug": "cowagent", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#cowagent", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/cowagent/", "description": "Self-hosted **harness** (formerly chatgpt-on-wechat) that plans tasks, runs tools/skills, and self-evolves via memory; multi-model, multi-channel (WeChat, Telegram, etc.), one-line install.", "category": "personal-agent-runtimes", "category_title": "Personal agent runtimes", "stars": 46642, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (multi-channel, self-evolving)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "unknown", "tags": [ "memory", "python" ], "example": { "label": "Project README", "url": "https://github.com/zhayujie/CowAgent#readme" }, "deep_dive": null }, { "name": "Khoj", "github_id": "khoj-ai/khoj", "url": "https://github.com/khoj-ai/khoj", "slug": "khoj", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#khoj", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/khoj/", "description": "Self-hostable \"AI second brain\": answers over your docs and the web, custom agents, scheduled automations, and multi-client reach (web, Obsidian, Emacs, WhatsApp). A personal-agent harness with retrieval at the core.", "category": "personal-agent-runtimes", "category_title": "Personal agent runtimes", "stars": 36674, "tier": "complex", "tier_rank": 4, "axis": "complex (server + clients — product suite)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "python" ], "example": { "label": "Feature tour", "url": "https://github.com/khoj-ai/khoj#readme" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "strong", "rank": 3, "detail": "Python code the assistant generates is run in an isolated sandbox, not the host process: either Cohere's Terrarium as a local Docker container, or E2B as a remote sandbox — \"both approaches isolate code execution rather than running it directly on the server.\"", "evidence": "https://docs.khoj.dev/features/code_execution" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Persistent, indexed document retrieval across sessions (PDFs, Markdown, Notion, Word, org-mode) backing semantic search over \"your docs,\" i.e. file/vector-backed memory that survives the process.", "evidence": "https://github.com/khoj-ai/khoj#readme" }, "lifecycle_hooks": { "rating": "none", "rank": 1, "detail": "The only related feature, Automations, is a scheduler that runs saved queries on a cron-like cadence and emails results — a trigger, not a pre/post interception point on tool calls.", "evidence": "https://docs.khoj.dev/category/features" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "\"You can use agents to setup custom system prompts with Khoj\" — server admins define reusable agent personas with their own system prompts that all users can select, with no automatic tuning.", "evidence": "https://docs.khoj.dev/category/features" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "\"Open-source, self-hostable. Always\" is the primary artifact the repo installs; a hosted Khoj Cloud app is offered as an optional convenience alternative rather than the required execution surface.", "evidence": "https://github.com/khoj-ai/khoj#readme" } } }, { "name": "Eliza", "github_id": "elizaOS/eliza", "url": "https://github.com/elizaOS/eliza", "slug": "eliza", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#eliza", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/eliza/", "description": "Open \"agentic operating system\" (elizaOS): persistent multi-agent runtime with character files, a plugin ecosystem, and social/platform integrations — the harness behind a large share of autonomous social agents.", "category": "personal-agent-runtimes", "category_title": "Personal agent runtimes", "stars": 19137, "tier": "complex", "tier_rank": 4, "axis": "complex (runtime + plugin ecosystem — product suite)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "memory", "multi-agent", "typescript" ], "example": { "label": "Agent quickstart", "url": "https://github.com/elizaOS/eliza#readme" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "none", "rank": 1, "detail": "Plugin actions (services/providers/evaluators) load and run in-process; the character-interface and plugin-management docs describe conditional loading by environment variable but no isolation, container, or restricted-filesystem mechanism for action/tool execution.", "evidence": "https://docs.elizaos.ai/agents/character-interface" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Memories (full text, embeddings, metadata, relationships) are persisted through an IDatabaseAdapter (SQLite/Postgres/Mongo/Supabase) with automatic embedding generation and semantic similarity search, so agents \"maintain continuous context across conversations\" beyond the live process.", "evidence": "https://docs.elizaos.ai/agents/memory-and-state" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "Plugins register actions, providers, services, and evaluators, where evaluators do \"post-response work\" — an after-the-fact hook, not a documented pre-execution blocking mechanism for actions.", "evidence": "https://raw.githubusercontent.com/elizaOS/eliza/HEAD/README.md" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Character files expose a `system` field to \"override the default system prompt to fine-tune agent behavior\" plus a `templates` object (messageTemplate, thoughtTemplate, actionTemplate, custom keys) for per-operation prompt customization — no automatic tuning loop.", "evidence": "https://docs.elizaos.ai/agents/character-interface" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "\"An open-source, local-first operating system\" you self-host and configure via plugins/character files is the primary artifact; \"Eliza Cloud is optional\" and \"never required — local-only is first-class,\" so the managed offering is a named alternative, not the base tier.", "evidence": "https://raw.githubusercontent.com/elizaOS/eliza/HEAD/README.md" } } }, { "name": "Agent Zero", "github_id": "agent0ai/agent-zero", "url": "https://github.com/agent0ai/agent-zero", "slug": "agent-zero", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#agent-zero", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/agent-zero/", "description": "Organic, prompt-defined personal agent framework: hierarchical sub-agents, persistent memory, browser and code tools, and self-modifying behavior; runs in Docker with a web UI.", "category": "personal-agent-runtimes", "category_title": "Personal agent runtimes", "stars": 18943, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (prompt-defined, Docker + web UI)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "resumable", "recovery_rank": 3, "license_signal": "unknown", "tags": [ "memory", "multi-agent", "browser", "sandbox", "python" ], "example": { "label": "Framework tour", "url": "https://github.com/agent0ai/agent-zero#readme" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "strong", "rank": 3, "detail": "The framework's default operating mode is a Dockerized Linux desktop — \"keep it running inside Docker or another isolated environment\" — with the container as the execution surface for shell/code tools and per-project workspace volumes, not host-level execution by default.", "evidence": "https://raw.githubusercontent.com/agent0ai/agent-zero/HEAD/README.md" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Projects keep \"files, instructions, secrets, memories, repositories, and model-preset choices isolated per project,\" and a Time Travel feature gives the agent's `/a0/usr` workspace \"snapshot history, diff inspection, travel, and revert\" — persistent, cross-session state.", "evidence": "https://raw.githubusercontent.com/agent0ai/agent-zero/HEAD/README.md" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "An Extension Framework provides directory-discovered hook points across the agent lifecycle (agent_init, before_main_llm_call, message_loop_start/end, system_prompt, etc.) to \"modify or enhance\" behavior, but the docs describe modification/observation, not a confirmed veto/block of a tool call before it executes.", "evidence": "https://www.agent-zero.ai/p/docs/extensions/" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Prompts are plain files under `prompts/` the operator can edit directly, plus a `system_prompt` extension hook point and support for \"custom prompts\" via plugins/MCP — first-class prompt customization with no automatic tuning loop documented.", "evidence": "https://raw.githubusercontent.com/agent0ai/agent-zero/HEAD/README.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Self-hosted, Docker-run framework (subordinate agents, projects, plugins/MCP/custom tools) you configure and operate yourself — no managed backend is offered or required.", "evidence": "https://raw.githubusercontent.com/agent0ai/agent-zero/HEAD/README.md" } } }, { "name": "OpenHarness (HKUDS)", "github_id": "HKUDS/OpenHarness", "url": "https://github.com/HKUDS/OpenHarness", "slug": "openharness", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#openharness", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/openharness/", "description": "Open agent harness with a built-in personal agent (\"Ohmo\") that runs across Feishu, Slack, Telegram, and Discord; core tool-use, skills, memory, multi-agent coordination with auto-compaction for multi-day sessions.", "category": "personal-agent-runtimes", "category_title": "Personal agent runtimes", "stars": 15492, "tier": "complex", "tier_rank": 4, "axis": "complex (personal agent + multi-channel — product suite)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "memory", "multi-agent" ], "example": { "label": "harness-eval skill", "url": "https://github.com/HKUDS/OpenHarness/blob/main/.claude/skills/harness-eval/SKILL.md" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "strong", "rank": 3, "detail": "Enforced restriction shipped in the harness itself: path-level allow/deny rules in `settings.json` (e.g. patterns like \"/etc/*\") and a `denied_commands` list that explicitly rejects patterns like \"DROP TABLE *\", on top of a Plan Mode that \"blocks all writes\" outright.", "evidence": "https://github.com/HKUDS/OpenHarness#readme" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "\"Auto-Compaction preserves task state and channel logs across context compression — agents can run multi-day sessions without manual compact/clear,\" plus a MEMORY.md persistent-memory subsystem and CLAUDE.md discovery/injection for durable knowledge across sessions.", "evidence": "https://github.com/HKUDS/OpenHarness#readme" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "PreToolUse/PostToolUse event hooks combine with Multi-Level Permission Modes — Default (\"ask before write/execute\") and Plan Mode (\"block all writes\") — plus a denied_commands list, giving genuine before-execution blocking alongside after-the-fact audit hooks.", "evidence": "https://github.com/HKUDS/OpenHarness#readme" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "\"CLAUDE.md Discovery & Injection\" is listed under the Context & Memory subsystem as a project-level system-prompt customization mechanism, with no automatic tuning loop described.", "evidence": "https://github.com/HKUDS/OpenHarness#readme" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Self-hosted Python framework installed via `pip install openharness-ai`, running the bundled Ohmo agent on your existing Claude Code/Codex subscription — an opinionated runtime you configure and run, not a managed service.", "evidence": "https://github.com/HKUDS/OpenHarness#readme" } } }, { "name": "AIlice", "github_id": "myshell-ai/AIlice", "url": "https://github.com/myshell-ai/AIlice", "slug": "ailice", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#ailice", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/ailice/", "description": "Fully autonomous general-purpose agent; one binary, Docker-ready, for when you want \"set goal and walk away\" without a framework.", "category": "personal-agent-runtimes", "category_title": "Personal agent runtimes", "stars": 1411, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (autonomous, one binary)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "none", "recovery_rank": 1, "license_signal": "open-source", "tags": [ "sandbox", "python" ], "example": { "label": "Task showcase", "url": "https://github.com/myshell-ai/AIlice#cool-things-we-can-do" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "Docker is offered as an optional \"sandbox run\" (`docker build -t ailice . && docker run ...`), but the primary, documented install path is native (`pip install -e .`) with the agent's shell/code tools running directly on the host — isolation is opt-in, not the default.", "evidence": "https://github.com/myshell-ai/AIlice#readme" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "A long-term memory module performs local embedding-model computation for semantic storage/retrieval (with a roadmap item to replace the vector database with a knowledge graph for better long-text/code comprehension), beyond plain chat-history logging.", "evidence": "https://github.com/myshell-ai/AIlice#readme" }, "lifecycle_hooks": { "rating": "none", "rank": 1, "detail": "The interpreter framework executes pattern-matched actions from LLM output immediately; no pre-execution blocking/veto point is documented, only a mid-run \"interrupt\" mode for the user to inject new prompts.", "evidence": "https://github.com/myshell-ai/AIlice#readme" }, "prompt_optimization": { "rating": "unknown", "rank": 0, "detail": "Looked for user-configurable system prompts or persona/rules files distinct from the internal \"highly dynamic prompt construction\" (a loop-avoidance mechanism, not a user-facing customization surface); none documented in the README.", "evidence": "" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "A standalone, self-hosted autonomous agent (pip install or Docker, configured via CLI flags like `--contextWindowRatio`), not a library you import into another application.", "evidence": "https://github.com/myshell-ai/AIlice#readme" } } }, { "name": "Talon", "github_id": "dylanneve1/talon", "url": "https://github.com/dylanneve1/talon", "slug": "talon", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#talon", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/talon/", "description": "Multi-platform personal agent living in Telegram, Discord, Teams, and the terminal. The **harness** is a pluggable-backend loop (Claude, Kilo, OpenCode, Codex, OpenAI Agents) with full MCP tool access and persistent background agents (Goals, Heartbeat, Dream); the chat apps are shells.", "category": "personal-agent-runtimes", "category_title": "Personal agent runtimes", "stars": 71, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (multi-platform, pluggable backends, MCP)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "mcp", "memory", "cli", "typescript" ], "example": { "label": "Multi-platform setup", "url": "https://github.com/dylanneve1/talon#readme" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "Config-level allowlist via disabledToolTags/disabledTools and an MCP-process boundary for external tool servers, but no container/VM isolation or restricted filesystem enforced by default.", "evidence": "https://raw.githubusercontent.com/dylanneve1/talon/HEAD/README.md" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Persistent cross-session memory via pluggable MemPalace (local vector search) or mem0 (durable semantic fact extraction) backends, plus a Dream agent that runs background memory consolidation and a storage layer that persists sessions/history across restarts.", "evidence": "https://raw.githubusercontent.com/dylanneve1/talon/HEAD/README.md" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "Plugins expose a handleAction() callback and the internal event bus is a pub-sub spine for task/turn lifecycle events, but this is post-event logging/subscription rather than a documented pre-tool-call blocking or rewriting hook.", "evidence": "https://github.com/dylanneve1/talon/tree/main/docs" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "System prompt is assembled at session start from editable templates in prompts/ and plugins can append content via getSystemPromptAddition(), but there is no automatic/self-tuning prompt optimization loop.", "evidence": "https://raw.githubusercontent.com/dylanneve1/talon/HEAD/README.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Self-hosted, config-driven TypeScript harness (pluggable model backends, memory backends, plugin system) that the user runs as a daemon/binary rather than a hosted service.", "evidence": "https://raw.githubusercontent.com/dylanneve1/talon/HEAD/README.md" } } }, { "name": "n8n", "github_id": "n8n-io/n8n", "url": "https://github.com/n8n-io/n8n", "slug": "n8n", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#n8n", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/n8n/", "description": "Fair-code workflow engine with 400+ nodes and native AI nodes; the self-hosted Zapier that actually does agents and LangChain.", "category": "frameworks", "category_title": "Frameworks", "stars": 202069, "tier": "complex", "tier_rank": 4, "axis": "complex (400+ nodes, workflow engine — product suite)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "durable", "recovery_rank": 4, "license_signal": "restricted (Fair-code)", "tags": [ "workflow", "local", "typescript" ], "example": { "label": "Agent vs chain workflow", "url": "https://github.com/n8n-io/n8n-docs/blob/main/docs/advanced-ai/examples/agent-chain-comparison.md" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "strong", "rank": 3, "detail": "The Code node runs JS in a vm2/isolated-vm sandbox with built-in and external module imports denied by default (NODE_FUNCTION_ALLOW_BUILTIN/NODE_FUNCTION_ALLOW_EXTERNAL), and Task Runners execute code in a separate process outside the main instance.", "evidence": "https://docs.n8n.io/deploy/host-n8n/configure-n8n/basic-configuration/configuration-examples/enable-modules-in-code-node" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "AI Agent nodes attach memory sub-nodes backed by persistent stores (Postgres Chat Memory, Redis Chat Memory, Zep, Motorhead) in addition to session-only Simple Memory, with a Chat Memory Manager node for advanced history trimming.", "evidence": "https://docs.n8n.io/build/integrate-ai/understand-ai-components/how-memory-works" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "Human-in-the-loop review attached to AI Agent tools pauses the workflow pre-execution for an Approve/Deny decision (Deny cancels the action) - a built-in approval gate, not a user-programmable tool-interception hook.", "evidence": "https://docs.n8n.io/build/integrate-ai/ai-examples/human-in-the-loop-for-tools" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "The Agent node exposes a manually-edited System Message field to guide agent behavior before conversation starts; there is no automatic prompt tuning or refinement loop.", "evidence": "https://docs.n8n.io/integrations/builtin/cluster-nodes/root-nodes/n8n-nodes-langchain.agent/tools-agent" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Primary artifact is the self-hostable fair-code workflow engine (Sustainable Use License) that users deploy and configure with nodes/credentials; n8n Cloud is a named hosted alternative, not the repo's installed artifact.", "evidence": "https://raw.githubusercontent.com/n8n-io/n8n/HEAD/README.md" } } }, { "name": "AutoGPT", "github_id": "Significant-Gravitas/AutoGPT", "url": "https://github.com/Significant-Gravitas/AutoGPT", "slug": "autogpt", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#autogpt", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/autogpt/", "description": "The original autonomous loop: goal in, agent iterates with tools and memory; Forge is the dev framework, Benchmark the eval harness.", "category": "frameworks", "category_title": "Frameworks", "stars": 186806, "tier": "complex", "tier_rank": 4, "axis": "complex (autonomous loop, tools, memory — product suite)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "resumable", "recovery_rank": 3, "license_signal": "restricted (Polyform-SU)", "tags": [ "memory", "evals", "python" ], "example": { "label": "Medium blogger graph", "url": "https://github.com/Significant-Gravitas/AutoGPT/blob/master/autogpt_platform/graph_templates/Medium%20Blogger_v28.json" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "strong", "rank": 3, "detail": "Classic AutoGPT's CodeExecutorConfiguration runs generated code inside a dedicated Docker sandbox container (docker_container_name: \"agent_sandbox\") and RESTRICT_TO_WORKSPACE (default True) confines file read/write to the agent workspace directory unless explicitly disabled.", "evidence": "https://docs.agpt.co/classic/configuration/options/" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Platform agents attach a Mem0 memory block whose stored content \"persist[s] across workflow executions\" and can be scoped per user/run/agent, with a companion retrieval block doing semantic search over stored memories.", "evidence": "https://agpt.co/docs/integrations/block-integrations/basic" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "The Platform dashboard surfaces \"every agent, run, cost, and action that needs your attention\" as post-hoc observability/logging; official docs pages describing a pre-execution approval-gate block could not be reached this session to confirm blocking behavior.", "evidence": "https://raw.githubusercontent.com/Significant-Gravitas/AutoGPT/HEAD/README.md" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Agents are authored either by describing \"the job in plain English\" or by wiring blocks (including LLM/prompt blocks) in the visual builder for exact per-step control; no automatic prompt-tuning loop is documented.", "evidence": "https://raw.githubusercontent.com/Significant-Gravitas/AutoGPT/HEAD/README.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "The repo installs the self-hostable AutoGPT Server/Frontend Platform (\"you provide the infrastructure and model API keys\"); the hosted AutoGPT Platform is offered as a named managed alternative for those who don't want to self-host.", "evidence": "https://raw.githubusercontent.com/Significant-Gravitas/AutoGPT/HEAD/README.md" } } }, { "name": "langflow", "github_id": "langflow-ai/langflow", "url": "https://github.com/langflow-ai/langflow", "slug": "langflow", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#langflow", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/langflow/", "description": "Low-code UI to build and deploy LangChain/LangGraph flows; visual DAG editor and one-click run.", "category": "frameworks", "category_title": "Frameworks", "stars": 153582, "tier": "complex", "tier_rank": 4, "axis": "complex (low-code, visual — product suite)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "retry", "recovery_rank": 2, "license_signal": "open-source", "tags": [ "low-code", "python" ], "example": { "label": "Chat with RAG flow", "url": "https://github.com/langflow-ai/langflow/blob/main/docs/docs/Tutorials/chat-with-rag.mdx" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "none", "rank": 1, "detail": "Custom Python components, the Python Interpreter node, and CSV Agent execute arbitrary Python directly in the host Langflow backend process with full filesystem/network access and no per-user isolation by design.", "evidence": "https://docs.langflow.org/security" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Memory bases are per-flow vector stores that auto-ingest conversation messages and persist context across sessions, on top of session-ID-scoped chat memory and optional external Mem0/Redis backends.", "evidence": "https://docs.langflow.org/memory" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "The beta Policies component wraps tools in a ToolGuard generated from natural-language business rules that \"runs before tool execution,\" blocking calls that violate policy, alongside a separate Guardrails component that validates input before it flows downstream.", "evidence": "https://docs.langflow.org/policies" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Prompt Template components feed a Language Model's System Message input and expose curly-brace template variables sourced from other components or global variables; there is no automatic prompt-tuning mechanism.", "evidence": "https://docs.langflow.org/components-prompts" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Repo installs the open-source, self-hostable (pip/Docker) visual flow builder; DataStax Langflow is a separate fully-managed cloud version of the same OSS foundation offered as a hosted alternative.", "evidence": "https://docs.datastax.com/en/langflow/index.html" } } }, { "name": "Dify", "github_id": "langgenius/dify", "url": "https://github.com/langgenius/dify", "slug": "dify", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#dify", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/dify/", "description": "One-stop LLM app platform: visual workflows, RAG pipeline, 50+ tools, model management; \"ship from prototype to prod\" in a single UI.", "category": "frameworks", "category_title": "Frameworks", "stars": 153269, "tier": "complex", "tier_rank": 4, "axis": "complex (one-stop platform — product suite)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "retry", "recovery_rank": 2, "license_signal": "restricted (Fair-code)", "tags": [ "low-code", "rag", "python" ], "example": { "label": "Customer-service bot", "url": "https://github.com/langgenius/dify-docs/blob/main/en/use-dify/tutorials/customer-service-bot.mdx" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "strong", "rank": 3, "detail": "DifySandbox executes Code/Template-Transform/Code-Interpreter node code behind a seccomp system-call whitelist, an isolated per-execution filesystem, and network-isolated proxy containers, rather than running user code directly on the host.", "evidence": "https://dify.ai/blog/difysandbox-goes-open-source-secure-execution-of-code" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Session-scoped Conversation Variables handle short-term context, while Knowledge Bases persist document content as vector embeddings (13+ supported vector DB backends) so the same processed knowledge is reused across applications and sessions.", "evidence": "https://docs.dify.ai/en/cloud/use-dify/knowledge/readme" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "The moderation extension exposes app.moderation.input (runs before the query reaches the app) and app.moderation.output hooks whose response can be direct_output (blocks the request with a preset reply) or overridden (rewrites the content) before/after the LLM call.", "evidence": "https://legacy-docs.dify.ai/guides/extension/api-based-extension/moderation" }, "prompt_optimization": { "rating": "native", "rank": 3, "detail": "The Prompt IDE ships a built-in Prompt Generator (INSTRUCTION_GENERATE_TEMPLATE_PROMPT / LLM_MODIFY_PROMPT_SYSTEM) that feeds the previous run's output back to an LLM to rewrite the prompt toward a stated ideal output, an in-harness feedback-driven refinement loop rather than only manual editing.", "evidence": "https://github.com/langgenius/dify/blob/main/api/core/llm_generator/prompts.py" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Repo installs the self-hostable Community Edition (Docker Compose); Dify Cloud is a named hosted alternative and Enterprise adds a separate license, but the primary published artifact is the self-run platform.", "evidence": "https://raw.githubusercontent.com/langgenius/dify/HEAD/README.md" } } }, { "name": "langchain", "github_id": "langchain-ai/langchain", "url": "https://github.com/langchain-ai/langchain", "slug": "langchain", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#langchain", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/langchain/", "description": "Chains, tools, retrievers, and agents; the usual entry point for \"add tools to an LLM\" in Python/JS.", "category": "frameworks", "category_title": "Frameworks", "stars": 144822, "tier": "complex", "tier_rank": 4, "axis": "complex (kitchen-sink ecosystem — product suite)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "retry", "recovery_rank": 2, "license_signal": "open-source", "tags": [ "python" ], "example": { "label": "Build an agent notebook", "url": "https://github.com/langchain-ai/langchain-academy/blob/main/module-1/agent.ipynb" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "Built-in tools like the Python REPL tool carry an explicit warning (\"can execute arbitrary code on the host machine... Use with caution\") with no sandbox by default; isolation is delegated to the separate opt-in langchain-sandbox (Pyodide/Deno) project.", "evidence": "https://docs.langchain.com/oss/python/integrations/tools/python" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Agent state is persisted via checkpointers (SqliteSaver, PostgresSaver, etc.) so a thread can be resumed across sessions, and the built-in SummarizationMiddleware compacts long message history through a chat model rather than just discarding it.", "evidence": "https://docs.langchain.com/oss/python/langchain/short-term-memory" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "The middleware system's wrap_tool_call hook wraps every tool call and can short-circuit it (zero handler invocations) by returning a substitute response instead of executing the tool, i.e. a user-definable pre-execution block; before_model/after_model hooks add further before/after interception.", "evidence": "https://docs.langchain.com/oss/python/langchain/middleware/custom" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "create_agent takes a system_prompt parameter and PromptTemplate/ChatPromptTemplate/SystemMessagePromptTemplate provide first-class templated prompt customization; the core framework has no automatic prompt-tuning loop of its own.", "evidence": "https://python.langchain.com/docs/concepts/prompt_templates/" }, "build_vs_buy": { "tier": 1, "label": "build", "detail": "Core LangChain is a Python/JS library of chains, tools, retrievers, and agent primitives that you compose and own the loop for; LangSmith/LangGraph Platform are separate paid deployment/observability products, not what pip install langchain gives you.", "evidence": "https://raw.githubusercontent.com/langchain-ai/langchain/HEAD/README.md" } } }, { "name": "browser-use", "github_id": "browser-use/browser-use", "url": "https://github.com/browser-use/browser-use", "slug": "browser-use", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#browser-use", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/browser-use/", "description": "Python web-agent **harness**: natural-language goals become browser actions, driven directly over the Chrome DevTools Protocol (it dropped Playwright in August 2025). The biggest community in the browser-agent category.", "category": "frameworks", "category_title": "Frameworks", "stars": 110219, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (LLM + browser, CDP)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "retry", "recovery_rank": 2, "license_signal": "open-source", "tags": [ "browser", "python" ], "example": { "label": "Grocery shopping agent", "url": "https://github.com/browser-use/browser-use/blob/main/examples/use-cases/shopping.py" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "The open-source library drives a real local (or user-supplied) browser directly with no isolation by default; the only enforced restriction is the configurable Browser(allowed_domains=[...]) navigation allowlist, which has had documented bypasses. True process isolation is a Browser Use Cloud (hosted) feature, not something the OSS library ships.", "evidence": "https://docs.browser-use.com/examples/templates/sensitive-data" }, "context_memory": { "rating": "basic", "rank": 2, "detail": "Within a run, agent.add_new_task() lets one long-lived session carry state/auth across chained tasks and save_conversation_path logs chat history to files, but there is no vector/semantic memory or intelligent compaction shipped in the OSS core; Browser Use Cloud separately advertises \"persistent filesystem and memory.\"", "evidence": "https://raw.githubusercontent.com/browser-use/browser-use/HEAD/README.md" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "on_step_start and on_step_end fire once per agent step (before the agent decides its next batch of actions / after it finishes executing them) giving read/modify access to agent state, but the docs state hooks cannot block or rewrite an individual action before it runs and recommend custom tools instead for that.", "evidence": "https://docs.browser-use.com/customize/hooks" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Agents support extend_system_message or override_system_message to customize the default system prompt; no automatic prompt-tuning loop is documented.", "evidence": "https://raw.githubusercontent.com/browser-use/browser-use/HEAD/README.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "The repo installs a self-hosted Python Agent class that already implements the ReAct-style browser loop (you configure task/llm/tools rather than write the loop); Browser Use Cloud is a named fully-hosted managed alternative.", "evidence": "https://raw.githubusercontent.com/browser-use/browser-use/HEAD/README.md" } } }, { "name": "llama-index", "github_id": "run-llama/llama_index", "url": "https://github.com/run-llama/llama_index", "slug": "llama_index", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#llama_index", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/llama_index/", "description": "Data-centric: indexing, RAG, and query engines; agent abstractions sit on top of your data pipelines.", "category": "frameworks", "category_title": "Frameworks", "stars": 51815, "tier": "complex", "tier_rank": 4, "axis": "complex (RAG + agents — product suite)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "retry", "recovery_rank": 2, "license_signal": "open-source", "tags": [ "rag", "python" ], "example": { "label": "Research assistant workflow", "url": "https://github.com/run-llama/llama_index/blob/main/docs/examples/agent/agent_workflow_research_assistant.ipynb" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "The built-in code-interpreter tool gives the agent direct subprocess.run access on the host (\"not recommended to be used in a production setting, and would require heavy sandboxing\"); real isolation is only available by opting into an external service like Azure Container Apps Dynamic Sessions.", "evidence": "https://pypi.org/project/llama-index-tools-code-interpreter/" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "The Memory class combines a short-term token-budgeted buffer with pluggable long-term MemoryBlocks (FactExtractionMemoryBlock, VectorMemoryBlock) and is backed by SQLite by default (swappable for Postgres), persisting chat/fact/vector history across restarts rather than only within the process.", "evidence": "https://developers.llamaindex.ai/python/framework/module_guides/deploying/agents/memory/" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "The CallbackManager fires handlers on event start/end and maintains a trace stack for debugging/tracing (e.g. LlamaDebugHandler, Wandb/OpenAI fine-tuning handlers); this is observability instrumentation, not a documented mechanism to veto or rewrite a tool call before it runs.", "evidence": "https://docs.llamaindex.ai/en/stable/module_guides/observability/callbacks/root/" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "ReActAgent/FunctionAgent accept a system_prompt constructor argument appended to the core agent prompt, and prompts can be swapped post-construction via agent.update_prompts(); no automatic prompt-tuning loop is documented.", "evidence": "https://developers.llamaindex.ai/python/framework/module_guides/deploying/agents/" }, "build_vs_buy": { "tier": 1, "label": "build", "detail": "llama-index-core is a Python library of indices, retrievers, and agent primitives you import and compose to write your own RAG/agent pipeline; LlamaCloud/LlamaParse are separate managed products for parsing/indexing, not the installed artifact itself.", "evidence": "https://raw.githubusercontent.com/run-llama/llama_index/HEAD/README.md" } } }, { "name": "agno", "github_id": "agno-agi/agno", "url": "https://github.com/agno-agi/agno", "slug": "agno", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#agno", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/agno/", "description": "Python agents with memory, knowledge bases, tools, and structured outputs; continues the PhiData-era product line under the Agno name—production apps, evals, and pipelines.", "category": "frameworks", "category_title": "Frameworks", "stars": 41848, "tier": "complex", "tier_rank": 4, "axis": "complex (memory, KB, observability — product suite)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "memory", "evals", "python" ], "example": { "label": "Agent with tools", "url": "https://github.com/agno-agi/agno/blob/main/cookbook/02_agents/01_quickstart/agent_with_tools.py" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "PythonTools can run/save-and-run Python and pip-install packages; the only restriction mechanism is a configurable include_tools/exclude_tools allowlist for \"safe mode\" (all functions available unrestricted by default), with no container/VM isolation documented for the built-in tool.", "evidence": "https://docs.agno.com/examples/tools/python-tools" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Adding a storage driver (Sqlite/Postgres/Mongo/etc.) persists agent sessions and user memories to a database keyed by session ID so state survives past the current execution cycle, on top of separate session-scoped chat history.", "evidence": "https://docs-v1.agno.com/agents/memory" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "tool_hooks receive the wrapped function call pre-execution and can raise or return a substitute result to block/replace it (pre_hook/post_hook on @tool), in addition to requires_confirmation human-in-the-loop gates.", "evidence": "https://docs.agno.com/tools/hooks" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Agents take an instructions list plus description/additional_context/expected_output parameters that assemble the system message, and instructions can be updated dynamically at runtime; no automatic prompt-tuning loop is documented.", "evidence": "https://docs.agno.com/context/agent/overview" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "The repo installs AgentOS, a self-hosted FastAPI runtime you deploy in your own cloud/data center with your own database; the os.agno.com Control Plane is only a browser client against your own instance, not a hosted execution backend.", "evidence": "https://www.agno.com/agentos" } } }, { "name": "langgraph", "github_id": "langchain-ai/langgraph", "url": "https://github.com/langchain-ai/langgraph", "slug": "langgraph", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#langgraph", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/langgraph/", "description": "State-machine graphs over LLM steps; checkpointing, human-in-the-loop, and durable execution so workflows survive restarts.", "category": "frameworks", "category_title": "Frameworks", "stars": 40276, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (graphs, checkpointing, durable exec)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "durable", "recovery_rank": 4, "license_signal": "open-source", "tags": [ "workflow", "python" ], "example": { "label": "Customer support agent", "url": "https://github.com/langchain-ai/langgraph/blob/main/examples/customer-support/customer-support.ipynb" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "Core langgraph's ToolNode executes tool calls directly in the host process with no built-in isolation; sandboxed code execution is delegated to the separate (now-archived) langchain-sandbox package or third-party platforms like Modal/E2B.", "evidence": "https://github.com/langchain-ai/langchain-sandbox" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "A checkpointer (PostgresSaver/SqliteSaver) persists full graph-state snapshots per thread_id across process restarts and enables resume-from-checkpoint, plus a separate long-term cross-thread store distinct from thread-scoped checkpoints.", "evidence": "https://docs.langchain.com/oss/python/langgraph/persistence" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "The interrupt() function can be called inside any node or tool to pause execution before it completes, letting a human approve, edit (override the input via Command(resume=...)), or cancel the pending action.", "evidence": "https://docs.langchain.com/oss/python/langgraph/interrupts" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "System prompts are assembled per-run through a @dynamic_prompt decorator that reads typed Runtime context (e.g. user_id) — entirely developer-written logic — with no automatic prompt tuning/optimization loop shipped in core langgraph.", "evidence": "https://docs.langchain.com/oss/python/langchain/runtime" }, "build_vs_buy": { "tier": 1, "label": "build", "detail": "langgraph is a pip/npm-installed low-level graph-orchestration library (StateGraph/Pregel API) you import and run in your own process with no required runtime; LangGraph Platform is the paid managed/self-hosted deployment tier for the same graphs.", "evidence": "https://raw.githubusercontent.com/langchain-ai/langgraph/main/README.md" } } }, { "name": "semantic-kernel", "github_id": "microsoft/semantic-kernel", "url": "https://github.com/microsoft/semantic-kernel", "slug": "semantic-kernel", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#semantic-kernel", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/semantic-kernel/", "description": "Microsoft's plugin and planner layer for LLMs; C#, Python, Java; strong on enterprise auth and orchestration.", "category": "frameworks", "category_title": "Frameworks", "stars": 28481, "tier": "complex", "tier_rank": 4, "axis": "complex (enterprise, multi-language — product suite)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "retry", "recovery_rank": 2, "license_signal": "open-source", "tags": [ "python" ], "example": { "label": "Chat completion agent", "url": "https://github.com/microsoft/semantic-kernel/blob/main/python/samples/getting_started_with_agents/chat_completion/step01_chat_completion_agent_simple.py" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "Native plugin functions execute in-process with no built-in container/VM isolation; Code Interpreter's sandboxing \"depends on the underlying agent provider\" (delegated to e.g. Azure/OpenAI's hosted sandbox) and Microsoft's own guidance says the LLM is not a security boundary and isolated execution must be architected by the developer.", "evidence": "https://github.com/MicrosoftDocs/semantic-kernel-docs/blob/main/agent-framework/agents/tools/code-interpreter.md" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Vector DB connectors (Azure AI Search, Elasticsearch, Chroma, etc.) provide semantic memory, and AgentThread/ChatHistoryAgentThread objects can be serialized (thread.Serialize()/DeserializeThreadAsync) to persist and resume conversation state across sessions rather than only in-process.", "evidence": "https://learn.microsoft.com/en-us/python/api/semantic-kernel/semantic_kernel.agents.chat_completion.chat_completion_agent.chathistoryagentthread?view=semantic-kernel-python" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "The Function Invocation Filter and Prompt Render Filter wrap every function call/prompt render with a next() delegate; not calling next() blocks the function or prevents prompt submission to the AI entirely, and the same filter can override the result both before and after execution.", "evidence": "https://learn.microsoft.com/en-us/semantic-kernel/concepts/enterprise-readiness/filters" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Agents are constructed with an \"instructions\" string (e.g. \"You are a helpful assistant\") for direct system-prompt control, and Prompt Render Filters can further rewrite the rendered prompt before it is sent; no automatic prompt-tuning loop is documented.", "evidence": "https://raw.githubusercontent.com/microsoft/semantic-kernel/HEAD/README.md" }, "build_vs_buy": { "tier": 1, "label": "build", "detail": "Semantic Kernel is a self-hosted C#/Python/Java SDK of kernels, plugins, and filters that developers compose into their own orchestration/planning code; there is no managed hosted execution product bundled in the repo.", "evidence": "https://raw.githubusercontent.com/microsoft/semantic-kernel/HEAD/README.md" } } }, { "name": "mastra", "github_id": "mastra-ai/mastra", "url": "https://github.com/mastra-ai/mastra", "slug": "mastra", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#mastra", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/mastra/", "description": "TypeScript-first; agents, tools, and workflows with a single runtime and minimal boilerplate.", "category": "frameworks", "category_title": "Frameworks", "stars": 27375, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (TS-first, minimal boilerplate)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "durable", "recovery_rank": 4, "license_signal": "restricted (Elastic-2.0)", "tags": [ "typed", "typescript" ], "example": { "label": "Durable research agent", "url": "https://github.com/mastra-ai/mastra/tree/main/examples/durable-agents" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "strong", "rank": 3, "detail": "Mastra ships a first-party WorkspaceSandbox abstraction with importable isolated providers (E2BSandbox, DaytonaSandbox, BlaxelSandbox cloud sandboxes; AppleContainerSandbox OCI containers; RailwaySandbox VMs) alongside LocalSandbox, so agents can be given \"an isolated computer to work in\" rather than only host execution.", "evidence": "https://mastra.ai/docs/workspace/sandbox" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "The Memory system persists conversation threads, messages, and working memory through configured storage providers plus vector-search-backed semantic recall of older messages, surviving application restarts rather than only the live request.", "evidence": "https://mastra.ai/docs/memory/overview" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "A workflow step calling suspend() saves the current execution state as a snapshot and halts; nothing past that step runs until resume() is explicitly called (e.g. after human approval), giving a genuine pre-execution block on the next action.", "evidence": "https://mastra.ai/docs/workflows/suspend-and-resume" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Agents take an instructions string, per-call system prompts can supplement it via generate(), and reusable prompt blocks with {{variable}} templating compose the system prompt; no automatic prompt-tuning loop is documented.", "evidence": "https://mastra.ai/docs/agents/overview" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Repo installs the self-hostable TypeScript agent/workflow runtime (deployable as a standalone server or embedded in Next.js/Node); Mastra Cloud/Studio are separate hosted tooling layered on top, not required to run the framework.", "evidence": "https://raw.githubusercontent.com/mastra-ai/mastra/HEAD/README.md" } } }, { "name": "Haystack", "github_id": "deepset-ai/haystack", "url": "https://github.com/deepset-ai/haystack", "slug": "haystack", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#haystack", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/haystack/", "description": "Open-source orchestration framework for context-engineered LLM apps: modular pipelines and agent workflows with explicit control over retrieval, routing, memory, and generation—closer to LangChain's territory than a coding-agent harness.", "category": "frameworks", "category_title": "Frameworks", "stars": 26293, "tier": "complex", "tier_rank": 4, "axis": "complex (modular pipelines, RAG + agents — product suite)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "memory", "rag", "python" ], "example": { "label": "Project README", "url": "https://github.com/deepset-ai/haystack#readme" }, "deep_dive": null }, { "name": "letta", "github_id": "letta-ai/letta", "url": "https://github.com/letta-ai/letta", "slug": "letta", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#letta", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/letta/", "description": "Python agent runtime with tool use and control flow; lean API; stateful agents with long-horizon memory.", "category": "frameworks", "category_title": "Frameworks", "stars": 24360, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (lean API)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "durable", "recovery_rank": 4, "license_signal": "open-source", "tags": [ "memory", "python" ], "example": { "label": "Loop .af agent file", "url": "https://github.com/letta-ai/agent-file/tree/main/agents/%40letta-ai/loop" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "Tool execution runs unsandboxed by default; secure sandboxed execution requires opting into the third-party E2B service via an E2B_API_KEY rather than being enforced by the harness itself.", "evidence": "https://docs.letta.com/guides/agents/tool-execution-sandbox/" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "An OS-inspired memory model pins editable, persistent 'memory blocks' (core memory) in the context window across all sessions and offloads unlimited facts to a semantically-searchable archival memory store queried on demand.", "evidence": "https://docs.letta.com/guides/core-concepts/memory/memory-blocks" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "Tools flagged default_requires_approval pause execution for an explicit approve/deny before the tool runs - a built-in human-approval gate rather than a programmable pre-execution hook (the sibling Letta Code CLI has a richer PreToolUse/PostToolUse hook system, out of scope for this repo).", "evidence": "https://docs.letta.com/guides/agents/human-in-the-loop/" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "The Agent Development Environment exposes a manually-edited 'System Instructions' panel plus editable persona/human memory blocks; no automatic prompt tuning or eval-driven optimization loop is documented for the core agent server.", "evidence": "https://docs.letta.com/guides/ade/settings/" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "The published artifact is an open-source, self-hostable stateful-agent server you deploy via Docker and configure through the ADE (memory blocks, tools, personas); Letta's 'Constellation' is the managed hosted agent-cloud alternative.", "evidence": "https://docs.letta.com/guides/selfhosting" } } }, { "name": "Stagehand", "github_id": "browserbase/stagehand", "url": "https://github.com/browserbase/stagehand", "slug": "stagehand", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#stagehand", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/stagehand/", "description": "Browserbase's SDK for browser agents: natural-language actions (act, extract, observe) and deterministic Playwright code mix in one script, so agent flexibility and repeatable automation live in the same **harness**.", "category": "frameworks", "category_title": "Frameworks", "stars": 24023, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (NL + code browser SDK)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "none", "recovery_rank": 1, "license_signal": "open-source", "tags": [ "browser", "typescript" ], "example": { "label": "Act/extract/observe quickstart", "url": "https://github.com/browserbase/stagehand#readme" }, "deep_dive": null }, { "name": "rasa", "github_id": "RasaHQ/rasa", "url": "https://github.com/RasaHQ/rasa", "slug": "rasa", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#rasa", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/rasa/", "description": "Conversational AI stack (NLU, dialogue, actions); long-standing OSS choice for chat and voice bots.", "category": "frameworks", "category_title": "Frameworks", "stars": 21302, "tier": "complex", "tier_rank": 4, "axis": "complex (full stack — product suite)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "voice", "python" ], "example": { "label": "Sara conversational demo", "url": "https://github.com/RasaHQ/rasa-demo" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "Custom actions execute on a separate action server over an HTTP /webhook contract - architectural process separation (often deployed as its own container) rather than enforced isolation of what the action code can do.", "evidence": "https://rasa.com/docs/action-server/" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Tracker stores persist each conversation's full event/slot history outside the process via SQLTrackerStore, RedisTrackerStore, MongoTrackerStore, or DynamoTrackerStore, maintaining context across sessions rather than defaulting to in-memory-only state.", "evidence": "https://rasa.com/docs/reference/integrations/tracker-stores/" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "CALM validates LLM-generated commands against allowed flows before execution via the deterministic Flow Policy state machine, blocking unauthorized actions - structural policy enforcement, not a registerable pre/post hook interface.", "evidence": "https://rasa.com/docs/learn/concepts/calm/" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Business logic and conversational structure are authored as user-defined Flows (with built-in reusable \"patterns\" for common repair situations); this is rules/flow-file configuration rather than an automatic prompt-tuning loop.", "evidence": "https://rasa.com/docs/reference/primitives/flows/" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "The OSS repo (now in maintenance mode) installs a self-hosted dialogue-management framework; forward-looking CALM development lives in Rasa Pro, a separate license tier, with the Rasa Platform as a named hosted/managed option for scaling.", "evidence": "https://raw.githubusercontent.com/RasaHQ/rasa/HEAD/README.md" } } }, { "name": "Google ADK", "github_id": "google/adk-python", "url": "https://github.com/google/adk-python", "slug": "adk-python", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#adk-python", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/adk-python/", "description": "Google's official Agent Development Kit: code-first Python toolkit for building, evaluating, and deploying agents. Optimized for Gemini but model-agnostic; deploys to Cloud Run / Vertex AI; ships a dev UI with eval and a code-execution sandbox.", "category": "frameworks", "category_title": "Frameworks", "stars": 21234, "tier": "complex", "tier_rank": 4, "axis": "complex (official Google SDK, eval, deploy — product suite)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "evals", "sandbox", "python" ], "example": { "label": "Travel concierge agent", "url": "https://github.com/google/adk-samples/tree/main/python/agents/travel-concierge" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "strong", "rank": 3, "detail": "Ships a GkeCodeExecutor that runs LLM-generated code in gVisor-sandboxed, non-root, read-only ephemeral pods (or Agent Sandbox instances) on GKE, plus a BuiltInCodeExecutor sandbox with no package installs and no network access.", "evidence": "https://raw.githubusercontent.com/google/adk-docs/main/docs/integrations/gke-code-executor.md" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "VertexAiMemoryBankService provides persistent, semantically-searchable memory across sessions distinct from short-term session state, retrieved on demand via load_memory/preload_memory/search_memory.", "evidence": "https://adk.dev/sessions/memory/" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "before_tool_callback (and before_agent_callback/before_model_callback) can return a replacement value that skips the actual tool/model call entirely, with matching after_* callbacks for post-execution handling.", "evidence": "https://adk.dev/callbacks/" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Agents take a first-class `instruction` (plus global_instruction) parameter for per-agent system prompt customization; no automatic prompt-tuning loop is documented.", "evidence": "https://raw.githubusercontent.com/google/adk-python/main/README.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Primary artifact is the open-source, code-first Python/multi-language agent framework (Agent/Runner/workflow-agent classes) you self-host via `adk run`/`adk web`; Vertex AI Agent Engine exists as a separate fully-managed deployment target for the same agents.", "evidence": "https://raw.githubusercontent.com/google/adk-python/main/README.md" } } }, { "name": "botpress", "github_id": "botpress/botpress", "url": "https://github.com/botpress/botpress", "slug": "botpress", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#botpress", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/botpress/", "description": "Visual bot builder and runtime; multi-channel, open-source alternative to commercial bot platforms.", "category": "frameworks", "category_title": "Frameworks", "stars": 14876, "tier": "complex", "tier_rank": 4, "axis": "complex (visual builder, multi-channel — product suite)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "low-code", "typescript" ], "example": { "label": "Inter-bot delegation", "url": "https://github.com/botpress/v12/tree/master/examples/interbot" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "unknown", "rank": 0, "detail": "Looked for how Execute Code cards/hooks/actions actually run (sandboxed container/VM vs. same process as bot server); the Studio 'use-code' guide documents the API surface but discloses no isolation mechanism.", "evidence": "" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Variables have a persistent 'User' scope that follows a user across all their conversations (distinct from single-conversation/session scopes), and Autonomous Nodes retrieve from Knowledge Bases on demand.", "evidence": "https://botpress.com/docs/studio/concepts/variables/overview/" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "Before/After Incoming and Before Outgoing hooks run custom JS that can inspect and modify event data (e.g. translate a message before further processing), but the docs describe them as event callbacks at fixed pipeline stages, not an enforced pre-tool-call veto mechanism.", "evidence": "https://botpress.com/docs/studio/concepts/hooks" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "The Autonomous Node's Instructions field is a first-class, freely-editable system prompt (plus a separate Personality agent), with no automatic prompt-tuning loop documented.", "evidence": "https://botpress.com/docs/studio/concepts/nodes/autonomous-node" }, "build_vs_buy": { "tier": 3, "label": "managed", "detail": "Botpress v12/self-hosted has been officially sunset ('no longer available for purchase, download, or new deployments'); the current botpress/botpress repo is CLI/SDK tooling for building integrations and bots that run on the managed Botpress Cloud platform.", "evidence": "https://botpress.com/docs/studio/guides/advanced/v12" } } }, { "name": "R2R", "github_id": "SciPhi-AI/R2R", "url": "https://github.com/SciPhi-AI/R2R", "slug": "r2r", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#r2r", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/r2r/", "description": "RAG-first: hybrid search, knowledge graphs, multimodal; the framework for \"production RAG\" when you care more about retrieval than chat UI.", "category": "frameworks", "category_title": "Frameworks", "stars": 7973, "tier": "complex", "tier_rank": 4, "axis": "complex (production RAG — product suite)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "retry", "recovery_rank": 2, "license_signal": "open-source", "tags": [ "vision", "rag", "workflow", "python" ], "example": { "label": "hello_r2r RAG example", "url": "https://github.com/SciPhi-AI/R2R/blob/main/py/core/examples/hello_r2r.py" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "unknown", "rank": 0, "detail": "The research agent ships a python_executor tool (alongside rag/reasoning/critique) per r2r.toml, but no source file or doc describing its isolation mechanism (container/subprocess/none) could be located; the r2r-docs.sciphi.ai docs site returns 404 site-wide.", "evidence": "" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Core function is persistent vector/knowledge-graph retrieval over ingested documents (search_file_knowledge tool) plus a Conversations API that threads and retrieves multi-turn chat by conversation_id, backed by the Postgres service in its Docker compose stack.", "evidence": "https://github.com/SciPhi-AI/R2R/blob/main/py/r2r/r2r.toml" }, "lifecycle_hooks": { "rating": "unknown", "rank": 0, "detail": "Looked for pre-ingestion/pre-request middleware or before/after tool-call hooks; found none in the r2r.toml config, README, or search results, and the docs site (r2r-docs.sciphi.ai) is entirely down (404 on every page including root).", "evidence": "" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "r2r.toml exposes named, swappable system-prompt templates per agent (rag_agent_static_prompt/rag_agent_dynamic_prompt) with no automatic tuning loop found.", "evidence": "https://github.com/SciPhi-AI/R2R/blob/main/py/r2r/r2r.toml" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Primary published artifact is the self-hosted, Docker/pip-installed RAG framework configured via r2r.toml rather than a hand-rolled loop; SciPhi Cloud exists as a separate managed serverless deployment of the same system with a free tier and enterprise plans.", "evidence": "https://raw.githubusercontent.com/SciPhi-AI/R2R/main/py/README.md" } } }, { "name": "agent-squad", "github_id": "2FastLabs/agent-squad", "url": "https://github.com/2FastLabs/agent-squad", "slug": "agent-squad", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#agent-squad", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/agent-squad/", "description": "AWS-originated orchestrator (now under 2FastLabs): intent classification, streaming, SupervisorAgent; \"agent-as-tools\" so one agent delegates to a squad.", "category": "frameworks", "category_title": "Frameworks", "stars": 7742, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (squad orchestration)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "multi-agent" ], "example": { "label": "E-commerce support sim", "url": "https://github.com/2FastLabs/agent-squad/tree/main/examples/ecommerce-support-simulator" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "unknown", "rank": 0, "detail": "README and docs describe tool integration (native functions, HTTP tools, MCP servers) but never address execution isolation, sandboxing, or containment for tool calls.", "evidence": "" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "ChatStorage is a pluggable interface with InMemoryChatStorage for dev and DynamoDbChatStorage/SqlChatStorage for production, persisting and fetching conversation history across sessions via save_chat_message()/fetch_chat().", "evidence": "https://github.com/2FastLabs/agent-squad" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "Agents accept an optional AgentCallbacks object for lifecycle events like new tokens in streaming responses; the docs describe it as event/observation handling (streaming), not a pre-execution veto mechanism.", "evidence": "https://2fastlabs.github.io/agent-squad/agents/overview/" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Agent descriptions and the classifier's system prompt are user-editable to fix misrouting, and classifiers can be fully replaced by extending the abstract Classifier class, but no automatic tuning loop is documented.", "evidence": "https://github.com/2FastLabs/agent-squad/blob/main/docs/src/content/docs/classifiers/overview.mdx" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Open-source (Apache 2.0), self-hosted orchestration framework (Python/TypeScript/Swift) you run yourself on Lambda, containers, or on-device, configuring classifiers/agents/storage rather than a hosted control plane.", "evidence": "https://raw.githubusercontent.com/2FastLabs/agent-squad/main/README.md" } } }, { "name": "AgentVerse", "github_id": "OpenBMB/AgentVerse", "url": "https://github.com/OpenBMB/AgentVerse", "slug": "agentverse", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#agentverse", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/agentverse/", "description": "Task-solving and simulation envs for multi-LLM agents; deploy many agents in custom environments without building infra from scratch.", "category": "frameworks", "category_title": "Frameworks", "stars": 5113, "tier": "complex", "tier_rank": 4, "axis": "complex (simulation envs, multi-agent — product suite)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "none", "recovery_rank": 1, "license_signal": "open-source", "tags": [ "multi-agent", "python" ], "example": { "label": "NLP classroom sim", "url": "https://github.com/OpenBMB/AgentVerse/blob/main/agentverse/tasks/simulation/nlp_classroom_9players/config.yaml" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "unknown", "rank": 0, "detail": "Tool execution (web browser, Jupyter notebook, bing search) is delegated to the external XAgent ToolServer; no isolation/containment mechanism shipped by AgentVerse itself is documented.", "evidence": "" }, "context_memory": { "rating": "basic", "rank": 2, "detail": "Agent `memory_type: chat_history` and the Updater rule component manage dialogue state only within one active simulation/task run; no persistence across sessions or compaction is documented.", "evidence": "https://raw.githubusercontent.com/OpenBMB/AgentVerse/main/README.md" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "The environment's Selector rule component filters/discards invalid agent-generated messages after they're produced (an after-the-fact output validator), not a pre-execution blocking hook; Order/Visibility/Updater govern turn-taking and message routing rather than intercepting tool calls.", "evidence": "https://github.com/OpenBMB/AgentVerse/blob/main/README.md" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Agent prompt_template fields with ${placeholder} substitution are set per-agent in config.yaml and filled at runtime via _fill_prompt_template; no automatic prompt tuning loop is documented.", "evidence": "https://raw.githubusercontent.com/OpenBMB/AgentVerse/main/README.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Self-hosted, config.yaml-driven framework for building task-solving or simulation multi-agent environments (describer/order/selector/updater/visibility rules) rather than raw API orchestration; no managed hosted offering beyond a HuggingFace Spaces demo.", "evidence": "https://raw.githubusercontent.com/OpenBMB/AgentVerse/main/README.md" } } }, { "name": "youtu-agent", "github_id": "TencentCloudADP/youtu-agent", "url": "https://github.com/TencentCloudADP/youtu-agent", "slug": "youtu-agent", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#youtu-agent", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/youtu-agent/", "description": "Tencent Cloud's agent framework: a minimal tool-calling **harness** designed to perform well with open-source models, positioned as a lighter alternative to heavier orchestration frameworks.", "category": "frameworks", "category_title": "Frameworks", "stars": 4601, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (minimal loop, open-model focus)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "retry", "recovery_rank": 2, "license_signal": "unknown", "tags": [], "example": { "label": "Project README", "url": "https://github.com/TencentCloudADP/youtu-agent#readme" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "strong", "rank": 3, "detail": "Ships BrowserE2BEnv/E2BEnv (cloud sandboxed execution) and a Docker-isolated BrowserEnv as first-class Environment options for production, distinct from the host-executing ShellLocalEnv which the docs explicitly flag as 'local development only.'", "evidence": "https://tencentcloudadp.github.io/youtu-agent/env/" }, "context_memory": { "rating": "basic", "rank": 2, "detail": "ContextManager is described as a configurable module for managing the agent's context window (token budgeting within a run); no documentation found of persistence across separate sessions.", "evidence": "https://raw.githubusercontent.com/TencentCloudADP/youtu-agent/main/README.md" }, "lifecycle_hooks": { "rating": "unknown", "rank": 0, "detail": "Looked for before/after tool-call hooks beyond the mentioned (future-release) DBTracingProcessor for trajectory analysis; no evidence of a blocking pre-execution hook surface was found in the README or reachable docs pages.", "evidence": "" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Agent instructions/system prompts are set via YAML config (agent.instructions) or overridden directly in the SimpleAgent constructor; no automatic prompt-tuning loop is documented.", "evidence": "https://tencentcloudadp.github.io/youtu-agent/agents/" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Self-hosted, YAML-config-driven agent framework (Environment/ContextManager/Benchmark components) deployable from source or Docker; Tencent Cloud's Agent Development Platform (ADP) is a separate enterprise offering named alongside it.", "evidence": "https://raw.githubusercontent.com/TencentCloudADP/youtu-agent/main/README.md" } } }, { "name": "Bee Agent Framework", "github_id": "i-am-bee/beeai-framework", "url": "https://github.com/i-am-bee/beeai-framework", "slug": "beeai-framework", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#beeai-framework", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/beeai-framework/", "description": "Python + TypeScript, LF AI–backed; MCP/ACP, workflows, Requirement Agent; the one that pushes \"production multi-agent\" without LangChain.", "category": "frameworks", "category_title": "Frameworks", "stars": 3383, "tier": "complex", "tier_rank": 4, "axis": "complex (production multi-agent — product suite)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "mcp", "multi-agent", "python", "typescript" ], "example": { "label": "ReAct agent example", "url": "https://github.com/i-am-bee/beeai-framework/blob/main/python/examples/agents/react.py" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "The SandboxTool for code execution requires the separate, optional beeai-code-interpreter service (its own Docker containers); isolation is delegated to that external companion project rather than enforced by beeai-framework's own execution path.", "evidence": "https://framework.beeai.dev/modules/tools" }, "context_memory": { "rating": "basic", "rank": 2, "detail": "Memory strategies (unlimited/last-k/token-limited/summarization) and BaseMemory snapshot methods manage the live run's conversation history; the docs describe no persistence of that state across separate sessions.", "evidence": "https://framework.beeai.dev/modules/memory" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "Middleware can set the `output` property on a component's `start` event to skip the underlying execution (e.g. an LLM call) entirely and return a mocked value instead, modify `input` before execution, and abort a run mid-stream via RunContext controller.abort().", "evidence": "https://framework.beeai.dev/modules/middleware" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Agents take first-class `role`/`instructions` parameters for system-prompt customization at construction time; no automatic prompt-tuning loop is documented.", "evidence": "https://raw.githubusercontent.com/i-am-bee/beeai-framework/main/README.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Self-hosted, pip/npm-installed agent SDK (Python + TypeScript) you configure and run; the companion Agent Stack (formerly BeeAI Platform) is itself open, self-hostable deployment infrastructure rather than a managed cloud service.", "evidence": "https://raw.githubusercontent.com/i-am-bee/beeai-framework/main/README.md" } } }, { "name": "AgentStack", "github_id": "agentstack-ai/AgentStack", "url": "https://github.com/agentstack-ai/AgentStack", "slug": "agentstack", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#agentstack", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/agentstack/", "description": "Scaffolds full agent projects; plugs in CrewAI, LangGraph, OpenAI Swarm, LlamaStack and wires AgentOps observability from day one.", "category": "frameworks", "category_title": "Frameworks", "stars": 2185, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (scaffold, multi-backend)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [], "example": { "label": "Research assistant crew", "url": "https://github.com/agentstack-ai/AgentStack/tree/main/examples/research_assistant" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "strong", "rank": 3, "detail": "The bundled CodeInterpreterTool runs agent-generated Python in a Docker container built on first use ('secure, isolated environment'), shipped as one of AgentStack's own core tools rather than delegated to a third party.", "evidence": "https://docs.agentstack.sh/tools/tool/code-interpreter" }, "context_memory": { "rating": "unknown", "rank": 0, "detail": "Looked for a memory/context-persistence module in AgentStack's own docs (vs. relying on whichever of CrewAI/LangGraph/OpenAI Swarms/LlamaIndex is scaffolded); found none in the docs index or introduction.", "evidence": "" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "AgentOps is 'baked into AgentStack projects by default' for monitoring, replay analytics, and compliance tracking -- observability/tracing, not a documented pre-execution blocking hook.", "evidence": "https://docs.agentstack.sh/llms.txt" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Agent role/goal/backstory and per-task description/expected-output are set declaratively in agents.yaml/tasks.yaml, independent of the underlying framework chosen; no automatic tuning loop is documented.", "evidence": "https://docs.agentstack.sh/llms.txt" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "A CLI scaffolder ('create-next-app for agents') that generates a self-hosted project pre-wired with a chosen framework (CrewAI/LangGraph/OpenAI Swarms/LlamaIndex), a standardized agents.yaml/tasks.yaml config layer, and built-in tools -- you configure rather than hand-write the agent loop; no managed hosting offering found.", "evidence": "https://docs.agentstack.sh/introduction" } } }, { "name": "AgentSilex", "github_id": "howl-anderson/agentsilex", "url": "https://github.com/howl-anderson/agentsilex", "slug": "agentsilex", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#agentsilex", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/agentsilex/", "description": "~300 lines of readable agent code on top of LiteLLM; the \"I want to see the whole loop\" option for learning or minimal production.", "category": "frameworks", "category_title": "Frameworks", "stars": 454, "tier": "super simple", "tier_rank": 1, "axis": "super simple (~300 LOC)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "none", "recovery_rank": 1, "license_signal": "open-source", "tags": [ "python" ], "example": { "label": "Simple weather agent", "url": "https://github.com/howl-anderson/agentsilex/blob/main/demo/simple_agent.py" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "unknown", "rank": 0, "detail": "Looked for any isolation mechanism around tool execution in the README and repo overview; none is documented one way or the other for this ~300-line minimal framework.", "evidence": "" }, "context_memory": { "rating": "basic", "rank": 2, "detail": "Session tracks dialog history and shared mutable state across tools/turns for the live run only; the README's own Roadmap lists 'State Persistence - Save and restore agent sessions' as a not-yet-implemented future feature.", "evidence": "https://raw.githubusercontent.com/howl-anderson/agentsilex/main/README.md" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "A 'pluggable callback system for implementing custom behaviors (ReAct, Chain-of-Thought, logging, etc.)' and callback-based memory management exist, but the README does not document specific before/after hook points or confirm callbacks can block/veto a tool call before it runs.", "evidence": "https://raw.githubusercontent.com/howl-anderson/agentsilex/main/README.md" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Agents are created with a first-class 'instructions' parameter for custom system-prompt text; no automatic tuning loop is documented.", "evidence": "https://raw.githubusercontent.com/howl-anderson/agentsilex/main/README.md" }, "build_vs_buy": { "tier": 1, "label": "build", "detail": "Explicitly positioned as '~300 lines of readable code. Full control, no magic' -- a transparent, hackable primitive set on top of LiteLLM that you read and compose yourself rather than an opinionated runtime; self-hosted, PyPI-installed, no managed offering.", "evidence": "https://raw.githubusercontent.com/howl-anderson/agentsilex/main/README.md" } } }, { "name": "SuperAgentX", "github_id": "superagentxai/superagentx", "url": "https://github.com/superagentxai/superagentx", "slug": "superagentx", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#superagentx", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/superagentx/", "description": "Lightweight multi-agent orchestrator with an AGI-angle; minimal surface, docs-first, for teams that want orchestration without the kitchen sink.", "category": "frameworks", "category_title": "Frameworks", "stars": 203, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (minimal surface)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "none", "recovery_rank": 1, "license_signal": "open-source", "tags": [ "multi-agent", "python" ], "example": { "label": "Parallel marketing agents", "url": "https://github.com/superagentxai/superagentx/blob/master/examples/agents/parallel_agents.py" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "Human-approval gates can pause/abort sensitive actions before they run, but this is an approval prompt, not execution isolation -- no container/VM/restricted-filesystem sandboxing for tools is documented; tool execution (e.g. Playwright browser automation) otherwise runs on the host.", "evidence": "https://github.com/superagentxai/superagentx" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Uses SQL + Vector databases (SQLite/PostgreSQL) to persist workflow state, agent decisions, tool outputs, and 'context & memory snapshots' for user-specific context retrieval across sessions.", "evidence": "https://github.com/superagentxai/superagentx" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "The Human Approval Governance Agent pauses a sensitive action for explicit human approval before resuming or aborting, with audit logs persisted separately - a built-in approval flow rather than a programmable pre-execution hook API.", "evidence": "https://github.com/superagentxai/superagentx" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Agents take PromptTemplate objects for customizable per-agent system messages; no automatic prompt-tuning loop is documented.", "evidence": "https://github.com/superagentxai/superagentx" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Self-hosted, policy-driven agent framework (MIT) with a 'Unified Control Plane' for tools/MCPs/models/policies you configure and run via WebSocket/REST/console interfaces, rather than a hosted managed platform.", "evidence": "https://github.com/superagentxai/superagentx" } } }, { "name": "MetaGPT", "github_id": "FoundationAgents/MetaGPT", "url": "https://github.com/FoundationAgents/MetaGPT", "slug": "metagpt", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#metagpt", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/metagpt/", "description": "The \"AI software company\" multi-agent framework: role-played PM, architect, and engineer agents turn a one-line requirement into specs, designs, and code along an SOP assembly line. The landmark of the genre; development pace has slowed in 2026.", "category": "multi-agent", "category_title": "Multi-agent and orchestration", "stars": 69962, "tier": "complex", "tier_rank": 4, "axis": "complex (role pipeline, SOPs — product suite)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "multi-agent", "python" ], "example": { "label": "Build a customized agent", "url": "https://github.com/FoundationAgents/MetaGPT/blob/main/examples/build_customized_agent.py" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "none", "rank": 1, "detail": "Code/shell tools execute directly in the host Python process via exec() and subprocess.run() (e.g. shell.py, terminal.py's persistent host shell with only a bypassable command blocklist), with an open issue proposing to add microVM sandboxing because none exists today.", "evidence": "https://github.com/FoundationAgents/MetaGPT/issues/1956" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Beyond per-run Memory (a list of observed Messages), MetaGPT's own paper describes a cross-project long-term memory where each role's 'handover feedback action' summarizes prior-project experience into long-term memory that is inherited by future constraint-prompt updates.", "evidence": "https://arxiv.org/html/2308.00352v6" }, "lifecycle_hooks": { "rating": "unknown", "rank": 0, "detail": "Looked for a before/after tool-call interception system beyond the core message-pool 'Environment' that roles observe; found no documented callback/middleware/hook API for blocking or tracing individual tool calls.", "evidence": "" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Agent roles and their prompt/action templates are user-definable (custom agents per the Agent 101 guide); the paper's self-improvement mechanism that auto-updates role constraint prompts is not a documented feature of the installable package.", "evidence": "https://raw.githubusercontent.com/FoundationAgents/MetaGPT/HEAD/README.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Self-hosted, pip-installed, role-based SOP framework (PM/Architect/Engineer roles configured via ~/.metagpt/config2.yaml) you run yourself; MGX (mgx.dev), 'the world's first AI agent development team,' is a separate managed cloud product built on the same ideas.", "evidence": "https://raw.githubusercontent.com/FoundationAgents/MetaGPT/main/README.md" } } }, { "name": "autogen", "github_id": "microsoft/autogen", "url": "https://github.com/microsoft/autogen", "slug": "autogen", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#autogen", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/autogen/", "description": "Conversable agents and group chats; code execution and human-in-the-loop; Microsoft origin, AG2 ecosystem. ⚠️ In maintenance mode since late 2025: no new features, and Microsoft directs new users to Agent Framework.", "category": "multi-agent", "category_title": "Multi-agent and orchestration", "stars": 60585, "tier": "complex", "tier_rank": 4, "axis": "complex (group chat, code exec, AG2 — product suite)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "multi-agent", "python" ], "example": { "label": "Distributed group chat", "url": "https://github.com/microsoft/autogen/tree/main/python/samples/core_distributed-group-chat" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "strong", "rank": 3, "detail": "DockerCommandLineCodeExecutor (autogen-ext[docker]) creates a Docker container and runs all generated code/shell commands inside it, and is the documented recommendation for production over the host-executing LocalCommandLineCodeExecutor.", "evidence": "https://microsoft.github.io/autogen/stable//user-guide/core-user-guide/components/command-line-code-executors.html" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "ChromaDBVectorMemory persists agent memory to a local folder (persistence_path/PersistentChromaDBVectorMemoryConfig) or a remote ChromaDB server, retrievable by similarity across sessions, in addition to in-process ListMemory.", "evidence": "https://microsoft.github.io/autogen/dev//reference/python/autogen_ext.memory.chromadb.html" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "The core-user-guide's Intervention Handler implements on_send to catch FunctionCall messages between the tool-use agent and tool executor and, on user denial, raises a ToolException that prevents the tool from ever executing -- a genuine pre-execution veto, not just logging.", "evidence": "https://microsoft.github.io/autogen/stable//user-guide/core-user-guide/cookbook/tool-use-with-intervention.html" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Agents take a first-class system_message parameter for custom instructions (e.g. system_message='You are a math expert'); no automatic prompt-tuning loop is documented.", "evidence": "https://raw.githubusercontent.com/microsoft/autogen/main/README.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Self-hosted, pip-installed multi-agent conversation framework (group chats, code execution, human-in-the-loop) you configure and run locally; the README itself states AutoGen is 'now in maintenance mode' with Microsoft Agent Framework recommended for new projects, and AutoGen Studio is a local no-code GUI, not a managed cloud service.", "evidence": "https://raw.githubusercontent.com/microsoft/autogen/main/README.md" } } }, { "name": "OpenManus", "github_id": "FoundationAgents/OpenManus", "url": "https://github.com/FoundationAgents/OpenManus", "slug": "openmanus", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#openmanus", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/openmanus/", "description": "Open, invite-free general agent from the MetaGPT team: planning plus tool use over a multi-agent loop, aimed at reproducing Manus-style autonomous task completion on your own keys.", "category": "multi-agent", "category_title": "Multi-agent and orchestration", "stars": 58048, "tier": "complex", "tier_rank": 4, "axis": "complex (multi-agent + tools)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "none", "recovery_rank": 1, "license_signal": "open-source", "tags": [ "multi-agent", "python" ], "example": { "label": "Quickstart", "url": "https://github.com/FoundationAgents/OpenManus#readme" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "Ships its own Docker-based [sandbox] execution mode (image, memory/cpu limits, network settings in config.toml) but it is opt-in and disabled by default (`#use_sandbox = false`, commented out), so tool execution runs on the host by default; a separate open issue proposes extending sandboxing to cover all tool operations by default.", "evidence": "https://raw.githubusercontent.com/FoundationAgents/OpenManus/main/config/config.example.toml" }, "context_memory": { "rating": "basic", "rank": 2, "detail": "Agents add results to an in-process memory/context object during a single interactive task loop; no database or file-based persistence across separate sessions is documented.", "evidence": "https://llmmultiagents.com/en/blogs/OpenManus_Technical_Analysis" }, "lifecycle_hooks": { "rating": "unknown", "rank": 0, "detail": "Looked for a before/after tool-call callback or middleware system beyond the BaseAgent/ReActAgent/ToolCallAgent class hierarchy; found no documented interception API for blocking or tracing individual tool calls.", "evidence": "" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Agent system_prompt/next_step_prompt were moved out of code and into config.toml so users can adjust agent behavior (e.g. Manus/PlanningAgent prompts) without code changes.", "evidence": "https://raw.githubusercontent.com/FoundationAgents/OpenManus/main/README.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Self-hosted, MIT-licensed, config.toml-driven agent framework (conda/uv local install) positioned as an open alternative to invite-only Manus; only a HuggingFace Spaces demo exists, no managed hosted offering.", "evidence": "https://raw.githubusercontent.com/FoundationAgents/OpenManus/main/README.md" } } }, { "name": "crewAI", "github_id": "crewAIInc/crewAI", "url": "https://github.com/crewAIInc/crewAI", "slug": "crewai", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#crewai", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/crewai/", "description": "Role-based agents (roles, goals, backstories) in Crews; Flows add event-driven and hierarchical control for production.", "category": "multi-agent", "category_title": "Multi-agent and orchestration", "stars": 57501, "tier": "complex", "tier_rank": 4, "axis": "complex (roles, Flows, production — product suite)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "python" ], "example": { "label": "Trip planner crew", "url": "https://github.com/crewAIInc/crewAI-examples/blob/main/crews/trip_planner/trip_agents.py" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "The former CodeInterpreterTool ran Python in an isolated Docker container by default but has been removed from crewai-tools in favor of delegating code execution to external sandbox services (E2B, Modal); other tools run directly on the host.", "evidence": "https://docs.crewai.com/en/tools/ai-ml/codeinterpretertool" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "A unified Memory class persists across sessions using LanceDB vector storage by default, retrieving via composite scoring of semantic similarity, recency decay, and LLM-assigned importance.", "evidence": "https://docs.crewai.com/en/concepts/memory" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "Task guardrails reject/transform task output with automatic retries and before/after_kickoff callbacks modify crew-level inputs/outputs - observation and post-hoc transformation, without per-tool-call pre-execution blocking.", "evidence": "https://docs.crewai.com/en/concepts/tasks" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Agents are customized via role/goal/backstory plus optional system_template/prompt_template/response_template overrides, with no automatic prompt tuning or eval-driven search documented.", "evidence": "https://docs.crewai.com/en/concepts/agents" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "The pip-installed crewai package is a self-hosted Python agent framework (agents, tasks, Flows) you configure and run yourself, with CrewAI AMP/Enterprise at app.crewai.com as the optional hosted control plane.", "evidence": "https://raw.githubusercontent.com/crewAIInc/crewAI/main/README.md" } } }, { "name": "ChatDev", "github_id": "OpenBMB/ChatDev", "url": "https://github.com/OpenBMB/ChatDev", "slug": "chatdev", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#chatdev", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/chatdev/", "description": "Multi-agent software-company simulation (CEO, CTO, programmer, tester) built on chat chains with communicative dehallucination; ChatDev 2.0 continues the line. MetaGPT's conversational sibling.", "category": "multi-agent", "category_title": "Multi-agent and orchestration", "stars": 34103, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (chat-chain simulation)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "none", "recovery_rank": 1, "license_signal": "open-source", "tags": [ "python" ], "example": { "label": "Company simulation quickstart", "url": "https://github.com/OpenBMB/ChatDev#readme" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "Python/function tool nodes execute in a per-session code_workspace/ directory with timeout/env config, but the docs describe no enforced container or VM isolation for individual tool calls — Docker Compose only wraps the whole backend/frontend app.", "evidence": "https://github.com/OpenBMB/ChatDev/blob/main/docs/user_guide/en/workflow_authoring.md" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Pluggable memory stores (SimpleMemory with FAISS + optional JSON disk persistence, Mem0Memory fully cloud-managed) persist agent memory across runs and sessions, not just within one conversation.", "evidence": "https://github.com/OpenBMB/ChatDev/blob/main/docs/user_guide/en/modules/memory.md" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "Workflows support a human-node review mode where a person can inspect intermediate artifacts and give feedback mid-run, but no documented pre-tool-call hook can block/rewrite an action before execution.", "evidence": "https://github.com/OpenBMB/ChatDev/blob/main/README.md" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Agent nodes take a prompt_template field with {{variable}} interpolation and YAML-defined role/phase/ChatChain settings, but no automatic prompt-tuning loop is documented.", "evidence": "https://github.com/OpenBMB/ChatDev/blob/main/docs/user_guide/en/workflow_authoring.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Ships as a self-hostable, YAML-configured multi-agent workflow runtime (DAG of model/python/tooling/human nodes) you run yourself via Docker Compose; an older hosted SaaS at chatdev.modelbest.cn is referenced as historical, not the primary artifact.", "evidence": "https://github.com/OpenBMB/ChatDev/blob/main/README.md" } } }, { "name": "openai-agents-python", "github_id": "openai/openai-agents-python", "url": "https://github.com/openai/openai-agents-python", "slug": "openai-agents-python", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#openai-agents-python", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/openai-agents-python/", "description": "Handoffs, guardrails, and multi-LLM routing; minimal surface so you own the loop.", "category": "multi-agent", "category_title": "Multi-agent and orchestration", "stars": 28887, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (minimal surface)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "python" ], "example": { "label": "Airline customer service handoffs", "url": "https://github.com/openai/openai-agents-python/blob/main/examples/customer_service/main.py" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "strong", "rank": 3, "detail": "Ships a Sandbox Agents execution harness with built-in UnixLocalSandboxClient and DockerSandboxClient backends that handle file staging, filesystem tools, shell access, and sandbox lifecycle for the agent.", "evidence": "https://openai.github.io/openai-agents-python/sandbox_agents/" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Sessions persist conversation history to SQLite/SQLAlchemy/Redis/MongoDB/Dapr backends across process restarts, and OpenAIResponsesCompactionSession auto-compacts long histories via the Responses API.", "evidence": "https://openai.github.io/openai-agents-python/sessions/" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "Tool guardrails run before a function tool executes and can skip the call, replace its output, or raise a tripwire, and blocking-mode input guardrails prevent the agent from starting at all.", "evidence": "https://openai.github.io/openai-agents-python/guardrails/" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Agents are configured with a free-text instructions field per agent/handoff, but no automatic prompt-tuning or eval-driven prompt search is documented.", "evidence": "https://raw.githubusercontent.com/openai/openai-agents-python/main/README.md" }, "build_vs_buy": { "tier": 1, "label": "build", "detail": "Installed via pip as an SDK where you write and own the Runner loop, agents, and tool wiring; OpenAIConversationsSession and hosted sandbox providers exist as optional managed backends but aren't required to run it.", "evidence": "https://raw.githubusercontent.com/openai/openai-agents-python/main/README.md" } } }, { "name": "Microsoft Agent Framework", "github_id": "microsoft/agent-framework", "url": "https://github.com/microsoft/agent-framework", "slug": "agent-framework", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#agent-framework", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/agent-framework/", "description": "Microsoft's convergence of AutoGen and Semantic Kernel: build, orchestrate, and deploy agents and multi-agent workflows in Python and .NET, with graph-based workflows and checkpointing — the designated successor harness for both lines.", "category": "multi-agent", "category_title": "Multi-agent and orchestration", "stars": 13060, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (Python/.NET SDK, graph workflows)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "multi-agent", "workflow", "python" ], "example": { "label": "Python samples", "url": "https://github.com/microsoft/agent-framework/tree/main/python/samples" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "HostedCodeInterpreterTool is documented as just a marker telling an external service it may execute generated code — actual isolation (e.g. Azure Container Apps Dynamic Sessions/Hyper-V) is delegated to a separate hosted service, not enforced by the OSS framework itself.", "evidence": "https://learn.microsoft.com/en-us/python/api/agent-framework-core/agent_framework.hostedcodeinterpretertool?view=agent-framework-python-latest" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Workflow checkpointing captures full executor/message/shared state at each superstep via FileCheckpointStorage or CosmosCheckpointStorage, which persist to disk or Cosmos DB and can rehydrate a new run across process restarts.", "evidence": "https://learn.microsoft.com/en-us/agent-framework/workflows/checkpoints" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "FunctionMiddleware/AgentMiddleware wrap each call with a call_next() continuation; the documented SecurityAgentMiddleware example blocks a function/agent call entirely by simply not invoking call_next(), overriding the result before execution.", "evidence": "https://learn.microsoft.com/en-us/agent-framework/agents/middleware/defining-middleware" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Agents take a free-text instructions parameter set per agent/client; no automatic prompt-tuning or eval-driven optimization loop is documented.", "evidence": "https://raw.githubusercontent.com/microsoft/agent-framework/main/README.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Ships as a self-hostable, multi-language (Python/.NET/Go) opinionated runtime with graph-based workflows, middleware, and checkpointing that you run yourself; optional 'Foundry Hosted Agents' cloud deployment is named as an alternative, not the primary artifact.", "evidence": "https://raw.githubusercontent.com/microsoft/agent-framework/main/README.md" } } }, { "name": "hive", "github_id": "aden-hive/hive", "url": "https://github.com/aden-hive/hive", "slug": "hive", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#hive", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/hive/", "description": "Self-hosted multi-agent **harness** aimed at production workloads: human-in-the-loop checkpoints and a self-improving agent loop, distinct from single-session coding-agent shells.", "category": "multi-agent", "category_title": "Multi-agent and orchestration", "stars": 10949, "tier": "complex", "tier_rank": 4, "axis": "complex (multi-agent, human-in-the-loop — product suite)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "resumable", "recovery_rank": 3, "license_signal": "unknown", "tags": [ "multi-agent", "python" ], "example": { "label": "Project README", "url": "https://github.com/aden-hive/hive#readme" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "Tool calls dispatch through a plain tool_executor callable with no documented OS-level container/VM isolation; the only execution-time guard found is content validation on SharedMemory writes, plus optional human-in-the-loop approval gates.", "evidence": "https://deepwiki.com/aden-hive/hive/3.2-graph-executor-and-node-execution" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "CheckpointStore snapshots graph state (memory, paused_at, execution_path, node_visit_counts) to state.json in the session directory at node boundaries, enabling resume after a process crash, alongside role-based memory that persists project context.", "evidence": "https://deepwiki.com/aden-hive/hive/3.2-graph-executor-and-node-execution" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "HITL nodes pause the graph at node boundaries via pause_nodes/approval_callback and route based on human response, but the docs describe this as pausing whole node execution rather than vetoing a specific tool call before it runs.", "evidence": "https://docs.adenhq.com/building/human-in-the-loop.md" }, "prompt_optimization": { "rating": "native", "rank": 3, "detail": "An automatic execute-evaluate-diagnose-regenerate evolution loop refines a node's system prompt based on diagnosed production failures without manual human editing — 'the most common fix' per the docs.", "evidence": "https://github.com/aden-hive/hive/blob/main/docs/key_concepts/evolution.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Apache-2.0 self-hosted graph-based agent runtime you deploy on your own infrastructure per the Self-Hosting Guide, with no managed hosted alternative advertised as the primary offering.", "evidence": "https://raw.githubusercontent.com/aden-hive/hive/main/README.md" } } }, { "name": "omnigent", "github_id": "omnigent-ai/omnigent", "url": "https://github.com/omnigent-ai/omnigent", "slug": "omnigent", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#omnigent", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/omnigent/", "description": "Open-source meta-**harness**: orchestrates Claude Code, Codex, Cursor, Pi, and custom agents behind one policy/sandboxing layer so teams swap harnesses without rewriting workflows.", "category": "multi-agent", "category_title": "Multi-agent and orchestration", "stars": 9189, "tier": "complex", "tier_rank": 4, "axis": "complex (meta-harness, policy + sandboxing — product suite)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "unknown", "tags": [ "sandbox", "ide", "python" ], "example": { "label": "Project README", "url": "https://github.com/omnigent-ai/omnigent#readme" }, "deep_dive": null }, { "name": "PraisonAI", "github_id": "MervinPraison/PraisonAI", "url": "https://github.com/MervinPraison/PraisonAI", "slug": "praisonai", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#praisonai", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/praisonai/", "description": "Autonomous multi-agent teams with a single entry point; emphasis on minimal config.", "category": "multi-agent", "category_title": "Multi-agent and orchestration", "stars": 8947, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (single entry, minimal config)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "none", "recovery_rank": 1, "license_signal": "open-source", "tags": [ "multi-agent", "python" ], "example": { "label": "Orchestrator-workers pattern", "url": "https://github.com/MervinPraison/PraisonAI/blob/main/examples/python/general/orchestrator-workers.py" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "CLI sandbox execution (command validation, resource limits, filesystem/network restrictions) is only activated via an explicit --sandbox flag — by default commands run unsandboxed on the host.", "evidence": "https://praison.ai/docs/cli/sandbox-execution" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Graph Memory persists agent-derived entities/relationships to an external Neo4j or Memgraph database, surviving across process restarts and sessions rather than living only in the conversation.", "evidence": "https://praison.ai/docs/features/graph-memory" }, "lifecycle_hooks": { "rating": "full", "rank": 3, "detail": "A documented BEFORE_TOOL hook can validate arguments and raise an exception to veto a tool call before it executes (e.g. blocking an unconfirmed delete_file call), with a separate AFTER_TOOL hook for result logging/transformation.", "evidence": "https://praison.ai/docs/concepts/hooks" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Agents take a free-text instructions parameter and YAML-defined role/goal/backstory config; guardrails only validate output post-generation with retry, and no automatic prompt-tuning loop is documented.", "evidence": "https://praison.ai/docs/concepts/guardrails" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Self-hosted, pip/npm-installed opinionated agent runtime with YAML-configured agents, built-in sessions/memory backends, and a CLI — you run the process yourself; third-party integrations (Langfuse, Tavily) are optional add-ons, not a required managed control plane.", "evidence": "https://raw.githubusercontent.com/MervinPraison/PraisonAI/main/README.md" } } }, { "name": "AG2", "github_id": "ag2ai/ag2", "url": "https://github.com/ag2ai/ag2", "slug": "ag2", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#ag2", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/ag2/", "description": "AG2 (formerly AutoGen): the community-governed continuation of the original AutoGen project after Microsoft's fork diverged—conversable multi-agent groups, code execution, and human-in-the-loop under an open-source AgentOS banner. Graduated off the radar this cycle.", "category": "multi-agent", "category_title": "Multi-agent and orchestration", "stars": 4883, "tier": "complex", "tier_rank": 4, "axis": "complex (conversable multi-agent, AgentOS — product suite)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "unknown", "tags": [ "multi-agent", "python" ], "example": { "label": "Project README", "url": "https://github.com/ag2ai/ag2#readme" }, "deep_dive": null }, { "name": "AgentRL", "github_id": "THUDM/AgentRL", "url": "https://github.com/THUDM/AgentRL", "slug": "agentrl", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#agentrl", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/agentrl/", "description": "Multitask, multiturn RL for LLM agents; Ray-based scaling, rollout/actor workers—for teams that want to train agents, not just run them.", "category": "multi-agent", "category_title": "Multi-agent and orchestration", "stars": 347, "tier": "complex", "tier_rank": 4, "axis": "complex (RL, Ray, train agents — product suite)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "training", "python" ], "example": { "label": "Async GRPO trainer", "url": "https://github.com/THUDM/AgentRL/blob/main/examples/training/async_trainer.py" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "Task workers can run as plain Python processes or, recommended for scaling, inside Docker containers built from a template Dockerfile — but the docs frame this as operational distribution across machines, not security sandboxing, and it isn't the default.", "evidence": "https://github.com/THUDM/AgentRL/blob/main/docs/deployment.md" }, "context_memory": { "rating": "unknown", "rank": 0, "detail": "Looked for session/cross-run persistence in the README and deployment docs; found only group_id-based trajectory grouping that keeps one multi-turn rollout together during training, not a memory or context-persistence mechanism across sessions.", "evidence": "" }, "lifecycle_hooks": { "rating": "unknown", "rank": 0, "detail": "Searched README, deployment.md, and architecture summaries for a hook/callback/middleware system that could intercept or block an agent/tool action; none is documented — the 'plugin registry' reference is about parameter streaming between Ray workers, not action interception.", "evidence": "" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "Each TaskWorker implementation is responsible for constructing the system and user prompts for its environment, so prompt customization happens per-task in code (per docs/tasks.md) rather than via a hardcoded fixed prompt; no automatic prompt-tuning loop is documented.", "evidence": "https://github.com/THUDM/AgentRL/blob/main/docs/deployment.md" }, "build_vs_buy": { "tier": 1, "label": "build", "detail": "Installed as pip packages (trainer/controller/worker) that you wire together yourself — a Ray-based RL training harness and set of primitives for rollout/actor/reference workers, not an opinionated turnkey runtime or hosted service.", "evidence": "https://raw.githubusercontent.com/THUDM/AgentRL/main/README.md" } } }, { "name": "MCP Servers", "github_id": "modelcontextprotocol/servers", "url": "https://github.com/modelcontextprotocol/servers", "slug": "servers", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#servers", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/servers/", "description": "The official reference collection of Model Context Protocol servers (filesystem, git, fetch, memory, time, and more)—the canonical, vetted toolset agents connect to, and the pattern every other MCP server is measured against.", "category": "plugins-mcp-cli", "category_title": "Plugins, MCPs, CLI tools", "stars": 89798, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (reference servers)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "mcp", "memory", "typescript" ], "example": { "label": "Server catalog", "url": "https://github.com/modelcontextprotocol/servers#readme" }, "deep_dive": null }, { "name": "Context7", "github_id": "upstash/context7", "url": "https://github.com/upstash/context7", "slug": "context7", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#context7", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/context7/", "description": "MCP server that injects up-to-date, version-specific library docs into an agent's context on demand; kills the stale-training-data hallucinations that plague codegen.", "category": "plugins-mcp-cli", "category_title": "Plugins, MCPs, CLI tools", "stars": 61104, "tier": "super simple", "tier_rank": 1, "axis": "super simple (drop-in MCP)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "mcp", "training", "typescript" ], "example": { "label": "Docs", "url": "https://context7.com" }, "deep_dive": null }, { "name": "chrome-devtools-mcp", "github_id": "ChromeDevTools/chrome-devtools-mcp", "url": "https://github.com/ChromeDevTools/chrome-devtools-mcp", "slug": "chrome-devtools-mcp", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#chrome-devtools-mcp", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/chrome-devtools-mcp/", "description": "Google's official Chrome DevTools MCP server: exposes console, network, and performance-trace inspection as tool calls agents can drive directly, instead of a human clicking through the DevTools panel.", "category": "plugins-mcp-cli", "category_title": "Plugins, MCPs, CLI tools", "stars": 49601, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (official browser-debug MCP)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "mcp", "browser", "typescript" ], "example": { "label": "Project README", "url": "https://github.com/ChromeDevTools/chrome-devtools-mcp#readme" }, "deep_dive": null }, { "name": "aider", "github_id": "Aider-AI/aider", "url": "https://github.com/Aider-AI/aider", "slug": "aider", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#aider", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/aider/", "description": "Git-aware CLI pair programmer; edits in-repo, supports multiple models and MCP so agents see version control and tools.", "category": "plugins-mcp-cli", "category_title": "Plugins, MCPs, CLI tools", "stars": 48420, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (CLI, git-aware, MCP)", "autonomy": "checkpoint-gated", "autonomy_rank": 2, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "mcp", "cli", "python" ], "example": { "label": "Repo map source", "url": "https://github.com/Aider-AI/aider/blob/main/aider/repomap.py" }, "deep_dive": null }, { "name": "Playwright MCP", "github_id": "microsoft/playwright-mcp", "url": "https://github.com/microsoft/playwright-mcp", "slug": "playwright-mcp", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#playwright-mcp", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/playwright-mcp/", "description": "Playwright's official MCP server: structured browser control (navigate, click, fill, extract) via the accessibility tree rather than screenshots, so web tasks stay fast and deterministic.", "category": "plugins-mcp-cli", "category_title": "Plugins, MCPs, CLI tools", "stars": 36382, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (browser MCP)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "mcp", "vision", "browser", "typescript" ], "example": { "label": "Setup & config", "url": "https://github.com/microsoft/playwright-mcp#readme" }, "deep_dive": null }, { "name": "continue", "github_id": "continuedev/continue", "url": "https://github.com/continuedev/continue", "slug": "continue", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#continue", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/continue/", "description": "Open-source IDE extension (VS Code, JetBrains); in-editor completion and chat with local or API models.", "category": "plugins-mcp-cli", "category_title": "Plugins, MCPs, CLI tools", "stars": 35601, "tier": "complex", "tier_rank": 4, "axis": "complex (IDE extension, multi-editor — product suite)", "autonomy": "checkpoint-gated", "autonomy_rank": 2, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "ide", "typescript" ], "example": { "label": "VS Code extension demos", "url": "https://github.com/continuedev/continue/blob/main/extensions/vscode/README.md" }, "deep_dive": null }, { "name": "github-mcp-server", "github_id": "github/github-mcp-server", "url": "https://github.com/github/github-mcp-server", "slug": "github-mcp-server", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#github-mcp-server", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/github-mcp-server/", "description": "GitHub's official MCP server (Go): repos, issues, PRs, code search, Actions. Replaces the older community `cyanheads/github-mcp-server` as the canonical way to give agents GitHub access.", "category": "plugins-mcp-cli", "category_title": "Plugins, MCPs, CLI tools", "stars": 32440, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (official GitHub MCP)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "mcp" ], "example": { "label": "Remote server toolsets", "url": "https://github.com/github/github-mcp-server/blob/main/docs/remote-server.md" }, "deep_dive": null }, { "name": "MCP Python SDK", "github_id": "modelcontextprotocol/python-sdk", "url": "https://github.com/modelcontextprotocol/python-sdk", "slug": "python-sdk", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#python-sdk", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/python-sdk/", "description": "Official SDK to build and consume MCP servers/clients in Python; stdio and SSE transports.", "category": "plugins-mcp-cli", "category_title": "Plugins, MCPs, CLI tools", "stars": 24090, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (SDK only)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "mcp", "python" ], "example": { "label": "Website fetcher server", "url": "https://github.com/modelcontextprotocol/python-sdk/blob/main/examples/servers/simple-tool/mcp_simple_tool/server.py" }, "deep_dive": null }, { "name": "MCP TypeScript SDK", "github_id": "modelcontextprotocol/typescript-sdk", "url": "https://github.com/modelcontextprotocol/typescript-sdk", "slug": "typescript-sdk", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#typescript-sdk", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/typescript-sdk/", "description": "Official MCP implementation for Node/TS; reference for the protocol.", "category": "plugins-mcp-cli", "category_title": "Plugins, MCPs, CLI tools", "stars": 13230, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (protocol reference)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "mcp", "typescript" ], "example": { "label": "Streamable HTTP server", "url": "https://github.com/modelcontextprotocol/typescript-sdk/blob/main/examples/server/src/simpleStreamableHttp.ts" }, "deep_dive": null }, { "name": "MCP Inspector", "github_id": "modelcontextprotocol/inspector", "url": "https://github.com/modelcontextprotocol/inspector", "slug": "inspector", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#inspector", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/inspector/", "description": "GUI to test and debug MCP servers; inspect tools, resources, and prompts.", "category": "plugins-mcp-cli", "category_title": "Plugins, MCPs, CLI tools", "stars": 10730, "tier": "super simple", "tier_rank": 1, "axis": "super simple (debug GUI)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "mcp", "typescript" ], "example": { "label": "Inspector UI walkthrough", "url": "https://github.com/modelcontextprotocol/inspector/blob/main/README.md" }, "deep_dive": null }, { "name": "MCP Registry", "github_id": "modelcontextprotocol/registry", "url": "https://github.com/modelcontextprotocol/registry", "slug": "registry", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#registry", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/registry/", "description": "Official, community-driven registry for MCP servers—the \"app store\" MCP clients use to discover servers. Maintained by Anthropic + ecosystem maintainers; v0.1 API frozen, production-grade.", "category": "plugins-mcp-cli", "category_title": "Plugins, MCPs, CLI tools", "stars": 7185, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (official discovery layer)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "mcp" ], "example": { "label": "Registry seed entries", "url": "https://github.com/modelcontextprotocol/registry/blob/main/data/seed.json" }, "deep_dive": null }, { "name": "Agent Governance Toolkit", "github_id": "microsoft/agent-governance-toolkit", "url": "https://github.com/microsoft/agent-governance-toolkit", "slug": "agent-governance-toolkit", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#agent-governance-toolkit", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/agent-governance-toolkit/", "description": "Microsoft's policy-enforcement layer for autonomous agents: zero-trust identity, execution sandboxing, and OWASP Agentic Top-10 coverage sit in front of the agent loop as a **harness** security layer—the governance counterpart to Infisical's agent-vault.", "category": "plugins-mcp-cli", "category_title": "Plugins, MCPs, CLI tools", "stars": 6091, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (policy engine, zero-trust, sandboxing)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "sandbox", "python" ], "example": { "label": "Project README", "url": "https://github.com/microsoft/agent-governance-toolkit#readme" }, "deep_dive": null }, { "name": "mcp-context-forge", "github_id": "IBM/mcp-context-forge", "url": "https://github.com/IBM/mcp-context-forge", "slug": "mcp-context-forge", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#mcp-context-forge", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/mcp-context-forge/", "description": "IBM's official AI gateway/registry/proxy that sits in front of any MCP, A2A, or REST/gRPC API: unified endpoint, centralized discovery, guardrails, and plugin support—the enterprise front door for **harness** tool calling. Graduated off the radar this cycle.", "category": "plugins-mcp-cli", "category_title": "Plugins, MCPs, CLI tools", "stars": 4350, "tier": "complex", "tier_rank": 4, "axis": "complex (gateway, federation, guardrails — product suite)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "unknown", "tags": [ "mcp", "python" ], "example": { "label": "Project README", "url": "https://github.com/IBM/mcp-context-forge#readme" }, "deep_dive": null }, { "name": "cocoindex-code", "github_id": "cocoindex-io/cocoindex-code", "url": "https://github.com/cocoindex-io/cocoindex-code", "slug": "cocoindex-code", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#cocoindex-code", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/cocoindex-code/", "description": "Embedded, tree-sitter/AST-based code-search CLI and MCP server that gives coding agents fast semantic lookups over a repo instead of grepping or re-reading whole files into context.", "category": "plugins-mcp-cli", "category_title": "Plugins, MCPs, CLI tools", "stars": 2670, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (embedded CLI + MCP server)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "unknown", "tags": [ "mcp", "cli" ], "example": { "label": "Project README", "url": "https://github.com/cocoindex-io/cocoindex-code#readme" }, "deep_dive": null }, { "name": "agent-vault", "github_id": "Infisical/agent-vault", "url": "https://github.com/Infisical/agent-vault", "slug": "agent-vault", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#agent-vault", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/agent-vault/", "description": "Infisical's HTTP credential proxy that fronts secrets for Claude Code, OpenClaw, and other agent harnesses so the agent's tool calls never see raw credentials—a **harness** security layer, not an agent loop itself.", "category": "plugins-mcp-cli", "category_title": "Plugins, MCPs, CLI tools", "stars": 2124, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (credential proxy)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "unknown", "tags": [], "example": { "label": "Project README", "url": "https://github.com/Infisical/agent-vault#readme" }, "deep_dive": null }, { "name": "Docker MCP Gateway", "github_id": "docker/mcp-gateway", "url": "https://github.com/docker/mcp-gateway", "slug": "mcp-gateway", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#mcp-gateway", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/mcp-gateway/", "description": "Docker's official MCP CLI plugin / gateway; container-aware MCP tooling from Docker (replaces deprecated `docker/mcp-servers` path).", "category": "plugins-mcp-cli", "category_title": "Plugins, MCPs, CLI tools", "stars": 1537, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (Docker-aware MCPs)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "mcp", "sandbox", "cli" ], "example": { "label": "Gateway usage walkthrough", "url": "https://github.com/docker/mcp-gateway/blob/main/docs/mcp-gateway.md" }, "deep_dive": null }, { "name": "puppeteer-real-browser-mcp", "github_id": "withLinda/puppeteer-real-browser-mcp-server", "url": "https://github.com/withLinda/puppeteer-real-browser-mcp-server", "slug": "puppeteer-real-browser-mcp-server", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#puppeteer-real-browser-mcp-server", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/puppeteer-real-browser-mcp-server/", "description": "Puppeteer MCP with real-browser and anti-detection; for agents that need to drive sites that block headless.", "category": "plugins-mcp-cli", "category_title": "Plugins, MCPs, CLI tools", "stars": 26, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (real browser, anti-detect)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "unknown", "tags": [ "mcp", "browser", "typescript" ], "example": { "label": "11 anti-detection tools", "url": "https://github.com/withLinda/puppeteer-real-browser-mcp-server/blob/main/README.md" }, "deep_dive": null }, { "name": "Better-OpenCodeMCP", "github_id": "ajhcs/Better-OpenCodeMCP", "url": "https://github.com/ajhcs/Better-OpenCodeMCP", "slug": "better-opencodemcp", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#better-opencodemcp", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/better-opencodemcp/", "description": "MCP server for OpenCode/Crush: async task execution, model bridging (e.g. Claude→Gemini), process pooling.", "category": "plugins-mcp-cli", "category_title": "Plugins, MCPs, CLI tools", "stars": 9, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (MCP server, model bridging)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "mcp", "typescript" ], "example": { "label": "opencode delegate tool", "url": "https://github.com/ajhcs/Better-OpenCodeMCP/blob/main/src/tools/opencode.tool.ts" }, "deep_dive": null }, { "name": "agentlog", "github_id": "RyanAlberts/agentlog", "url": "https://github.com/RyanAlberts/agentlog", "slug": "agentlog", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#agentlog", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/agentlog/", "description": "Persistent decision memory for any project: `remember`, `recall`, `reflect`. Single-file Python CLI that stores decisions as JSONL and uses Claude or Gemini to retrieve and synthesize patterns—Karpathy's LLM Wiki concept as a CLI.", "category": "plugins-mcp-cli", "category_title": "Plugins, MCPs, CLI tools", "stars": 1, "tier": "super simple", "tier_rank": 1, "axis": "super simple (one file, three commands)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "memory", "cli", "python" ], "example": { "label": "Sample decisions.jsonl", "url": "https://github.com/RyanAlberts/agentlog/blob/main/example-log/decisions.jsonl" }, "deep_dive": null }, { "name": "claude-mem", "github_id": "thedotmack/claude-mem", "url": "https://github.com/thedotmack/claude-mem", "slug": "claude-mem", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#claude-mem", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/claude-mem/", "description": "Session-memory plugin for Claude Code, Codex, OpenClaw, Gemini, Copilot, and more: captures everything an agent does during a session, AI-compresses it, and injects the relevant context into future sessions. Session-to-session memory as a drop-in.", "category": "memory", "category_title": "Memory and state", "stars": 91578, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (session capture + compression)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "memory" ], "example": { "label": "Lifecycle hooks config", "url": "https://github.com/thedotmack/claude-mem/blob/main/plugin/hooks/hooks.json" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "none", "rank": 1, "detail": "claude-mem only observes; its hooks documentation describes every hook (SessionStart, UserPromptSubmit, PostToolUse, Stop, SessionEnd) as non-blocking fire-and-forget calls that never intercept or restrict what the host agent's tools do.", "evidence": "https://docs.claude-mem.ai/architecture/hooks" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "A local SQLite database stores sessions/observations/summaries and a Chroma vector store enables hybrid semantic+keyword search, so compressed context from past sessions is retrieved and re-injected into future sessions rather than being lost when the process ends.", "evidence": "https://raw.githubusercontent.com/thedotmack/claude-mem/main/README.md" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "All 5 lifecycle hooks (SessionStart, UserPromptSubmit, PostToolUse, Stop, SessionEnd) are documented as observer-only, fire-and-forget HTTP calls with 2-second timeouts — none can block or veto a tool call before it executes.", "evidence": "https://docs.claude-mem.ai/architecture/hooks" }, "prompt_optimization": { "rating": "none", "rank": 1, "detail": "Configuration only covers claude-mem's own behavior (workflow mode, observation language, which model compresses observations, what gets injected as additionalContext) — there is no mechanism to customize or auto-tune the host agent's own system prompt.", "evidence": "https://docs.claude-mem.ai/configuration" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Installs as a self-hosted local service (Bun-managed worker, SQLite + Chroma storage, hook scripts) via a single npx command that you run on your own machine; optional cmem.ai cloud backup exists as a sync convenience, not the primary artifact.", "evidence": "https://raw.githubusercontent.com/thedotmack/claude-mem/main/README.md" } } }, { "name": "Mem0", "github_id": "mem0ai/mem0", "url": "https://github.com/mem0ai/mem0", "slug": "mem0", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#mem0", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/mem0/", "description": "Universal memory layer for AI agents: stores user/org/session memory, retrieves on demand. Apache-2.0; the de-facto memory primitive paired with most harnesses in 2026.", "category": "memory", "category_title": "Memory and state", "stars": 63868, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (memory layer, multi-platform)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "memory", "python" ], "example": { "label": "Next.js memory demo", "url": "https://github.com/mem0ai/mem0/tree/main/examples/mem0-demo" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "none", "rank": 1, "detail": "Mem0 is purely a memory storage/retrieval API (add, extract, recall) and does not execute tools, code, or actions on behalf of an agent, so there is nothing to sandbox.", "evidence": "https://docs.mem0.ai/overview" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "Memories persist across sessions in a vector store with a parallel per-collection entity index, retrieved via multi-signal hybrid search (semantic + BM25 keyword + entity matching).", "evidence": "https://docs.mem0.ai/open-source/graph_memory/overview" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "Hosted-Platform webhooks send an HTTP POST notification after memory_add/update/delete/categorize events, but the documentation confirms they cannot block or modify the underlying memory operation.", "evidence": "https://docs.mem0.ai/platform/features/webhooks" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "custom_instructions (superseding custom_fact_extraction_prompt/custom_update_memory_prompt) let developers hand-write and version their own fact-extraction and update prompts, with no automatic optimization of them.", "evidence": "https://docs.mem0.ai/open-source/features/custom-update-memory-prompt" }, "build_vs_buy": { "tier": 1, "label": "build", "detail": "The mem0ai/mem0 pip and npm packages are a Memory() client library you import into your own app with no opinionated runtime process; Mem0 Platform at app.mem0.ai is the fully managed hosted alternative.", "evidence": "https://raw.githubusercontent.com/mem0ai/mem0/main/README.md" } } }, { "name": "Graphiti (Zep)", "github_id": "getzep/graphiti", "url": "https://github.com/getzep/graphiti", "slug": "graphiti", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#graphiti", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/graphiti/", "description": "Zep's open-source memory engine: real-time temporal knowledge graphs that track how facts about users and entities change over time, so agents can answer \"what was true when.\" The layer behind Zep's hosted memory platform.", "category": "memory", "category_title": "Memory and state", "stars": 30212, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (temporal knowledge graph)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "memory", "rag", "workflow", "python" ], "example": { "label": "Temporal graph quickstart", "url": "https://github.com/getzep/graphiti#readme" }, "deep_dive": null }, { "name": "cognee", "github_id": "topoteretes/cognee", "url": "https://github.com/topoteretes/cognee", "slug": "cognee", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#cognee", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/cognee/", "description": "Open-source memory layer for agents: an extract–cognify–load pipeline that turns your data into a queryable knowledge graph plus vector store, so agents recall facts and relationships across sessions instead of re-reading context.", "category": "memory", "category_title": "Memory and state", "stars": 30194, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (graph + vector memory)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "memory", "rag", "workflow", "python" ], "example": { "label": "Quickstart", "url": "https://github.com/topoteretes/cognee#readme" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "none", "rank": 1, "detail": "Pipeline Tasks execute in-process as sequential chained function calls; the pipelines guide documents error propagation (PipelineRunErrored) but no container/VM isolation or restricted execution environment for task code.", "evidence": "https://docs.cognee.ai/guides/custom-tasks-pipelines" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "remember()/improve() persist extracted entities into a self-hosted knowledge graph plus vector store, with session-scoped memory (session_id, fast cache) explicitly designed to sync into the permanent graph in the background — memory survives past the single conversation.", "evidence": "https://raw.githubusercontent.com/topoteretes/cognee/main/README.md" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "The Claude Code integration hooks SessionStart/UserPromptSubmit/PostToolUse/Stop/PreCompact/SessionEnd to capture context and sync memory, but these only observe and inject context after the fact — there is no documented PreToolUse-style hook that can veto a tool call before it runs.", "evidence": "https://docs.cognee.ai/guides/custom-tasks-pipelines" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "cognify()/remember() accept a custom_prompt parameter that lets a user manually override the default entity/relationship-extraction system prompt; the docs confirm this is a manual override with no automatic tuning or iterative refinement.", "evidence": "https://docs.cognee.ai/guides/custom-prompts" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Primary artifact is the self-hosted, pip-installable knowledge-graph memory engine you run yourself (Docker/CLI, Postgres+pgvector option); Cognee Cloud is named as an alternative fully-managed option, not the repo's core deliverable.", "evidence": "https://raw.githubusercontent.com/topoteretes/cognee/main/README.md" } } }, { "name": "beads", "github_id": "gastownhall/beads", "url": "https://github.com/gastownhall/beads", "slug": "beads", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#beads", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/beads/", "description": "Portable persistent-memory layer for coding agents: tracks decisions and task state outside the harness's own context window so it survives session resets and model swaps.", "category": "memory", "category_title": "Memory and state", "stars": 26534, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (portable memory store)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "unknown", "tags": [ "memory" ], "example": { "label": "Project README", "url": "https://github.com/gastownhall/beads#readme" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "none", "rank": 1, "detail": "Beads is a CLI task/memory tracker backed by a Dolt database; no code execution, containerization, or permission-based isolation is documented anywhere in the README or architecture docs.", "evidence": "https://raw.githubusercontent.com/gastownhall/beads/main/README.md" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "bd remember stores project insights and bd prime injects them back into context; all state lives in an embedded or remote Dolt (versioned SQL) database (.beads/embeddeddolt/) that persists across CLI invocations, sessions, and even model/agent swaps rather than living in one conversation's context window.", "evidence": "https://raw.githubusercontent.com/gastownhall/beads/main/README.md" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "bd setup claude installs a SessionStart hook that auto-injects workflow context (and auto-installs the bd CLI) at session start; this and the optional git hooks (bd hooks install, auto-commit on write) are observational/context-injection only, with no documented PreToolUse-style veto of a tool call.", "evidence": "https://github.com/gastownhall/beads/tree/main/integrations/claude-code" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "bd setup codex/claude writes or updates an AGENTS.md/CLAUDE.md-style instruction file that teaches the agent beads commands (bd prime, bd ready, bd claim, bd close) — first-class rules-file layering, but no automatic prompt-tuning loop.", "evidence": "https://raw.githubusercontent.com/gastownhall/beads/main/README.md" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Self-hosted CLI tool (Homebrew/npm/script install) that runs its own Dolt-backed database and multi-agent workflow conventions locally; no hosted/SaaS version is offered — you install and run it yourself.", "evidence": "https://raw.githubusercontent.com/gastownhall/beads/main/README.md" } } }, { "name": "Agent Lightning", "github_id": "microsoft/agent-lightning", "url": "https://github.com/microsoft/agent-lightning", "slug": "agent-lightning", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#agent-lightning", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/agent-lightning/", "description": "Microsoft's training-oriented harness: optimization loops for agent behavior—when you need to improve policies over rollouts, not only score a fixed prompt.", "category": "evaluation", "category_title": "Evaluation and benchmarking harnesses", "stars": 17609, "tier": "complex", "tier_rank": 4, "axis": "complex (agent training, Microsoft stack — product suite)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "evals", "training", "python" ], "example": { "label": "APO room-booking example", "url": "https://github.com/microsoft/agent-lightning/blob/main/examples/apo/README.md" }, "deep_dive": null }, { "name": "SWE-bench", "github_id": "SWE-bench/SWE-bench", "url": "https://github.com/SWE-bench/SWE-bench", "slug": "swe-bench", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#swe-bench", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/swe-bench/", "description": "LMs resolve real GitHub issues; Docker harness, instance IDs; standard for code-agent evals.", "category": "evaluation", "category_title": "Evaluation and benchmarking harnesses", "stars": 5691, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (real GitHub issues, standard)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "evals", "sandbox", "python" ], "example": { "label": "SWE-bench Verified leaderboard", "url": "https://www.swebench.com/verified.html" }, "deep_dive": null }, { "name": "AgentBench", "github_id": "THUDM/AgentBench", "url": "https://github.com/THUDM/AgentBench", "slug": "agentbench", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#agentbench", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/agentbench/", "description": "ICLR'24 benchmark: agents across AlfWorld, DB, knowledge graphs, OS, webshop; Docker Compose, function-calling interface.", "category": "evaluation", "category_title": "Evaluation and benchmarking harnesses", "stars": 3682, "tier": "complex", "tier_rank": 4, "axis": "complex (multi-env, Docker Compose — product suite)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "none", "recovery_rank": 1, "license_signal": "open-source", "tags": [ "evals", "sandbox", "rag", "workflow", "python" ], "example": { "label": "AgentBench ICLR'24 paper", "url": "https://arxiv.org/abs/2308.03688" }, "deep_dive": null }, { "name": "inspect_ai", "github_id": "UKGovernmentBEIS/inspect_ai", "url": "https://github.com/UKGovernmentBEIS/inspect_ai", "slug": "inspect_ai", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#inspect_ai", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/inspect_ai/", "description": "Inspect AI core: composable eval tasks, sandboxes, scorers, and multi-model runs; the framework behind inspect_evals, not just the task bundle.", "category": "evaluation", "category_title": "Evaluation and benchmarking harnesses", "stars": 2606, "tier": "complex", "tier_rank": 4, "axis": "complex (eval framework, AISI stack — product suite)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "evals", "sandbox", "python" ], "example": { "label": "Inspect tutorial example", "url": "https://inspect.aisi.org.uk/tutorial.html" }, "deep_dive": null }, { "name": "WebArena", "github_id": "web-arena-x/webarena", "url": "https://github.com/web-arena-x/webarena", "slug": "webarena", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#webarena", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/webarena/", "description": "Realistic web env (e.g. e‑commerce, CMS, dev tools); 812 tasks; measures end-to-end web agent success.", "category": "evaluation", "category_title": "Evaluation and benchmarking harnesses", "stars": 1584, "tier": "complex", "tier_rank": 4, "axis": "complex (812 tasks, web env — product suite)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "none", "recovery_rank": 1, "license_signal": "open-source", "tags": [ "python" ], "example": { "label": "WebArena leaderboard", "url": "https://docs.google.com/spreadsheets/d/1M801lEpBbKSNwP-vDBkC_pF7LdyGU1f_ufZb_NWNBZQ/edit" }, "deep_dive": null }, { "name": "WebVoyager", "github_id": "MinorJerry/WebVoyager", "url": "https://github.com/MinorJerry/WebVoyager", "slug": "webvoyager", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#webvoyager", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/webvoyager/", "description": "End-to-end web agent with LMMs: screenshots + actions on real sites; benchmark on 15 sites, GPT-4V for automatic eval.", "category": "evaluation", "category_title": "Evaluation and benchmarking harnesses", "stars": 1122, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (LMMs, screenshots, 15 sites)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "none", "recovery_rank": 1, "license_signal": "open-source", "tags": [ "evals", "vision" ], "example": { "label": "643 web tasks dataset", "url": "https://github.com/MinorJerry/WebVoyager/blob/main/data/WebVoyager_data.jsonl" }, "deep_dive": null }, { "name": "agent-qa", "github_id": "vostride/agent-qa", "url": "https://github.com/vostride/agent-qa", "slug": "agent-qa", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#agent-qa", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/agent-qa/", "description": "Self-improving QA **harness** for web and mobile apps: natural-language tests, memory-backed self-healing, dashboard/CLI, MCP and skills support, plus sandboxed hooks for production regression checks.", "category": "evaluation", "category_title": "Evaluation and benchmarking harnesses", "stars": 936, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (web/mobile QA, memory, MCP)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "retry", "recovery_rank": 2, "license_signal": "restricted (FSL-1.1-ALv2)", "tags": [ "mcp", "memory", "sandbox", "cli", "typescript" ], "example": { "label": "Natural-language QA harness", "url": "https://github.com/vostride/agent-qa#readme" }, "deep_dive": null }, { "name": "swe-smith", "github_id": "SWE-bench/SWE-smith", "url": "https://github.com/SWE-bench/SWE-smith", "slug": "swe-smith", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#swe-smith", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/swe-smith/", "description": "Data generation for SWE agents; 50k+ instances across 128 repos; used for SWE-agent-LM training.", "category": "evaluation", "category_title": "Evaluation and benchmarking harnesses", "stars": 748, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (50k+ instances, data gen)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "none", "recovery_rank": 1, "license_signal": "open-source", "tags": [ "training", "python" ], "example": { "label": "SWE-smith trajectories", "url": "https://huggingface.co/datasets/SWE-bench/SWE-smith-trajectories" }, "deep_dive": null }, { "name": "ARC-AGI-2", "github_id": "arcprize/ARC-AGI-2", "url": "https://github.com/arcprize/ARC-AGI-2", "slug": "arc-agi-2", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#arc-agi-2", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/arc-agi-2/", "description": "ARC Prize task set: grid-based abstraction/reasoning; public and private splits for generalization.", "category": "evaluation", "category_title": "Evaluation and benchmarking harnesses", "stars": 734, "tier": "super simple", "tier_rank": 1, "axis": "super simple (task set)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [], "example": { "label": "ARC Prize leaderboard", "url": "https://arcprize.org/leaderboard" }, "deep_dive": null }, { "name": "SWE-Gym", "github_id": "SWE-Gym/SWE-Gym", "url": "https://github.com/SWE-Gym/SWE-Gym", "slug": "swe-gym", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#swe-gym", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/swe-gym/", "description": "Training and evaluation for SWE agents and verifiers (ICML 2025).", "category": "evaluation", "category_title": "Evaluation and benchmarking harnesses", "stars": 723, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (training + eval, ICML)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "none", "recovery_rank": 1, "license_signal": "open-source", "tags": [ "evals", "training", "python" ], "example": { "label": "SWE-Gym ICML 2025 paper", "url": "https://arxiv.org/abs/2412.21139" }, "deep_dive": null }, { "name": "inspect_evals", "github_id": "UKGovernmentBEIS/inspect_evals", "url": "https://github.com/UKGovernmentBEIS/inspect_evals", "slug": "inspect_evals", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#inspect_evals", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/inspect_evals/", "description": "UK AISI/Arcadia/Vector: GAIA and other evals in Inspect AI; level 1–3, sandboxed, tool-calling solvers.", "category": "evaluation", "category_title": "Evaluation and benchmarking harnesses", "stars": 639, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (Inspect AI, UK gov)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "evals", "sandbox" ], "example": { "label": "inspect SWE-bench eval", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/main/src/inspect_evals/swe_bench/README.md" }, "deep_dive": null }, { "name": "Terminal-Bench", "github_id": "harbor-framework/terminal-bench", "url": "https://github.com/harbor-framework/terminal-bench", "slug": "terminal-bench", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#terminal-bench", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/terminal-bench/", "description": "The terminal-task benchmark coding agents now cite next to SWE-bench: hard, containerized terminal tasks scored end to end. Terminal-Bench 2.0 runs on the harbor evaluation framework; the 1.0 tasks live on in the org's terminal-bench-1 repo.", "category": "evaluation", "category_title": "Evaluation and benchmarking harnesses", "stars": 533, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (containerized terminal tasks)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "none", "recovery_rank": 1, "license_signal": "open-source", "tags": [ "evals", "cli", "python" ], "example": { "label": "Terminal-Bench leaderboard", "url": "https://www.tbench.ai/leaderboard" }, "deep_dive": null }, { "name": "arc-agi-benchmarking", "github_id": "arcprize/arc-agi-benchmarking", "url": "https://github.com/arcprize/arc-agi-benchmarking", "slug": "arc-agi-benchmarking", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#arc-agi-benchmarking", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/arc-agi-benchmarking/", "description": "Runner for ARC-AGI: multi-provider (OpenAI, Anthropic, Gemini, etc.), rate limits, retries, and scoring.", "category": "evaluation", "category_title": "Evaluation and benchmarking harnesses", "stars": 363, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (runner, multi-provider)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "retry", "recovery_rank": 2, "license_signal": "open-source", "tags": [ "evals", "provider-agnostic", "python" ], "example": { "label": "o3 prompt example", "url": "https://github.com/arcprize/arc-agi-benchmarking/blob/main/docs/examples/prompt_example_o3.md" }, "deep_dive": null }, { "name": "VitaBench", "github_id": "meituan-longcat/vitabench", "url": "https://github.com/meituan-longcat/vitabench", "slug": "vitabench", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#vitabench", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/vitabench/", "description": "ICLR'26: 66 tools, real-world apps (delivery, travel, retail); 100 cross-scenario + 300 single-scenario tasks; adopted by Qwen/Seed.", "category": "evaluation", "category_title": "Evaluation and benchmarking harnesses", "stars": 164, "tier": "complex", "tier_rank": 4, "axis": "complex (66 tools, cross-scenario — product suite)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "none", "recovery_rank": 1, "license_signal": "open-source", "tags": [], "example": { "label": "VitaBench paper", "url": "https://arxiv.org/abs/2509.26490" }, "deep_dive": null }, { "name": "AgencyBench", "github_id": "GAIR-NLP/AgencyBench", "url": "https://github.com/GAIR-NLP/AgencyBench", "slug": "agencybench", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#agencybench", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/agencybench/", "description": "Long-horizon agent benchmark: 32 scenarios, 138 tasks, ~1M tokens and ~90 tool calls; Docker sandbox and rubric-based + LLM judges.", "category": "evaluation", "category_title": "Evaluation and benchmarking harnesses", "stars": 94, "tier": "complex", "tier_rank": 4, "axis": "complex (32 scenarios, Docker, judges — product suite)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "none", "recovery_rank": 1, "license_signal": "open-source", "tags": [ "evals", "sandbox", "python" ], "example": { "label": "AgencyBench leaderboard", "url": "https://github.com/GAIR-NLP/AgencyBench#leaderboard" }, "deep_dive": null }, { "name": "letta-evals", "github_id": "letta-ai/letta-evals", "url": "https://github.com/letta-ai/letta-evals", "slug": "letta-evals", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#letta-evals", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/letta-evals/", "description": "Eval harness for stateful Letta agents; configurable suites and grading (LLM or rule-based) so you can measure what you ship.", "category": "evaluation", "category_title": "Evaluation and benchmarking harnesses", "stars": 83, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (Letta-specific harness)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "none", "recovery_rank": 1, "license_signal": "open-source", "tags": [ "memory", "python" ], "example": { "label": "LoCoMo memory benchmark", "url": "https://github.com/letta-ai/letta-leaderboard/blob/main/leaderboard/locomo/locomo_benchmark.py" }, "deep_dive": null }, { "name": "SUPER", "github_id": "allenai/super-benchmark", "url": "https://github.com/allenai/super-benchmark", "slug": "super-benchmark", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#super-benchmark", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/super-benchmark/", "description": "Agents that set up and run ML/NLP from GitHub repos; 45 expert problems, 152 masked tasks, 602 AutoGen tasks; Docker-based.", "category": "evaluation", "category_title": "Evaluation and benchmarking harnesses", "stars": 58, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (ML/NLP repos, Docker)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "none", "recovery_rank": 1, "license_signal": "open-source", "tags": [ "sandbox", "python" ], "example": { "label": "SUPER EMNLP paper", "url": "https://arxiv.org/abs/2409.07440" }, "deep_dive": null }, { "name": "TRAIL", "github_id": "patronus-ai/trail-benchmark", "url": "https://github.com/patronus-ai/trail-benchmark", "slug": "trail-benchmark", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#trail-benchmark", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/trail-benchmark/", "description": "Trace reasoning and agentic issue localization; 148 long-context traces, 841 errors, 20+ error types; Hugging Face dataset.", "category": "evaluation", "category_title": "Evaluation and benchmarking harnesses", "stars": 22, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (traces, Hugging Face)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [], "example": { "label": "TRAIL dataset card", "url": "https://huggingface.co/datasets/PatronusAI/TRAIL" }, "deep_dive": null }, { "name": "Langfuse", "github_id": "langfuse/langfuse", "url": "https://github.com/langfuse/langfuse", "slug": "langfuse", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#langfuse", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/langfuse/", "description": "Open-source LLM engineering platform: full-trace observability, online and offline evals, prompt management, and cost metrics for agent runs in production—the monitoring layer most harnesses lack out of the box.", "category": "observability", "category_title": "Observability and eval-ops", "stars": 33575, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (tracing + evals platform)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "evals", "typescript" ], "example": { "label": "Docs", "url": "https://langfuse.com/docs" }, "deep_dive": null }, { "name": "MLflow", "github_id": "mlflow/mlflow", "url": "https://github.com/mlflow/mlflow", "slug": "mlflow", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#mlflow", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/mlflow/", "description": "Mature ML platform now covering GenAI: MLflow Tracing captures every agent step, tool call, and token, with built-in LLM evals and prompt versioning—observability for teams already standardized on MLflow.", "category": "observability", "category_title": "Observability and eval-ops", "stars": 27630, "tier": "complex", "tier_rank": 4, "axis": "complex (full ML + GenAI platform)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "evals", "python" ], "example": { "label": "Docs", "url": "https://mlflow.org" }, "deep_dive": null }, { "name": "Opik", "github_id": "comet-ml/opik", "url": "https://github.com/comet-ml/opik", "slug": "opik", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#opik", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/opik/", "description": "Comet's open-source agent observability and evaluation platform: tracing, scoring, and experiment comparison with the whole core feature set free to self-host under Apache-2.0.", "category": "observability", "category_title": "Observability and eval-ops", "stars": 21549, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (tracing + evals platform)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "evals", "python" ], "example": { "label": "Docs", "url": "https://www.comet.com/docs/opik/" }, "deep_dive": null }, { "name": "Arize Phoenix", "github_id": "Arize-ai/phoenix", "url": "https://github.com/Arize-ai/phoenix", "slug": "phoenix", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#phoenix", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/phoenix/", "description": "Arize's source-available, local-first tracing and eval layer: run it on your laptop or your own infra, and graduate to the managed Arize AX platform only when you need it.", "category": "observability", "category_title": "Observability and eval-ops", "stars": 11148, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (local-first tracing + evals)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "restricted (Elastic-2.0)", "tags": [ "evals", "python" ], "example": { "label": "Docs", "url": "https://arize.com/docs/phoenix" }, "deep_dive": null }, { "name": "DeerFlow", "github_id": "bytedance/deer-flow", "url": "https://github.com/bytedance/deer-flow", "slug": "deer-flow", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#deer-flow", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/deer-flow/", "description": "ByteDance's long-horizon research **harness**: a LangGraph-based agent loop that researches, codes, and creates using sandboxes, memory, tools, and subagents behind a message gateway—an alternative to the reference gpt-researcher stack for multi-hour tasks.", "category": "research-task", "category_title": "Research and task-specific harnesses", "stars": 80648, "tier": "complex", "tier_rank": 4, "axis": "complex (sandboxes, memory, subagents — product suite)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "memory", "multi-agent", "sandbox", "python" ], "example": { "label": "Project README", "url": "https://github.com/bytedance/deer-flow#readme" }, "deep_dive": null }, { "name": "gpt-researcher", "github_id": "assafelovic/gpt-researcher", "url": "https://github.com/assafelovic/gpt-researcher", "slug": "gpt-researcher", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#gpt-researcher", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/gpt-researcher/", "description": "Autonomous deep-research agent: web + local sources, citation-grounded reports, multi-agent and deep-research modes. The reference open-source research harness.", "category": "research-task", "category_title": "Research and task-specific harnesses", "stars": 29099, "tier": "complex", "tier_rank": 4, "axis": "complex (deep research, multi-agent — product suite)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "retry", "recovery_rank": 2, "license_signal": "open-source", "tags": [ "multi-agent", "python" ], "example": { "label": "Multi-agent LangGraph walkthrough", "url": "https://github.com/assafelovic/gpt-researcher/blob/master/docs/blog/2024-05-19-gptr-langgraph/index.md" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "Default scrapers (BeautifulSoup HTTP requests, local Selenium/NoDriver browser instances) run directly on the host with no containerization; isolation only exists by opting into external Tavily Extract or FireCrawl scraping APIs.", "evidence": "https://docs.gptr.dev/docs/gpt-researcher/gptr/scraping" }, "context_memory": { "rating": "basic", "rank": 2, "detail": "Deep Research mode's \"Smart Context Management\" aggregates and synthesizes findings across concurrent research branches in-process during a single run, but no cross-session persistence is documented.", "evidence": "https://docs.gptr.dev/docs/gpt-researcher/gptr/deep_research" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "An include_human_feedback setting lets a human review/revise the Editor agent's research outline before the pipeline proceeds, and a websocket/on_progress callback traces steps, but there is no general user-definable pre-tool-use blocking hook system.", "evidence": "https://docs.gptr.dev/docs/gpt-researcher/multi_agents/langgraph" }, "prompt_optimization": { "rating": "configurable", "rank": 2, "detail": "write_report accepts a custom_prompt for report structure, and PROMPT_FAMILY/AGENT_ROLE settings plus an external config.json let users override built-in prompts, with no automatic tuning loop.", "evidence": "https://docs.gptr.dev/docs/gpt-researcher/gptr/config" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "The primary artifact is the cloned repo's opinionated self-hostable multi-agent research runtime (Docker Compose, frontend, backend server) you configure via retrievers/scrapers/env vars; a lighter pip-installed GPTResearcher library is offered as a secondary integration path.", "evidence": "https://docs.gptr.dev/docs/gpt-researcher/gptr/pip-package" } } }, { "name": "AutoResearchClaw", "github_id": "aiming-lab/AutoResearchClaw", "url": "https://github.com/aiming-lab/AutoResearchClaw", "slug": "autoresearchclaw", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#autoresearchclaw", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/autoresearchclaw/", "description": "Fully autonomous, self-evolving research **harness**: multi-agent debate and citation verification carry a project from idea to a written paper without a human in the loop.", "category": "research-task", "category_title": "Research and task-specific harnesses", "stars": 14071, "tier": "complex", "tier_rank": 4, "axis": "complex (autonomous research, multi-agent debate — product suite)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "unknown", "tags": [ "multi-agent" ], "example": { "label": "Project README", "url": "https://github.com/aiming-lab/AutoResearchClaw#readme" }, "deep_dive": null }, { "name": "MiroThinker", "github_id": "MiroMindAI/MiroThinker", "url": "https://github.com/MiroMindAI/MiroThinker", "slug": "mirothinker", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#mirothinker", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/mirothinker/", "description": "Deep-research **harness** tuned for long browsing-and-reasoning chains; benchmarked on BrowseComp, GAIA, and HLE by pairing a dedicated agent loop with its own MiroThinker models rather than bolting search onto a generic chat agent.", "category": "research-task", "category_title": "Research and task-specific harnesses", "stars": 8359, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (benchmark-tuned research loop)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "retry", "recovery_rank": 2, "license_signal": "unknown", "tags": [ "evals" ], "example": { "label": "Project README", "url": "https://github.com/MiroMindAI/MiroThinker#readme" }, "deep_dive": null }, { "name": "openagents", "github_id": "OpenAgentsInc/openagents", "url": "https://github.com/OpenAgentsInc/openagents", "slug": "openagents", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#openagents", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/openagents/", "description": "Platform for autonomous agents and autopilot-style workflows; decentralized/Nostr-oriented (Pylon runtime, actively shipped in 2026).", "category": "research-task", "category_title": "Research and task-specific harnesses", "stars": 447, "tier": "complex", "tier_rank": 4, "axis": "complex (platform, decentralized — product suite)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [], "example": { "label": "Production earning proof", "url": "https://github.com/OpenAgentsInc/openagents/blob/main/docs/reports/nexus/2026-04-23-autopilot-pylon-production-earning-proof.md" }, "deep_dive": { "researched": "2026-07-19", "tooling_sandboxing": { "rating": "basic", "rank": 2, "detail": "Pylon's composer sessions default to a 'workspace-write' bounded mode with a Claude/Codex tool allowlist and acceptEdits approval; explicit --claude-danger/--codex-danger flags bypass the allowlist entirely (bypassPermissions, approvalPolicy: 'never') rather than the harness enforcing container/VM isolation itself.", "evidence": "https://raw.githubusercontent.com/OpenAgentsInc/openagents/main/apps/pylon/README.md" }, "context_memory": { "rating": "strong", "rank": 3, "detail": "The Agent IDE persists 'durable conversations, repository grants, and explicit work state' with findable history, exact retry reconciliation, and reload/restart recovery, so session context survives process restarts rather than living only in one run.", "evidence": "https://raw.githubusercontent.com/OpenAgentsInc/openagents/main/README.md" }, "lifecycle_hooks": { "rating": "partial", "rank": 2, "detail": "Bounded sessions require acceptEdits human approval before code edits are applied, and the README lists supervision states (stop, steer, queue, question, approval, refusal), but these are inherited from the underlying Claude/Codex CLI's own permission modes rather than a documented user-definable pre/post tool-call hook API covering every call.", "evidence": "https://raw.githubusercontent.com/OpenAgentsInc/openagents/main/apps/pylon/README.md" }, "prompt_optimization": { "rating": "unknown", "rank": 0, "detail": "Checked the top-level README, the effect-native docs, and the Pylon app README for system-prompt customization or automatic prompt-tuning; none is documented.", "evidence": "" }, "build_vs_buy": { "tier": 2, "label": "blueprint", "detail": "Primary artifact is a self-hosted, Node/Bun/Effect monorepo Agent IDE plus Pylon (local account custody and execution) that runs on the operator's machine — the README states useful desktop work needs no OpenAgents account or hosted control plane, with any hosted platform layer being additive.", "evidence": "https://raw.githubusercontent.com/OpenAgentsInc/openagents/main/README.md" } } }, { "name": "Daytona", "github_id": "daytonaio/daytona", "url": "https://github.com/daytonaio/daytona", "slug": "daytona", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#daytona", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/daytona/", "description": "Elastic dev environments for AI-generated code: workspaces, Git, previews—infra harness between \"the model wrote a patch\" and \"it ran in a real machine.\" ⚠️ Public repo unmaintained since June 2026: core development moved to a private codebase (final open release v0.190.0, AGPL-3.0).", "category": "libraries-sdks", "category_title": "Libraries and SDKs", "stars": 71914, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (dev env API, isolation)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "sandbox" ], "example": { "label": "Charts in sandbox", "url": "https://github.com/daytonaio/daytona/tree/main/examples/python/charts" }, "deep_dive": null }, { "name": "LiteLLM", "github_id": "BerriAI/litellm", "url": "https://github.com/BerriAI/litellm", "slug": "litellm", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#litellm", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/litellm/", "description": "One interface to 100+ LLMs; routing, caching, budgets. Not an agent framework—the pipe every agent framework uses.", "category": "libraries-sdks", "category_title": "Libraries and SDKs", "stars": 57066, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (LLM pipe only)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "retry", "recovery_rank": 2, "license_signal": "open-source", "tags": [ "provider-agnostic", "python" ], "example": { "label": "Anthropic Agent SDK gateway", "url": "https://github.com/BerriAI/litellm/blob/main/cookbook/anthropic_agent_sdk/main.py" }, "deep_dive": null }, { "name": "Composio", "github_id": "ComposioHQ/composio", "url": "https://github.com/ComposioHQ/composio", "slug": "composio", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#composio", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/composio/", "description": "1,000+ toolkits with auth, tool search, and a sandboxed workbench—drop-in tool layer so agents stop reinventing OAuth + integrations. Python and TypeScript.", "category": "libraries-sdks", "category_title": "Libraries and SDKs", "stars": 29840, "tier": "complex", "tier_rank": 4, "axis": "complex (1k+ tools, auth, search — product suite)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "sandbox", "tool-discovery", "python", "typescript" ], "example": { "label": "HackerNews agent quickstart", "url": "https://github.com/ComposioHQ/composio#quick-start" }, "deep_dive": null }, { "name": "smolagents", "github_id": "huggingface/smolagents", "url": "https://github.com/huggingface/smolagents", "slug": "smolagents", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#smolagents", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/smolagents/", "description": "Code-as-action agents: model outputs Python executed in sandbox (E2B, Modal, etc.); ~1k LOC core.", "category": "libraries-sdks", "category_title": "Libraries and SDKs", "stars": 28938, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (code-as-action, ~1k LOC)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "none", "recovery_rank": 1, "license_signal": "open-source", "tags": [ "sandbox", "python" ], "example": { "label": "RAG code agent", "url": "https://github.com/huggingface/smolagents/blob/main/examples/rag.py" }, "deep_dive": null }, { "name": "deepagents", "github_id": "langchain-ai/deepagents", "url": "https://github.com/langchain-ai/deepagents", "slug": "deepagents", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#deepagents", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/deepagents/", "description": "LangChain's Python+TypeScript agent harness on top of LangGraph: planning tool, virtual filesystem, shell sandbox, sub-agent spawning—the \"Claude Code-style\" harness as a reusable library.", "category": "libraries-sdks", "category_title": "Libraries and SDKs", "stars": 28172, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (planning, files, sub-agents)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "durable", "recovery_rank": 4, "license_signal": "open-source", "tags": [ "multi-agent", "sandbox", "python", "typescript" ], "example": { "label": "Deep research agent", "url": "https://github.com/langchain-ai/deepagents/tree/main/examples/deep_research" }, "deep_dive": null }, { "name": "vercel/ai", "github_id": "vercel/ai", "url": "https://github.com/vercel/ai", "slug": "ai", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#ai", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/ai/", "description": "React and Node SDK for streaming, tool calls, and agent-style UIs; provider-agnostic.", "category": "libraries-sdks", "category_title": "Libraries and SDKs", "stars": 26368, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (React/Node SDK, provider-agnostic)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "retry", "recovery_rank": 2, "license_signal": "open-source", "tags": [ "provider-agnostic", "typescript" ], "example": { "label": "Next.js agent example", "url": "https://github.com/vercel/ai/tree/main/examples/next-agent" }, "deep_dive": null }, { "name": "pydantic-ai", "github_id": "pydantic/pydantic-ai", "url": "https://github.com/pydantic/pydantic-ai", "slug": "pydantic-ai", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#pydantic-ai", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/pydantic-ai/", "description": "Type-safe Python agents with Pydantic I/O; multi-provider, MCP, Logfire observability, and human-in-the-loop.", "category": "libraries-sdks", "category_title": "Libraries and SDKs", "stars": 19453, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (type-safe, MCP, Logfire)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "durable", "recovery_rank": 4, "license_signal": "open-source", "tags": [ "mcp", "typed", "provider-agnostic", "python" ], "example": { "label": "Bank support agent", "url": "https://github.com/pydantic/pydantic-ai/blob/main/examples/pydantic_ai_examples/bank_support.py" }, "deep_dive": null }, { "name": "E2B", "github_id": "e2b-dev/E2B", "url": "https://github.com/e2b-dev/E2B", "slug": "e2b", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#e2b", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/e2b/", "description": "Firecracker sandboxes for executing agent-generated code; the hosted isolation layer many tool-calling demos use instead of running arbitrary LLM output on your laptop.", "category": "libraries-sdks", "category_title": "Libraries and SDKs", "stars": 13523, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (sandbox API, code execution)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "sandbox", "python" ], "example": { "label": "Claude Code in sandbox", "url": "https://github.com/e2b-dev/e2b-cookbook/tree/main/examples/anthropic-claude-code-in-sandbox-python" }, "deep_dive": null }, { "name": "Steel", "github_id": "steel-dev/steel-browser", "url": "https://github.com/steel-dev/steel-browser", "slug": "steel-browser", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#steel-browser", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/steel-browser/", "description": "Open-source browser API for agents: cloud or self-hosted Chrome sessions with stealth, residential proxies, CAPTCHA solving, and persistent profiles. The only open-source core in the hosted browser-infrastructure lane (Browserbase and Hyperbrowser are closed).", "category": "libraries-sdks", "category_title": "Libraries and SDKs", "stars": 7529, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (browser sessions API, self-hostable)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "memory", "browser", "local" ], "example": { "label": "Sessions API docs", "url": "https://docs.steel.dev" }, "deep_dive": null }, { "name": "strands-agents", "github_id": "strands-agents/harness-sdk", "url": "https://github.com/strands-agents/harness-sdk", "slug": "harness-sdk", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#harness-sdk", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/harness-sdk/", "description": "Model-driven Python SDK; decorators for tools, native MCP, multi-agent; \"minimal code\" without sacrificing provider choice.", "category": "libraries-sdks", "category_title": "Libraries and SDKs", "stars": 6984, "tier": "mostly simple", "tier_rank": 2, "axis": "mostly simple (decorators, MCP, minimal code)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "mcp", "multi-agent", "typed", "python" ], "example": { "label": "First agent tutorial", "url": "https://github.com/strands-agents/samples/tree/main/python/01-learn/01-first-agent" }, "deep_dive": null }, { "name": "Cloudflare Agents", "github_id": "cloudflare/agents", "url": "https://github.com/cloudflare/agents", "slug": "agents-2", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#agents-2", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/agents-2/", "description": "Persistent, stateful agents on Durable Objects: state, websockets, scheduling, and AI chat baked in. The serverless answer to \"where does the agent live?\"", "category": "libraries-sdks", "category_title": "Libraries and SDKs", "stars": 5478, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (Durable Objects, stateful)", "autonomy": "headless", "autonomy_rank": 4, "recovery": "durable", "recovery_rank": 4, "license_signal": "open-source", "tags": [ "memory", "typescript" ], "example": { "label": "SDK playground app", "url": "https://github.com/cloudflare/agents/tree/main/examples/playground" }, "deep_dive": null }, { "name": "openai-agents-js", "github_id": "openai/openai-agents-js", "url": "https://github.com/openai/openai-agents-js", "slug": "openai-agents-js", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#openai-agents-js", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/openai-agents-js/", "description": "Official OpenAI Agents SDK for Node/TS: handoffs, guardrails, voice; the JS counterpart to openai-agents-python.", "category": "libraries-sdks", "category_title": "Libraries and SDKs", "stars": 3685, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (handoffs, guardrails, voice)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "resumable", "recovery_rank": 3, "license_signal": "open-source", "tags": [ "multi-agent", "voice", "typescript" ], "example": { "label": "Financial research agent", "url": "https://github.com/openai/openai-agents-js/tree/main/examples/financial-research-agent" }, "deep_dive": null }, { "name": "Agent Sandbox", "github_id": "kubernetes-sigs/agent-sandbox", "url": "https://github.com/kubernetes-sigs/agent-sandbox", "slug": "agent-sandbox", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#agent-sandbox", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/agent-sandbox/", "description": "Kubernetes-native sandbox primitive for agent runtimes: a Sandbox resource plus warm pools and claims for fast-start, isolated, stateful workloads. The self-hosted answer to hosted sandbox APIs, from the Kubernetes SIGs org.", "category": "libraries-sdks", "category_title": "Libraries and SDKs", "stars": 3591, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (Kubernetes resource, warm pools)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "open-source", "tags": [ "memory", "sandbox", "local" ], "example": { "label": "Sandbox resource quickstart", "url": "https://github.com/kubernetes-sigs/agent-sandbox#readme" }, "deep_dive": null }, { "name": "open-harness", "github_id": "MaxGfeller/open-harness", "url": "https://github.com/MaxGfeller/open-harness", "slug": "open-harness", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#open-harness", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/open-harness/", "description": "TypeScript Agent class on Vercel AI SDK; streaming events, filesystem/bash tools, MCP, and subagent delegation.", "category": "libraries-sdks", "category_title": "Libraries and SDKs", "stars": 599, "tier": "slightly complex", "tier_rank": 3, "axis": "slightly complex (streaming, tools, subagents)", "autonomy": "bounded", "autonomy_rank": 3, "recovery": "none", "recovery_rank": 1, "license_signal": "open-source", "tags": [ "mcp", "multi-agent", "typescript" ], "example": { "label": "Terminal CLI agent", "url": "https://github.com/MaxGfeller/open-harness/tree/main/examples/cli" }, "deep_dive": null }, { "name": "Community-curated agent lists", "github_id": "brandonhimpfen/awesome-ai-agents", "url": "https://github.com/brandonhimpfen/awesome-ai-agents", "slug": "awesome-ai-agents", "anchor_url": "https://github.com/RyanAlberts/best-of-Agent-Harnesses#awesome-ai-agents", "page_url": "https://ryanalberts.github.io/best-of-Agent-Harnesses/h/awesome-ai-agents/", "description": "Broader directories: e.g. [brandonhimpfen/awesome-ai-agents](https://github.com/brandonhimpfen/awesome-ai-agents), [axioma-ai-labs/awesome-ai-agent-frameworks](https://github.com/axioma-ai-labs/awesome-ai-agent-frameworks), [mb-mal/awesome-ai-agents-frameworks](https://github.com/mb-mal/awesome-ai-agents-frameworks)—differ by scope and update cadence.", "category": "libraries-sdks", "category_title": "Libraries and SDKs", "stars": 15, "tier": "super simple", "tier_rank": 1, "axis": "super simple (curated lists)", "autonomy": "n/a", "autonomy_rank": 0, "recovery": "n/a", "recovery_rank": 0, "license_signal": "unknown", "tags": [], "example": { "label": "Frameworks section", "url": "https://github.com/brandonhimpfen/awesome-ai-agents#frameworks" }, "deep_dive": null } ], "graveyard": [ { "github_id": "affaan-m/ECC", "name": "everything-claude-code", "last_stars": 242388, "since": "2026-07-11", "reason": "suspected star manipulation — ~228k stars / ~35k forks on a repo created 2026-01 with no matching install base, dependents, or discussion; fork-to-star ratio and growth curve are inconsistent with organic adoption" }, { "github_id": "FlowiseAI/Flowise", "name": "Flowise", "last_stars": 55385, "since": "2026-08-16", "reason": "archived upstream — kept for citation" }, { "github_id": "spring-ai-community/spring-ai-tool-search-tool", "name": "spring-ai-tool-search-tool", "last_stars": 78, "since": "2026-07-03", "reason": "archived upstream — kept for citation" }, { "github_id": "SeanHogg/BuilderForceAgents", "name": "coderClaw", "last_stars": 3, "since": "2026-07-03", "reason": "archived upstream — kept for citation" } ], "radar": [ { "github_id": "Fosowl/agenticSeek", "stars": 26961, "desc": "Fully Local Manus AI. No APIs, No $200 monthly bills. Enjoy an autonomous agent that thinks, browses the web, and code for the sole cost of electricity.", "via": "weekly discovery" }, { "github_id": "RightNow-AI/openfang", "stars": 18129, "desc": "Open-source Agent Operating System", "via": "weekly discovery" }, { "github_id": "EvoMap/evolver", "stars": 8990, "desc": "The GEP-powered self-evolving engine for AI agents. Auditable evolution with Genes, Capsules, and Events. \\| evomap.ai", "via": "weekly discovery" }, { "github_id": "algorithmicsuperintelligence/openevolve", "stars": 7254, "desc": "Open-source implementation of AlphaEvolve", "via": "weekly discovery" }, { "github_id": "open-multi-agent/open-multi-agent", "stars": 6815, "desc": "TypeScript AI agent orchestration framework with dynamic workflows. Describe the goal, not the graph: a coordinator plans the task DAG at runtime and runs it…", "via": "weekly discovery" }, { "github_id": "crestalnetwork/intentkit", "stars": 6509, "desc": "IntentKit is an open-source, self-hosted cloud agent cluster that manages a collaborative team of AI agents for you.", "via": "weekly discovery" }, { "github_id": "cloudflare/vibesdk", "stars": 5327, "desc": "An open-source vibe coding platform that helps you build your own vibe-coding platform, built entirely on Cloudflare stack", "via": "weekly discovery" }, { "github_id": "agentscope-ai/agentscope-java", "stars": 5229, "desc": "Build distributed, production-grade, long-running agents.", "via": "weekly discovery" }, { "github_id": "FellouAI/eko", "stars": 4951, "desc": "Eko (Eko Keeps Operating) - Build Production-ready Agentic Workflow with Natural Language - eko.fellou.ai", "via": "weekly discovery" }, { "github_id": "evalstate/fast-agent", "stars": 3898, "desc": "Code, Build and Evaluate agents - excellent Model and Skills/MCP/ACP/A2A Support", "via": "weekly discovery" }, { "github_id": "Mirix-AI/MIRIX", "stars": 3435, "desc": "Mirix is a multi-agent personal assistant designed to track on-screen activities and answer user questions intelligently. By capturing real-time visual data…", "via": "weekly discovery" }, { "github_id": "google-antigravity/antigravity-cli", "stars": 2003, "desc": "Antigravity CLI brings the reasoning, execution, and orchestration capabilities of Antigravity agent harness directly into your terminal.", "via": "weekly discovery" }, { "github_id": "rishabhpoddar/teamcopilot", "stars": 14, "desc": "Deploy AI agents for your team to automate business workflows and coding.", "via": "community · PR #21" } ] }