{ "model": "qwen3.8-27b-unleashed", "tasks": 28, "toolTasks": 22, "conditions": { "classic": { "overall": 96.4, "toolTasks": 100, "noToolTasks": 83.3, "falseToolCalls": 1, "exhausted": 0, "perServer": { "playwright": { "total": 8, "correct": 8, "percent": 100 }, "memory": { "total": 6, "correct": 6, "percent": 100 }, "everything": { "total": 6, "correct": 6, "percent": 100 }, "sequential-thinking": { "total": 2, "correct": 2, "percent": 100 } }, "avgTurns": 1, "avgPromptTokens": 7269, "avgCompletionTokens": 49, "rows": [ { "id": "pw-navigate", "expect": "browser_navigate", "picked": "browser_navigate", "correct": true, "turns": 1, "promptTokens": 7266, "completionTokens": 30 }, { "id": "pw-screenshot", "expect": "browser_take_screenshot", "picked": "browser_take_screenshot", "correct": true, "turns": 1, "promptTokens": 7268, "completionTokens": 29 }, { "id": "pw-resize", "expect": "browser_resize", "picked": "browser_resize", "correct": true, "turns": 1, "promptTokens": 7273, "completionTokens": 44 }, { "id": "pw-back", "expect": "browser_navigate_back", "picked": "browser_navigate_back", "correct": true, "turns": 1, "promptTokens": 7268, "completionTokens": 17 }, { "id": "pw-console", "expect": "browser_console_messages", "picked": "browser_console_messages", "correct": true, "turns": 1, "promptTokens": 7268, "completionTokens": 28 }, { "id": "pw-tabs", "expect": "browser_tabs", "picked": "browser_tabs", "correct": true, "turns": 1, "promptTokens": 7267, "completionTokens": 27 }, { "id": "pw-wait", "expect": "browser_wait_for", "picked": "browser_wait_for", "correct": true, "turns": 1, "promptTokens": 7270, "completionTokens": 28 }, { "id": "pw-close", "expect": "browser_close", "picked": "browser_close", "correct": true, "turns": 1, "promptTokens": 7262, "completionTokens": 15 }, { "id": "mem-create", "expect": "create_entities", "picked": "create_entities", "correct": true, "turns": 1, "promptTokens": 7275, "completionTokens": 44 }, { "id": "mem-read", "expect": "read_graph", "picked": "read_graph", "correct": true, "turns": 1, "promptTokens": 7264, "completionTokens": 14 }, { "id": "mem-search", "expect": "search_nodes", "picked": "search_nodes", "correct": true, "turns": 1, "promptTokens": 7268, "completionTokens": 26 }, { "id": "mem-delete", "expect": "delete_entities", "picked": "delete_entities", "correct": true, "turns": 1, "promptTokens": 7270, "completionTokens": 31 }, { "id": "mem-observe", "expect": "add_observations", "picked": "add_observations", "correct": true, "turns": 1, "promptTokens": 7272, "completionTokens": 44 }, { "id": "mem-relate", "expect": "create_relations", "picked": "create_relations", "correct": true, "turns": 1, "promptTokens": 7279, "completionTokens": 49 }, { "id": "ev-sum", "expect": "get-sum", "picked": "get-sum", "correct": true, "turns": 1, "promptTokens": 7273, "completionTokens": 38 }, { "id": "ev-echo", "expect": "echo", "picked": "echo", "correct": true, "turns": 1, "promptTokens": 7270, "completionTokens": 25 }, { "id": "ev-env", "expect": "get-env", "picked": "get-env", "correct": true, "turns": 1, "promptTokens": 7270, "completionTokens": 14 }, { "id": "ev-gzip", "expect": "gzip-file-as-resource", "picked": "gzip-file-as-resource", "correct": true, "turns": 1, "promptTokens": 7266, "completionTokens": 42 }, { "id": "ev-image", "expect": "get-tiny-image", "picked": "get-tiny-image", "correct": true, "turns": 1, "promptTokens": 7265, "completionTokens": 16 }, { "id": "ev-logging", "expect": "toggle-simulated-logging", "picked": "toggle-simulated-logging", "correct": true, "turns": 1, "promptTokens": 7265, "completionTokens": 17 }, { "id": "st-1", "expect": "sequentialthinking", "picked": "sequentialthinking", "correct": true, "turns": 1, "promptTokens": 7274, "completionTokens": 247 }, { "id": "st-2", "expect": "sequentialthinking", "picked": "sequentialthinking", "correct": true, "turns": 1, "promptTokens": 7273, "completionTokens": 133 }, { "id": "none-capital", "expect": null, "picked": null, "correct": true, "turns": 1, "promptTokens": 7264, "completionTokens": 9 }, { "id": "none-tcp", "expect": null, "picked": null, "correct": true, "turns": 1, "promptTokens": 7269, "completionTokens": 64 }, { "id": "none-haiku", "expect": null, "picked": null, "correct": true, "turns": 1, "promptTokens": 7264, "completionTokens": 25 }, { "id": "none-acronym", "expect": null, "picked": null, "correct": true, "turns": 1, "promptTokens": 7265, "completionTokens": 128 }, { "id": "none-prime", "expect": null, "picked": "sequentialthinking", "correct": false, "turns": 1, "promptTokens": 7270, "completionTokens": 161 }, { "id": "none-translate", "expect": null, "picked": null, "correct": true, "turns": 1, "promptTokens": 7265, "completionTokens": 15 } ], "residentTokens": 6200 }, "hubVagueCatalog": { "overall": 82.1, "toolTasks": 81.8, "noToolTasks": 83.3, "falseToolCalls": 1, "exhausted": 3, "perServer": { "playwright": { "total": 8, "correct": 8, "percent": 100 }, "memory": { "total": 6, "correct": 6, "percent": 100 }, "everything": { "total": 6, "correct": 2, "percent": 33.3 }, "sequential-thinking": { "total": 2, "correct": 2, "percent": 100 } }, "avgTurns": 3, "avgPromptTokens": 2818, "avgCompletionTokens": 160, "rows": [ { "id": "pw-navigate", "expect": "browser_navigate", "picked": "browser_navigate", "server": "playwright", "correct": true, "turns": 2, "promptTokens": 1623, "completionTokens": 124, "trace": [ { "action": "tools", "name": "playwright" }, { "action": "call", "name": "playwright", "tool": "browser_navigate" } ] }, { "id": "pw-screenshot", "expect": "browser_take_screenshot", "picked": "browser_take_screenshot", "server": "playwright", "correct": true, "turns": 2, "promptTokens": 1651, "completionTokens": 118, "trace": [ { "action": "tools", "name": "playwright" }, { "action": "call", "name": "playwright", "tool": "browser_take_screenshot" } ] }, { "id": "pw-resize", "expect": "browser_resize", "picked": "browser_resize", "server": "playwright", "correct": true, "turns": 3, "promptTokens": 3133, "completionTokens": 172, "trace": [ { "action": "tools", "name": "playwright" }, { "action": "tools", "name": "playwright" }, { "action": "call", "name": "playwright", "tool": "browser_resize" } ] }, { "id": "pw-back", "expect": "browser_navigate_back", "picked": "browser_navigate_back", "server": "playwright", "correct": true, "turns": 3, "promptTokens": 3114, "completionTokens": 157, "trace": [ { "action": "tools", "name": "playwright" }, { "action": "tools", "name": "playwright" }, { "action": "call", "name": "playwright", "tool": "browser_navigate_back" } ] }, { "id": "pw-console", "expect": "browser_console_messages", "picked": "browser_console_messages", "server": "playwright", "correct": true, "turns": 2, "promptTokens": 1608, "completionTokens": 116, "trace": [ { "action": "tools", "name": "playwright" }, { "action": "call", "name": "playwright", "tool": "browser_console_messages" } ] }, { "id": "pw-tabs", "expect": "browser_tabs", "picked": "browser_tabs", "server": "playwright", "correct": true, "turns": 2, "promptTokens": 1613, "completionTokens": 120, "trace": [ { "action": "tools", "name": "playwright" }, { "action": "call", "name": "playwright", "tool": "browser_tabs" } ] }, { "id": "pw-wait", "expect": "browser_wait_for", "picked": "browser_wait_for", "server": "playwright", "correct": true, "turns": 2, "promptTokens": 1623, "completionTokens": 124, "trace": [ { "action": "tools", "name": "playwright" }, { "action": "call", "name": "playwright", "tool": "browser_wait_for" } ] }, { "id": "pw-close", "expect": "browser_close", "picked": "browser_close", "server": "playwright", "correct": true, "turns": 2, "promptTokens": 1614, "completionTokens": 115, "trace": [ { "action": "tools", "name": "playwright" }, { "action": "call", "name": "playwright", "tool": "browser_close" } ] }, { "id": "mem-create", "expect": "create_entities", "picked": "create_entities", "server": "memory", "correct": true, "turns": 2, "promptTokens": 1757, "completionTokens": 114, "trace": [ { "action": "tools", "name": "memory" }, { "action": "call", "name": "memory", "tool": "create_entities" } ] }, { "id": "mem-read", "expect": "read_graph", "picked": "read_graph", "server": "memory", "correct": true, "turns": 2, "promptTokens": 1735, "completionTokens": 102, "trace": [ { "action": "tools", "name": "memory" }, { "action": "call", "name": "memory", "tool": "read_graph" } ] }, { "id": "mem-search", "expect": "search_nodes", "picked": "search_nodes", "server": "memory", "correct": true, "turns": 2, "promptTokens": 1611, "completionTokens": 119, "trace": [ { "action": "tools", "name": "memory" }, { "action": "call", "name": "memory", "tool": "search_nodes" } ] }, { "id": "mem-delete", "expect": "delete_entities", "picked": "delete_entities", "server": "memory", "correct": true, "turns": 2, "promptTokens": 1747, "completionTokens": 108, "trace": [ { "action": "tools", "name": "memory" }, { "action": "call", "name": "memory", "tool": "delete_entities" } ] }, { "id": "mem-observe", "expect": "add_observations", "picked": "add_observations", "server": "memory", "correct": true, "turns": 2, "promptTokens": 1751, "completionTokens": 121, "trace": [ { "action": "tools", "name": "memory" }, { "action": "call", "name": "memory", "tool": "add_observations" } ] }, { "id": "mem-relate", "expect": "create_relations", "picked": "create_relations", "server": "memory", "correct": true, "turns": 2, "promptTokens": 1765, "completionTokens": 127, "trace": [ { "action": "tools", "name": "memory" }, { "action": "call", "name": "memory", "tool": "create_relations" } ] }, { "id": "ev-sum", "expect": "get-sum", "picked": "get-sum", "server": "everything", "correct": true, "turns": 8, "promptTokens": 8733, "completionTokens": 347, "trace": [ { "action": "search" }, { "action": "search" }, { "action": "search" }, { "action": "status" }, { "action": "inspect", "name": "everything" }, { "action": "tools", "name": "everything" }, { "action": "tools", "name": "everything" }, { "action": "call", "name": "everything", "tool": "get-sum" } ] }, { "id": "ev-echo", "expect": "echo", "picked": "echo", "server": "everything", "correct": true, "turns": 5, "promptTokens": 4565, "completionTokens": 231, "trace": [ { "action": "search" }, { "action": "search" }, { "action": "search" }, { "action": "tools", "name": "everything" }, { "action": "call", "name": "everything", "tool": "echo" } ] }, { "id": "ev-env", "expect": "get-env", "picked": null, "correct": false, "turns": 6, "promptTokens": 6599, "completionTokens": 591, "trace": [ { "action": "status" }, { "action": "inspect", "name": "playwright" }, { "action": "inspect", "name": "memory" }, { "action": "inspect", "name": "everything" }, { "action": "inspect", "name": "sequential-thinking" } ] }, { "id": "ev-gzip", "expect": "gzip-file-as-resource", "picked": null, "correct": false, "turns": 8, "promptTokens": 7979, "completionTokens": 338, "trace": [ { "action": "search" }, { "action": "search" }, { "action": "search" }, { "action": "search" }, { "action": "search" }, { "action": "search" }, { "action": "search" }, { "action": "tools", "name": "everything" } ], "exhausted": true }, { "id": "ev-image", "expect": "get-tiny-image", "picked": null, "correct": false, "turns": 8, "promptTokens": 7992, "completionTokens": 336, "trace": [ { "action": "search" }, { "action": "search" }, { "action": "search" }, { "action": "search" }, { "action": "search" }, { "action": "search" }, { "action": "search" }, { "action": "search" } ], "exhausted": true }, { "id": "ev-logging", "expect": "toggle-simulated-logging", "picked": null, "correct": false, "turns": 8, "promptTokens": 7948, "completionTokens": 328, "trace": [ { "action": "search" }, { "action": "search" }, { "action": "search" }, { "action": "search" }, { "action": "search" }, { "action": "search" }, { "action": "search" }, { "action": "search" } ], "exhausted": true }, { "id": "st-1", "expect": "sequentialthinking", "picked": "sequentialthinking", "server": "sequential-thinking", "correct": true, "turns": 2, "promptTokens": 1682, "completionTokens": 153, "trace": [ { "action": "tools", "name": "sequential-thinking" }, { "action": "call", "name": "sequential-thinking", "tool": "sequentialthinking" } ] }, { "id": "st-2", "expect": "sequentialthinking", "picked": "sequentialthinking", "server": "sequential-thinking", "correct": true, "turns": 3, "promptTokens": 2577, "completionTokens": 178, "trace": [ { "action": "enable", "name": "sequential-thinking" }, { "action": "tools", "name": "sequential-thinking" }, { "action": "call", "name": "sequential-thinking", "tool": "sequentialthinking" } ] }, { "id": "none-capital", "expect": null, "picked": null, "correct": true, "turns": 1, "promptTokens": 746, "completionTokens": 8, "trace": [] }, { "id": "none-tcp", "expect": null, "picked": null, "correct": true, "turns": 1, "promptTokens": 751, "completionTokens": 82, "trace": [] }, { "id": "none-haiku", "expect": null, "picked": "sample", "server": "everything", "correct": false, "turns": 1, "promptTokens": 746, "completionTokens": 100, "trace": [ { "action": "call", "name": "everything", "tool": "sample" } ] }, { "id": "none-acronym", "expect": null, "picked": null, "correct": true, "turns": 1, "promptTokens": 747, "completionTokens": 28, "trace": [] }, { "id": "none-prime", "expect": null, "picked": null, "correct": true, "turns": 1, "promptTokens": 752, "completionTokens": 2, "trace": [] }, { "id": "none-translate", "expect": null, "picked": null, "correct": true, "turns": 1, "promptTokens": 747, "completionTokens": 15, "trace": [] } ], "catalog": "catalog.json", "strippedList": false, "residentTokens": 422 }, "hubNoList": { "overall": 85.7, "toolTasks": 81.8, "noToolTasks": 100, "falseToolCalls": 0, "exhausted": 2, "perServer": { "playwright": { "total": 8, "correct": 7, "percent": 87.5 }, "memory": { "total": 6, "correct": 6, "percent": 100 }, "everything": { "total": 6, "correct": 5, "percent": 83.3 }, "sequential-thinking": { "total": 2, "correct": 0, "percent": 0 } }, "avgTurns": 3.54, "avgPromptTokens": 3047, "avgCompletionTokens": 180, "rows": [ { "id": "pw-navigate", "expect": "browser_navigate", "picked": "browser_navigate", "server": "playwright", "correct": true, "turns": 4, "promptTokens": 3762, "completionTokens": 191, "trace": [ { "action": "search" }, { "action": "search" }, { "action": "tools", "name": "playwright" }, { "action": "call", "name": "playwright", "tool": "browser_navigate" } ] }, { "id": "pw-screenshot", "expect": "browser_take_screenshot", "picked": "browser_take_screenshot", "server": "playwright", "correct": true, "turns": 4, "promptTokens": 3315, "completionTokens": 199, "trace": [ { "action": "search" }, { "action": "search" }, { "action": "tools", "name": "playwright" }, { "action": "call", "name": "playwright", "tool": "browser_take_screenshot" } ] }, { "id": "pw-resize", "expect": "browser_resize", "picked": "browser_resize", "server": "playwright", "correct": true, "turns": 3, "promptTokens": 2416, "completionTokens": 171, "trace": [ { "action": "search" }, { "action": "tools", "name": "playwright" }, { "action": "call", "name": "playwright", "tool": "browser_resize" } ] }, { "id": "pw-back", "expect": "browser_navigate_back", "picked": "browser_navigate_back", "server": "playwright", "correct": true, "turns": 4, "promptTokens": 3273, "completionTokens": 196, "trace": [ { "action": "search" }, { "action": "search" }, { "action": "tools", "name": "playwright" }, { "action": "call", "name": "playwright", "tool": "browser_navigate_back" } ] }, { "id": "pw-console", "expect": "browser_console_messages", "picked": "browser_console_messages", "server": "playwright", "correct": true, "turns": 3, "promptTokens": 2402, "completionTokens": 156, "trace": [ { "action": "search" }, { "action": "tools", "name": "playwright" }, { "action": "call", "name": "playwright", "tool": "browser_console_messages" } ] }, { "id": "pw-tabs", "expect": "browser_tabs", "picked": "browser_tabs", "server": "playwright", "correct": true, "turns": 3, "promptTokens": 2404, "completionTokens": 159, "trace": [ { "action": "search" }, { "action": "tools", "name": "playwright" }, { "action": "call", "name": "playwright", "tool": "browser_tabs" } ] }, { "id": "pw-wait", "expect": "browser_wait_for", "picked": null, "correct": false, "turns": 8, "promptTokens": 7328, "completionTokens": 348, "trace": [ { "action": "search" }, { "action": "search" }, { "action": "search" }, { "action": "search" }, { "action": "search" }, { "action": "search" }, { "action": "search" }, { "action": "search" } ], "exhausted": true }, { "id": "pw-close", "expect": "browser_close", "picked": "browser_close", "server": "playwright", "correct": true, "turns": 4, "promptTokens": 3261, "completionTokens": 194, "trace": [ { "action": "search" }, { "action": "search" }, { "action": "tools", "name": "playwright" }, { "action": "call", "name": "playwright", "tool": "browser_close" } ] }, { "id": "mem-create", "expect": "create_entities", "picked": "create_entities", "server": "memory", "correct": true, "turns": 4, "promptTokens": 3391, "completionTokens": 194, "trace": [ { "action": "search" }, { "action": "search" }, { "action": "tools", "name": "memory" }, { "action": "call", "name": "memory", "tool": "create_entities" } ] }, { "id": "mem-read", "expect": "read_graph", "picked": "read_graph", "server": "memory", "correct": true, "turns": 3, "promptTokens": 2481, "completionTokens": 141, "trace": [ { "action": "search" }, { "action": "tools", "name": "memory" }, { "action": "call", "name": "memory", "tool": "read_graph" } ] }, { "id": "mem-search", "expect": "search_nodes", "picked": "search_nodes", "server": "memory", "correct": true, "turns": 4, "promptTokens": 3360, "completionTokens": 187, "trace": [ { "action": "search" }, { "action": "search" }, { "action": "tools", "name": "memory" }, { "action": "call", "name": "memory", "tool": "search_nodes" } ] }, { "id": "mem-delete", "expect": "delete_entities", "picked": "delete_entities", "server": "memory", "correct": true, "turns": 4, "promptTokens": 3368, "completionTokens": 187, "trace": [ { "action": "search" }, { "action": "search" }, { "action": "tools", "name": "memory" }, { "action": "call", "name": "memory", "tool": "delete_entities" } ] }, { "id": "mem-observe", "expect": "add_observations", "picked": "add_observations", "server": "memory", "correct": true, "turns": 5, "promptTokens": 4187, "completionTokens": 223, "trace": [ { "action": "search" }, { "action": "search" }, { "action": "status" }, { "action": "tools", "name": "memory" }, { "action": "call", "name": "memory", "tool": "add_observations" } ] }, { "id": "mem-relate", "expect": "create_relations", "picked": "create_relations", "server": "memory", "correct": true, "turns": 4, "promptTokens": 3407, "completionTokens": 200, "trace": [ { "action": "search" }, { "action": "search" }, { "action": "tools", "name": "memory" }, { "action": "call", "name": "memory", "tool": "create_relations" } ] }, { "id": "ev-sum", "expect": "get-sum", "picked": "get-sum", "server": "everything", "correct": true, "turns": 4, "promptTokens": 3799, "completionTokens": 204, "trace": [ { "action": "search" }, { "action": "tools", "name": "everything" }, { "action": "tools", "name": "everything" }, { "action": "call", "name": "everything", "tool": "get-sum" } ] }, { "id": "ev-echo", "expect": "echo", "picked": "echo", "server": "everything", "correct": true, "turns": 3, "promptTokens": 2410, "completionTokens": 154, "trace": [ { "action": "search" }, { "action": "tools", "name": "everything" }, { "action": "call", "name": "everything", "tool": "echo" } ] }, { "id": "ev-env", "expect": "get-env", "picked": "get-env", "server": "everything", "correct": true, "turns": 6, "promptTokens": 6335, "completionTokens": 369, "trace": [ { "action": "status" }, { "action": "inspect", "name": "playwright" }, { "action": "inspect", "name": "memory" }, { "action": "inspect", "name": "everything" }, { "action": "tools", "name": "everything" }, { "action": "call", "name": "everything", "tool": "get-env" } ] }, { "id": "ev-gzip", "expect": "gzip-file-as-resource", "picked": "gzip-file-as-resource", "server": "everything", "correct": true, "turns": 4, "promptTokens": 3314, "completionTokens": 223, "trace": [ { "action": "search" }, { "action": "search" }, { "action": "tools", "name": "everything" }, { "action": "call", "name": "everything", "tool": "gzip-file-as-resource" } ] }, { "id": "ev-image", "expect": "get-tiny-image", "picked": null, "correct": false, "turns": 8, "promptTokens": 7330, "completionTokens": 313, "trace": [ { "action": "search" }, { "action": "search" }, { "action": "search" }, { "action": "search" }, { "action": "status" }, { "action": "search" }, { "action": "search" }, { "action": "tools", "name": "everything" } ], "exhausted": true }, { "id": "ev-logging", "expect": "toggle-simulated-logging", "picked": "toggle-simulated-logging", "server": "everything", "correct": true, "turns": 4, "promptTokens": 3292, "completionTokens": 201, "trace": [ { "action": "search" }, { "action": "search" }, { "action": "tools", "name": "everything" }, { "action": "call", "name": "everything", "tool": "toggle-simulated-logging" } ] }, { "id": "st-1", "expect": "sequentialthinking", "picked": null, "correct": false, "turns": 1, "promptTokens": 662, "completionTokens": 330, "trace": [] }, { "id": "st-2", "expect": "sequentialthinking", "picked": null, "correct": false, "turns": 6, "promptTokens": 5906, "completionTokens": 339, "trace": [ { "action": "search" }, { "action": "search" }, { "action": "inspect", "name": "sequential-thinking" }, { "action": "tools", "name": "sequential-thinking" }, { "action": "enable", "name": "sequential-thinking" } ] }, { "id": "none-capital", "expect": null, "picked": null, "correct": true, "turns": 1, "promptTokens": 652, "completionTokens": 9, "trace": [] }, { "id": "none-tcp", "expect": null, "picked": null, "correct": true, "turns": 1, "promptTokens": 657, "completionTokens": 74, "trace": [] }, { "id": "none-haiku", "expect": null, "picked": null, "correct": true, "turns": 1, "promptTokens": 652, "completionTokens": 25, "trace": [] }, { "id": "none-acronym", "expect": null, "picked": null, "correct": true, "turns": 1, "promptTokens": 653, "completionTokens": 28, "trace": [] }, { "id": "none-prime", "expect": null, "picked": null, "correct": true, "turns": 1, "promptTokens": 658, "completionTokens": 2, "trace": [] }, { "id": "none-translate", "expect": null, "picked": null, "correct": true, "turns": 1, "promptTokens": 653, "completionTokens": 15, "trace": [] } ], "catalog": "catalog-described.json", "strippedList": true, "residentTokens": 328 }, "hub": { "overall": 96.4, "toolTasks": 95.5, "noToolTasks": 100, "falseToolCalls": 0, "exhausted": 0, "perServer": { "playwright": { "total": 8, "correct": 8, "percent": 100 }, "memory": { "total": 6, "correct": 6, "percent": 100 }, "everything": { "total": 6, "correct": 6, "percent": 100 }, "sequential-thinking": { "total": 2, "correct": 1, "percent": 50 } }, "avgTurns": 1.96, "avgPromptTokens": 1873, "avgCompletionTokens": 110, "rows": [ { "id": "pw-navigate", "expect": "browser_navigate", "picked": "browser_navigate", "server": "playwright", "correct": true, "turns": 2, "promptTokens": 1845, "completionTokens": 124, "trace": [ { "action": "tools", "name": "playwright" }, { "action": "call", "name": "playwright", "tool": "browser_navigate" } ] }, { "id": "pw-screenshot", "expect": "browser_take_screenshot", "picked": "browser_take_screenshot", "server": "playwright", "correct": true, "turns": 2, "promptTokens": 1873, "completionTokens": 118, "trace": [ { "action": "tools", "name": "playwright" }, { "action": "call", "name": "playwright", "tool": "browser_take_screenshot" } ] }, { "id": "pw-resize", "expect": "browser_resize", "picked": "browser_resize", "server": "playwright", "correct": true, "turns": 2, "promptTokens": 1839, "completionTokens": 131, "trace": [ { "action": "tools", "name": "playwright" }, { "action": "call", "name": "playwright", "tool": "browser_resize" } ] }, { "id": "pw-back", "expect": "browser_navigate_back", "picked": "browser_navigate_back", "server": "playwright", "correct": true, "turns": 2, "promptTokens": 1837, "completionTokens": 118, "trace": [ { "action": "tools", "name": "playwright" }, { "action": "call", "name": "playwright", "tool": "browser_navigate_back" } ] }, { "id": "pw-console", "expect": "browser_console_messages", "picked": "browser_console_messages", "server": "playwright", "correct": true, "turns": 2, "promptTokens": 1830, "completionTokens": 116, "trace": [ { "action": "tools", "name": "playwright" }, { "action": "call", "name": "playwright", "tool": "browser_console_messages" } ] }, { "id": "pw-tabs", "expect": "browser_tabs", "picked": "browser_tabs", "server": "playwright", "correct": true, "turns": 2, "promptTokens": 1835, "completionTokens": 120, "trace": [ { "action": "tools", "name": "playwright" }, { "action": "call", "name": "playwright", "tool": "browser_tabs" } ] }, { "id": "pw-wait", "expect": "browser_wait_for", "picked": "browser_wait_for", "server": "playwright", "correct": true, "turns": 2, "promptTokens": 1845, "completionTokens": 124, "trace": [ { "action": "tools", "name": "playwright" }, { "action": "call", "name": "playwright", "tool": "browser_wait_for" } ] }, { "id": "pw-close", "expect": "browser_close", "picked": "browser_close", "server": "playwright", "correct": true, "turns": 2, "promptTokens": 1836, "completionTokens": 115, "trace": [ { "action": "tools", "name": "playwright" }, { "action": "call", "name": "playwright", "tool": "browser_close" } ] }, { "id": "mem-create", "expect": "create_entities", "picked": "create_entities", "server": "memory", "correct": true, "turns": 3, "promptTokens": 3087, "completionTokens": 164, "trace": [ { "action": "tools", "name": "memory" }, { "action": "tools", "name": "memory" }, { "action": "call", "name": "memory", "tool": "create_entities" } ] }, { "id": "mem-read", "expect": "read_graph", "picked": "read_graph", "server": "memory", "correct": true, "turns": 2, "promptTokens": 1957, "completionTokens": 102, "trace": [ { "action": "tools", "name": "memory" }, { "action": "call", "name": "memory", "tool": "read_graph" } ] }, { "id": "mem-search", "expect": "search_nodes", "picked": "search_nodes", "server": "memory", "correct": true, "turns": 2, "promptTokens": 1833, "completionTokens": 119, "trace": [ { "action": "tools", "name": "memory" }, { "action": "call", "name": "memory", "tool": "search_nodes" } ] }, { "id": "mem-delete", "expect": "delete_entities", "picked": "delete_entities", "server": "memory", "correct": true, "turns": 2, "promptTokens": 1870, "completionTokens": 119, "trace": [ { "action": "tools", "name": "memory" }, { "action": "call", "name": "memory", "tool": "delete_entities" } ] }, { "id": "mem-observe", "expect": "add_observations", "picked": "add_observations", "server": "memory", "correct": true, "turns": 2, "promptTokens": 1859, "completionTokens": 128, "trace": [ { "action": "tools", "name": "memory" }, { "action": "call", "name": "memory", "tool": "add_observations" } ] }, { "id": "mem-relate", "expect": "create_relations", "picked": "create_relations", "server": "memory", "correct": true, "turns": 2, "promptTokens": 1895, "completionTokens": 137, "trace": [ { "action": "tools", "name": "memory" }, { "action": "call", "name": "memory", "tool": "create_relations" } ] }, { "id": "ev-sum", "expect": "get-sum", "picked": "get-sum", "server": "everything", "correct": true, "turns": 4, "promptTokens": 4619, "completionTokens": 204, "trace": [ { "action": "search" }, { "action": "tools", "name": "everything" }, { "action": "tools", "name": "everything" }, { "action": "call", "name": "everything", "tool": "get-sum" } ] }, { "id": "ev-echo", "expect": "echo", "picked": "echo", "server": "everything", "correct": true, "turns": 1, "promptTokens": 863, "completionTokens": 68, "trace": [ { "action": "call", "name": "everything", "tool": "echo" } ] }, { "id": "ev-env", "expect": "get-env", "picked": "get-env", "server": "everything", "correct": true, "turns": 2, "promptTokens": 1838, "completionTokens": 113, "trace": [ { "action": "tools", "name": "everything" }, { "action": "call", "name": "everything", "tool": "get-env" } ] }, { "id": "ev-gzip", "expect": "gzip-file-as-resource", "picked": "gzip-file-as-resource", "server": "everything", "correct": true, "turns": 3, "promptTokens": 3060, "completionTokens": 161, "trace": [ { "action": "search" }, { "action": "tools", "name": "everything" }, { "action": "call", "name": "everything", "tool": "gzip-file-as-resource" } ] }, { "id": "ev-image", "expect": "get-tiny-image", "picked": "get-tiny-image", "server": "everything", "correct": true, "turns": 3, "promptTokens": 3014, "completionTokens": 153, "trace": [ { "action": "search" }, { "action": "tools", "name": "everything" }, { "action": "call", "name": "everything", "tool": "get-tiny-image" } ] }, { "id": "ev-logging", "expect": "toggle-simulated-logging", "picked": "toggle-simulated-logging", "server": "everything", "correct": true, "turns": 2, "promptTokens": 1833, "completionTokens": 121, "trace": [ { "action": "tools", "name": "everything" }, { "action": "call", "name": "everything", "tool": "toggle-simulated-logging" } ] }, { "id": "st-1", "expect": "sequentialthinking", "picked": "sequentialthinking", "server": "sequential-thinking", "correct": true, "turns": 2, "promptTokens": 1904, "completionTokens": 160, "trace": [ { "action": "tools", "name": "sequential-thinking" }, { "action": "call", "name": "sequential-thinking", "tool": "sequentialthinking" } ] }, { "id": "st-2", "expect": "sequentialthinking", "picked": null, "correct": false, "turns": 3, "promptTokens": 2910, "completionTokens": 179, "trace": [ { "action": "enable", "name": "sequential-thinking" }, { "action": "tools", "name": "sequential-thinking" } ] }, { "id": "none-capital", "expect": null, "picked": null, "correct": true, "turns": 1, "promptTokens": 857, "completionTokens": 8, "trace": [] }, { "id": "none-tcp", "expect": null, "picked": null, "correct": true, "turns": 1, "promptTokens": 862, "completionTokens": 82, "trace": [] }, { "id": "none-haiku", "expect": null, "picked": null, "correct": true, "turns": 1, "promptTokens": 857, "completionTokens": 25, "trace": [] }, { "id": "none-acronym", "expect": null, "picked": null, "correct": true, "turns": 1, "promptTokens": 858, "completionTokens": 28, "trace": [] }, { "id": "none-prime", "expect": null, "picked": null, "correct": true, "turns": 1, "promptTokens": 863, "completionTokens": 3, "trace": [] }, { "id": "none-translate", "expect": null, "picked": null, "correct": true, "turns": 1, "promptTokens": 858, "completionTokens": 28, "trace": [] } ], "catalog": "catalog-described.json", "strippedList": false, "residentTokens": 533 } } }