- model: claude-3-5-haiku average: 70.5 total_cost: 5.84 letta_bench_archival_memory_read_benchmark_100: 100.0 letta_bench_core_memory_read_benchmark_100: 62.0 letta_bench_core_memory_update_benchmark_100: 28.33 letta_bench_core_memory_write_benchmark_hard_100: 91.67 - model: openai-gpt-4.1-mini average: 80.17 total_cost: 1.9 letta_bench_archival_memory_read_benchmark_100: 99.67 letta_bench_core_memory_read_benchmark_100: 99.67 letta_bench_core_memory_update_benchmark_100: 51.33 letta_bench_core_memory_write_benchmark_hard_100: 70.0 - model: gemini-2-5-pro average: 84.92 total_cost: 6.44 letta_bench_archival_memory_read_benchmark_100: 98.67 letta_bench_core_memory_read_benchmark_100: 100.0 letta_bench_core_memory_update_benchmark_100: 46.67 letta_bench_core_memory_write_benchmark_hard_100: 94.33 - model: claude-4-sonnet-extended average: 92.08 total_cost: 18.14 letta_bench_archival_memory_read_benchmark_100: 98.33 letta_bench_core_memory_read_benchmark_100: 97.67 letta_bench_core_memory_update_benchmark_100: 72.33 letta_bench_core_memory_write_benchmark_hard_100: 100.0 - model: openai-gpt-4.1 average: 90.83 total_cost: 9.18 letta_bench_archival_memory_read_benchmark_100: 98.33 letta_bench_core_memory_read_benchmark_100: 100.0 letta_bench_core_memory_update_benchmark_100: 66.0 letta_bench_core_memory_write_benchmark_hard_100: 99.0 - model: openai-gpt-4.1-nano average: 47.75 total_cost: 0.97 letta_bench_archival_memory_read_benchmark_100: 94.33 letta_bench_core_memory_read_benchmark_100: 18.0 letta_bench_core_memory_update_benchmark_100: 0.0 letta_bench_core_memory_write_benchmark_hard_100: 78.67 - model: gemini-2-5-flash-0520 average: 87.58 total_cost: 0.68 letta_bench_archival_memory_read_benchmark_100: 92.33 letta_bench_core_memory_read_benchmark_100: 99.67 letta_bench_core_memory_update_benchmark_100: 61.33 letta_bench_core_memory_write_benchmark_hard_100: 97.0 - model: openai-gpt-4o average: 90.5 total_cost: 10.16 letta_bench_archival_memory_read_benchmark_100: 92.33 letta_bench_core_memory_read_benchmark_100: 100.0 letta_bench_core_memory_update_benchmark_100: 73.0 letta_bench_core_memory_write_benchmark_hard_100: 96.67 - model: gemini-2-5-flash average: 67.92 total_cost: 0.86 letta_bench_archival_memory_read_benchmark_100: 89.0 letta_bench_core_memory_read_benchmark_100: 37.67 letta_bench_core_memory_update_benchmark_100: 49.33 letta_bench_core_memory_write_benchmark_hard_100: 95.67 - model: openai-gpt-4o-mini average: 87.83 total_cost: 0.52 letta_bench_archival_memory_read_benchmark_100: 82.67 letta_bench_core_memory_read_benchmark_100: 99.67 letta_bench_core_memory_update_benchmark_100: 70.67 letta_bench_core_memory_write_benchmark_hard_100: 98.33 - model: claude-4-sonnet average: 91.25 total_cost: 17.3 letta_bench_archival_memory_read_benchmark_100: 81.33 letta_bench_core_memory_read_benchmark_100: 100.0 letta_bench_core_memory_update_benchmark_100: 83.67 letta_bench_core_memory_write_benchmark_hard_100: 100.0 - model: together-llama-4-scout-17b average: 47.0 total_cost: 1.42 letta_bench_archival_memory_read_benchmark_100: 64.33 letta_bench_core_memory_read_benchmark_100: 31.33 letta_bench_core_memory_update_benchmark_100: 13.67 letta_bench_core_memory_write_benchmark_hard_100: 78.67 - model: together-qwen-3-235b average: 62.08 total_cost: 1.08 letta_bench_archival_memory_read_benchmark_100: 52.33 letta_bench_core_memory_read_benchmark_100: 96.0 letta_bench_core_memory_update_benchmark_100: 43.33 letta_bench_core_memory_write_benchmark_hard_100: 56.67 - model: together-llama-4-maverick-17b average: 61.25 total_cost: 1.57 letta_bench_archival_memory_read_benchmark_100: 49.33 letta_bench_core_memory_read_benchmark_100: 92.33 letta_bench_core_memory_update_benchmark_100: 46.67 letta_bench_core_memory_write_benchmark_hard_100: 56.67 - model: claude-3-7-sonnet average: 78.58 total_cost: 17.69 letta_bench_archival_memory_read_benchmark_100: 47.0 letta_bench_core_memory_read_benchmark_100: 99.0 letta_bench_core_memory_update_benchmark_100: 76.33 letta_bench_core_memory_write_benchmark_hard_100: 92.0 - model: together-llama-3-2-3b average: 3.67 total_cost: 0.43 letta_bench_archival_memory_read_benchmark_100: 12.67 letta_bench_core_memory_read_benchmark_100: 0.0 letta_bench_core_memory_update_benchmark_100: 0.0 letta_bench_core_memory_write_benchmark_hard_100: 2.0 - model: together-llama-3-3-70b average: 55.67 total_cost: 4.28 letta_bench_archival_memory_read_benchmark_100: 8.33 letta_bench_core_memory_read_benchmark_100: 97.0 letta_bench_core_memory_update_benchmark_100: 44.33 letta_bench_core_memory_write_benchmark_hard_100: 73.0 - model: openai-o4-mini average: 66.5 total_cost: 4.71 letta_bench_archival_memory_read_benchmark_100: 6.33 letta_bench_core_memory_read_benchmark_100: 100.0 letta_bench_core_memory_update_benchmark_100: 66.33 letta_bench_core_memory_write_benchmark_hard_100: 93.33 - model: together-meta-llama-3-1-8b average: 5.92 total_cost: 3.82 letta_bench_archival_memory_read_benchmark_100: 6.33 letta_bench_core_memory_read_benchmark_100: 5.67 letta_bench_core_memory_update_benchmark_100: 1.33 letta_bench_core_memory_write_benchmark_hard_100: 10.33 - model: together-deepseek-v3 average: 54.08 total_cost: 3.42 letta_bench_archival_memory_read_benchmark_100: 4.67 letta_bench_core_memory_read_benchmark_100: 97.33 letta_bench_core_memory_update_benchmark_100: 50.33 letta_bench_core_memory_write_benchmark_hard_100: 64.0 - model: openai-o3-mini average: 43.92 total_cost: 4.89 letta_bench_archival_memory_read_benchmark_100: 4.33 letta_bench_core_memory_read_benchmark_100: 100.0 letta_bench_core_memory_update_benchmark_100: 16.67 letta_bench_core_memory_write_benchmark_hard_100: 54.67