{
  "$comment": "One direct run per model profile over the current case set; reasoning-effort profiles remain separate.",
  "benchmark": "almanbench",
  "version": "v0.1",
  "caseSetSize": 1029,
  "caseSetId": "sha256:9024c3088a8321d7e621d30bee8ccd8401dd4a2655bcdd66f5ad1b448576ebc4",
  "scoringRevision": "33099bd73b37945b424aefdcade0087bb7118d48",
  "evaluated": "2026-09-13",
  "dataset": "https://huggingface.co/datasets/osolmaz/almanbench",
  "results": "https://huggingface.co/datasets/osolmaz/almanbench-results",
  "annotations": {
    "Claude Fable 5 high": "These are single direct runs. Max received 14 content-filtered responses versus 8 for high, accounting for six of its seven additional misses. A max-effort run is not guaranteed to outscore a high-effort run.",
    "Claude Fable 5 max": "These are single direct runs. Max received 14 content-filtered responses versus 8 for high, accounting for six of its seven additional misses. A max-effort run is not guaranteed to outscore a high-effort run."
  },
  "entries": [
    {
      "label": "GPT-5.5 xhigh",
      "vendor": "openai",
      "route": "OpenAI API",
      "reasoning": "xhigh",
      "acceptance": [
        982,
        1029
      ],
      "compliance": [
        1029,
        1029
      ],
      "tiers": {
        "curated": [
          90,
          93
        ],
        "guards": [
          105,
          120
        ],
        "naturalistic": [
          571,
          600
        ],
        "targeted": [
          216,
          216
        ]
      }
    },
    {
      "label": "GPT-5.6 Sol max",
      "vendor": "openai",
      "route": "OpenAI API",
      "reasoning": "max",
      "acceptance": [
        979,
        1029
      ],
      "compliance": [
        1029,
        1029
      ],
      "tiers": {
        "curated": [
          90,
          93
        ],
        "guards": [
          105,
          120
        ],
        "naturalistic": [
          568,
          600
        ],
        "targeted": [
          216,
          216
        ]
      }
    },
    {
      "label": "Claude Opus 5 max",
      "vendor": "anthropic",
      "route": "Anthropic API",
      "reasoning": "max",
      "thinking": "adaptive, max effort",
      "acceptance": [
        977,
        1029
      ],
      "compliance": [
        1029,
        1029
      ],
      "tiers": {
        "curated": [
          92,
          93
        ],
        "guards": [
          106,
          120
        ],
        "naturalistic": [
          572,
          600
        ],
        "targeted": [
          207,
          216
        ]
      }
    },
    {
      "label": "Claude Fable 5 high",
      "vendor": "anthropic",
      "route": "Anthropic API",
      "reasoning": "high",
      "acceptance": [
        967,
        1029
      ],
      "compliance": [
        1028,
        1029
      ],
      "tiers": {
        "curated": [
          89,
          93
        ],
        "guards": [
          105,
          120
        ],
        "naturalistic": [
          561,
          600
        ],
        "targeted": [
          212,
          216
        ]
      }
    },
    {
      "label": "Claude Fable 5 max",
      "vendor": "anthropic",
      "route": "Anthropic API",
      "reasoning": "max",
      "acceptance": [
        966,
        1029
      ],
      "compliance": [
        1028,
        1029
      ],
      "tiers": {
        "curated": [
          90,
          93
        ],
        "guards": [
          107,
          120
        ],
        "naturalistic": [
          561,
          600
        ],
        "targeted": [
          208,
          216
        ]
      },
      "thinking": "adaptive, max effort, summarized"
    },
    {
      "label": "GPT-5.6 Sol xhigh",
      "vendor": "openai",
      "route": "OpenAI API",
      "reasoning": "xhigh",
      "acceptance": [
        962,
        1029
      ],
      "compliance": [
        1027,
        1029
      ],
      "tiers": {
        "curated": [
          89,
          93
        ],
        "guards": [
          105,
          120
        ],
        "naturalistic": [
          553,
          600
        ],
        "targeted": [
          215,
          216
        ]
      }
    },
    {
      "label": "DeepSeek V4 Flash 0731 max",
      "vendor": "deepseek",
      "route": "HF Inference Providers, Novita",
      "reasoning": "max",
      "acceptance": [
        957,
        1029
      ],
      "compliance": [
        1024,
        1029
      ],
      "tiers": {
        "curated": [
          90,
          93
        ],
        "guards": [
          106,
          120
        ],
        "naturalistic": [
          552,
          600
        ],
        "targeted": [
          209,
          216
        ]
      },
      "modelUrl": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731"
    },
    {
      "label": "Kimi K3 max",
      "vendor": "moonshot",
      "route": "HF Inference Providers, Together",
      "reasoning": "max",
      "acceptance": [
        936,
        1029
      ],
      "compliance": [
        1029,
        1029
      ],
      "tiers": {
        "curated": [
          89,
          93
        ],
        "guards": [
          106,
          120
        ],
        "naturalistic": [
          540,
          600
        ],
        "targeted": [
          201,
          216
        ]
      },
      "modelUrl": "https://huggingface.co/moonshotai/Kimi-K3/tree/9f62e4e9fffbd0a83ddd60e1c209d828994b3569"
    },
    {
      "label": "GLM 5.2",
      "vendor": "zai",
      "route": "HF Inference Providers, Novita",
      "reasoning": "default",
      "acceptance": [
        934,
        1029
      ],
      "compliance": [
        1027,
        1029
      ],
      "tiers": {
        "curated": [
          88,
          93
        ],
        "guards": [
          104,
          120
        ],
        "naturalistic": [
          534,
          600
        ],
        "targeted": [
          208,
          216
        ]
      }
    },
    {
      "label": "DeepSeek V4.1 Flash max",
      "vendor": "deepseek",
      "route": "HF Inference Providers, Novita",
      "reasoning": "max",
      "acceptance": [
        931,
        1029
      ],
      "compliance": [
        1026,
        1029
      ],
      "tiers": {
        "curated": [
          87,
          93
        ],
        "guards": [
          105,
          120
        ],
        "naturalistic": [
          540,
          600
        ],
        "targeted": [
          199,
          216
        ]
      },
      "modelUrl": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash"
    },
    {
      "label": "DeepSeek V4 Flash 0422",
      "vendor": "deepseek",
      "route": "HF Inference Providers, Novita",
      "reasoning": "default",
      "acceptance": [
        924,
        1029
      ],
      "compliance": [
        1028,
        1029
      ],
      "tiers": {
        "curated": [
          88,
          93
        ],
        "guards": [
          106,
          120
        ],
        "naturalistic": [
          520,
          600
        ],
        "targeted": [
          210,
          216
        ]
      }
    },
    {
      "label": "Inkling max",
      "vendor": "thinkingmachines",
      "route": "HF Inference Providers, Together",
      "reasoning": "max",
      "acceptance": [
        920,
        1029
      ],
      "compliance": [
        1028,
        1029
      ],
      "tiers": {
        "curated": [
          87,
          93
        ],
        "guards": [
          103,
          120
        ],
        "naturalistic": [
          514,
          600
        ],
        "targeted": [
          216,
          216
        ]
      }
    },
    {
      "label": "GPT-5.6 Luna xhigh",
      "vendor": "openai",
      "route": "OpenAI API",
      "reasoning": "xhigh",
      "acceptance": [
        903,
        1029
      ],
      "compliance": [
        1014,
        1029
      ],
      "tiers": {
        "curated": [
          87,
          93
        ],
        "guards": [
          103,
          120
        ],
        "naturalistic": [
          501,
          600
        ],
        "targeted": [
          212,
          216
        ]
      }
    },
    {
      "label": "GPT-5.6 Terra xhigh",
      "vendor": "openai",
      "route": "OpenAI API",
      "reasoning": "xhigh",
      "acceptance": [
        902,
        1029
      ],
      "compliance": [
        1027,
        1029
      ],
      "tiers": {
        "curated": [
          88,
          93
        ],
        "guards": [
          106,
          120
        ],
        "naturalistic": [
          500,
          600
        ],
        "targeted": [
          208,
          216
        ]
      }
    },
    {
      "label": "Kimi K2.7 Code",
      "vendor": "moonshot",
      "route": "HF Inference Providers, Novita",
      "reasoning": "default",
      "acceptance": [
        895,
        1029
      ],
      "compliance": [
        1023,
        1029
      ],
      "tiers": {
        "curated": [
          87,
          93
        ],
        "guards": [
          105,
          120
        ],
        "naturalistic": [
          498,
          600
        ],
        "targeted": [
          205,
          216
        ]
      }
    },
    {
      "label": "GoePT-1-20M",
      "vendor": "alman",
      "route": "Browser, ONNX Runtime Web, single-threaded WASM",
      "modelUrl": "https://huggingface.co/osolmaz/GoePT-1-20M",
      "reasoning": "none",
      "acceptance": [
        894,
        1029
      ],
      "compliance": [
        1026,
        1029
      ],
      "tiers": {
        "curated": [
          75,
          93
        ],
        "guards": [
          113,
          120
        ],
        "naturalistic": [
          497,
          600
        ],
        "targeted": [
          209,
          216
        ]
      }
    },
    {
      "label": "DeepSeek V4 Pro",
      "vendor": "deepseek",
      "route": "HF Inference Providers, Novita",
      "reasoning": "default",
      "acceptance": [
        878,
        1029
      ],
      "compliance": [
        1028,
        1029
      ],
      "tiers": {
        "curated": [
          84,
          93
        ],
        "guards": [
          105,
          120
        ],
        "naturalistic": [
          478,
          600
        ],
        "targeted": [
          211,
          216
        ]
      }
    },
    {
      "label": "Claude Sonnet 5 xhigh",
      "vendor": "anthropic",
      "route": "Anthropic API",
      "reasoning": "xhigh",
      "acceptance": [
        858,
        1029
      ],
      "compliance": [
        983,
        1029
      ],
      "tiers": {
        "curated": [
          82,
          93
        ],
        "guards": [
          107,
          120
        ],
        "naturalistic": [
          463,
          600
        ],
        "targeted": [
          206,
          216
        ]
      }
    },
    {
      "label": "Qwen3.8 27B medium",
      "vendor": "qwen",
      "route": "HF Inference Endpoint, BF16, 1× A100",
      "reasoning": "medium",
      "acceptance": [
        857,
        1029
      ],
      "compliance": [
        1023,
        1029
      ],
      "tiers": {
        "curated": [
          82,
          93
        ],
        "guards": [
          105,
          120
        ],
        "naturalistic": [
          463,
          600
        ],
        "targeted": [
          207,
          216
        ]
      },
      "modelUrl": "https://huggingface.co/Qwen/Qwen3.8-27B/tree/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0"
    },
    {
      "label": "Qwen3.6 27B",
      "vendor": "qwen",
      "route": "HF Inference Providers, DeepInfra",
      "reasoning": "default",
      "acceptance": [
        842,
        1029
      ],
      "compliance": [
        1023,
        1029
      ],
      "tiers": {
        "curated": [
          81,
          93
        ],
        "guards": [
          104,
          120
        ],
        "naturalistic": [
          447,
          600
        ],
        "targeted": [
          210,
          216
        ]
      },
      "modelUrl": "https://huggingface.co/Qwen/Qwen3.6-27B/tree/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9"
    },
    {
      "label": "MiniMax M3",
      "vendor": "minimax",
      "route": "HF Inference Providers, Novita",
      "reasoning": "default",
      "acceptance": [
        820,
        1029
      ],
      "compliance": [
        1011,
        1029
      ],
      "tiers": {
        "curated": [
          81,
          93
        ],
        "guards": [
          102,
          120
        ],
        "naturalistic": [
          431,
          600
        ],
        "targeted": [
          206,
          216
        ]
      }
    },
    {
      "label": "Gemma 4 26B A4B IT",
      "vendor": "google",
      "route": "HF Inference Providers, Novita",
      "reasoning": "default",
      "acceptance": [
        789,
        1029
      ],
      "compliance": [
        1022,
        1029
      ],
      "tiers": {
        "curated": [
          70,
          93
        ],
        "guards": [
          93,
          120
        ],
        "naturalistic": [
          424,
          600
        ],
        "targeted": [
          202,
          216
        ]
      },
      "modelUrl": "https://huggingface.co/google/gemma-4-26B-A4B-it/tree/4d7ae4984b7db7de8f8457170b3f1a419ee76d52"
    },
    {
      "label": "Claude Opus 4.8 max",
      "vendor": "anthropic",
      "route": "Anthropic API",
      "reasoning": "max",
      "acceptance": [
        770,
        1029
      ],
      "compliance": [
        937,
        1029
      ],
      "tiers": {
        "curated": [
          80,
          93
        ],
        "guards": [
          101,
          120
        ],
        "naturalistic": [
          395,
          600
        ],
        "targeted": [
          194,
          216
        ]
      }
    },
    {
      "label": "Nemotron 3 Ultra",
      "vendor": "nvidia",
      "route": "HF Inference Providers, Together",
      "reasoning": "default",
      "acceptance": [
        769,
        1029
      ],
      "compliance": [
        1014,
        1029
      ],
      "tiers": {
        "curated": [
          80,
          93
        ],
        "guards": [
          93,
          120
        ],
        "naturalistic": [
          397,
          600
        ],
        "targeted": [
          199,
          216
        ]
      }
    },
    {
      "label": "Qwen3.6 35B A3B",
      "vendor": "qwen",
      "route": "HF Inference Providers, Scaleway",
      "reasoning": "default",
      "acceptance": [
        766,
        1029
      ],
      "compliance": [
        999,
        1029
      ],
      "tiers": {
        "curated": [
          79,
          93
        ],
        "guards": [
          101,
          120
        ],
        "naturalistic": [
          386,
          600
        ],
        "targeted": [
          200,
          216
        ]
      }
    },
    {
      "label": "Step 3.5 Flash",
      "vendor": "stepfun",
      "route": "HF Inference Endpoint, FP8, 2× H200",
      "reasoning": "default",
      "acceptance": [
        716,
        1029
      ],
      "compliance": [
        966,
        1029
      ],
      "tiers": {
        "curated": [
          80,
          93
        ],
        "guards": [
          95,
          120
        ],
        "naturalistic": [
          350,
          600
        ],
        "targeted": [
          191,
          216
        ]
      },
      "modelUrl": "https://huggingface.co/stepfun-ai/Step-3.5-Flash-FP8/tree/6eebda59dd87ca5729648ec7cfed0becfceb273e"
    },
    {
      "label": "GPT-OSS 120B high",
      "vendor": "openai",
      "route": "HF Inference Providers, Cerebras",
      "reasoning": "high",
      "acceptance": [
        709,
        1029
      ],
      "compliance": [
        1013,
        1029
      ],
      "tiers": {
        "curated": [
          71,
          93
        ],
        "guards": [
          91,
          120
        ],
        "naturalistic": [
          363,
          600
        ],
        "targeted": [
          184,
          216
        ]
      },
      "modelUrl": "https://huggingface.co/openai/gpt-oss-120b/tree/b5c939de8f754692c1647ca79fbf85e8c1e70f8a"
    },
    {
      "label": "Gemma 4 31B IT",
      "vendor": "google",
      "route": "HF Inference Providers, Novita",
      "reasoning": "default",
      "acceptance": [
        667,
        1029
      ],
      "compliance": [
        976,
        1029
      ],
      "tiers": {
        "curated": [
          67,
          93
        ],
        "guards": [
          98,
          120
        ],
        "naturalistic": [
          302,
          600
        ],
        "targeted": [
          200,
          216
        ]
      }
    },
    {
      "label": "MiMo V2.5 Pro",
      "vendor": "xiaomi",
      "route": "HF Inference Providers, DeepInfra",
      "reasoning": "default",
      "acceptance": [
        661,
        1029
      ],
      "compliance": [
        912,
        1029
      ],
      "tiers": {
        "curated": [
          71,
          93
        ],
        "guards": [
          97,
          120
        ],
        "naturalistic": [
          303,
          600
        ],
        "targeted": [
          190,
          216
        ]
      }
    },
    {
      "label": "Ternary Bonsai 27B",
      "vendor": "prism-ml",
      "route": "HF Inference Providers, Together",
      "reasoning": "default",
      "acceptance": [
        496,
        1029
      ],
      "compliance": [
        930,
        1029
      ],
      "tiers": {
        "curated": [
          45,
          93
        ],
        "guards": [
          79,
          120
        ],
        "naturalistic": [
          209,
          600
        ],
        "targeted": [
          163,
          216
        ]
      }
    },
    {
      "label": "Laguna S 2.1",
      "vendor": "poolside",
      "modelUrl": "https://huggingface.co/poolside/Laguna-S-2.1-FP8",
      "route": "HF Inference Endpoint, FP8, 2× H200",
      "reasoning": "default",
      "acceptance": [
        416,
        1029
      ],
      "compliance": [
        709,
        1029
      ],
      "tiers": {
        "curated": [
          46,
          93
        ],
        "guards": [
          79,
          120
        ],
        "naturalistic": [
          158,
          600
        ],
        "targeted": [
          133,
          216
        ]
      }
    },
    {
      "label": "LongCat Flash Lite Sparse",
      "vendor": "meituan",
      "route": "HF Inference Endpoint reproduction, BF16, 1× H200",
      "reasoning": "default",
      "acceptance": [
        252,
        1029
      ],
      "compliance": [
        516,
        1029
      ],
      "tiers": {
        "curated": [
          30,
          93
        ],
        "guards": [
          56,
          120
        ],
        "naturalistic": [
          71,
          600
        ],
        "targeted": [
          95,
          216
        ]
      },
      "modelUrl": "https://huggingface.co/meituan-longcat/LongCat-Flash-Lite-Sparse/tree/bc076cbcc3a7ee14c813995dc4e1901ba9b9488a"
    }
  ]
}