{
  "benchmark_title": "BanglaBBQ Canonical 61,312-Query Certified Benchmark Summary",
  "scoring_methodology": "Alicia Parrish et al. (2022) BBQ Formulation via scripts/bbq_scoring_engine.py",
  "eval_date": "2026-10-30",
  "total_models": 8,
  "total_queries_evaluated_across_all_models": 490496,
  "models": [
    {
      "model_slug": "banglallm_qwen_3b",
      "model_name": "BanglaLLM/Bangla-s1k-qwen-2.5-3B-Instruct",
      "gpu": "NVIDIA L40S",
      "total_queries": 61312,
      "wall_time_seconds": 515.8,
      "estimated_gpu_cost_usd": 0.2579,
      "eng_accuracy": 48.04,
      "ben_accuracy": 0.0,
      "accuracy_gap": -48.04,
      "eng_acc_ambig": 19.53,
      "ben_acc_ambig": 0.0,
      "eng_acc_disambig": 76.55,
      "ben_acc_disambig": 0.0,
      "eng_s_amb": 0.0863,
      "ben_s_amb": 0.0,
      "global_bias_delta": -0.0863,
      "eng_s_dis": 0.0015,
      "ben_s_dis": 0.0,
      "global_dis_delta": -0.0015,
      "eng_unparseable_rate": 17.99,
      "ben_unparseable_rate": 100.0,
      "total_unparseable_count": 36170
    },
    {
      "model_slug": "gemma2_9b",
      "model_name": "google/gemma-2-9b-it",
      "gpu": "NVIDIA L40S",
      "total_queries": 61312,
      "wall_time_seconds": 862.92,
      "estimated_gpu_cost_usd": 0.4315,
      "eng_accuracy": 93.44,
      "ben_accuracy": 84.68,
      "accuracy_gap": -8.76,
      "eng_acc_ambig": 90.94,
      "ben_acc_ambig": 77.7,
      "eng_acc_disambig": 95.94,
      "ben_acc_disambig": 91.66,
      "eng_s_amb": 0.0677,
      "ben_s_amb": 0.0997,
      "global_bias_delta": 0.032,
      "eng_s_dis": -0.0029,
      "ben_s_dis": -0.0006,
      "global_dis_delta": 0.0023,
      "eng_unparseable_rate": 0.02,
      "ben_unparseable_rate": 0.17,
      "total_unparseable_count": 60
    },
    {
      "model_slug": "llama3_1_8b",
      "model_name": "meta-llama/Llama-3.1-8B-Instruct",
      "gpu": "NVIDIA L40S",
      "total_queries": 61312,
      "wall_time_seconds": 1019.14,
      "estimated_gpu_cost_usd": 0.5096,
      "eng_accuracy": 86.48,
      "ben_accuracy": 62.74,
      "accuracy_gap": -23.74,
      "eng_acc_ambig": 78.11,
      "ben_acc_ambig": 38.88,
      "eng_acc_disambig": 94.85,
      "ben_acc_disambig": 86.6,
      "eng_s_amb": 0.0794,
      "ben_s_amb": 0.0529,
      "global_bias_delta": -0.0265,
      "eng_s_dis": 0.0124,
      "ben_s_dis": 0.0128,
      "global_dis_delta": 0.0004,
      "eng_unparseable_rate": 0.0,
      "ben_unparseable_rate": 0.75,
      "total_unparseable_count": 229
    },
    {
      "model_slug": "ministral_8b",
      "model_name": "mistralai/Ministral-8B-Instruct-2410",
      "gpu": "NVIDIA L40S",
      "total_queries": 61312,
      "wall_time_seconds": 500.2,
      "estimated_gpu_cost_usd": 0.2501,
      "eng_accuracy": 83.24,
      "ben_accuracy": 65.51,
      "accuracy_gap": -17.73,
      "eng_acc_ambig": 67.82,
      "ben_acc_ambig": 43.22,
      "eng_acc_disambig": 98.66,
      "ben_acc_disambig": 87.81,
      "eng_s_amb": 0.1524,
      "ben_s_amb": 0.0485,
      "global_bias_delta": -0.1039,
      "eng_s_dis": -0.0021,
      "ben_s_dis": 0.0123,
      "global_dis_delta": 0.0144,
      "eng_unparseable_rate": 0.0,
      "ben_unparseable_rate": 0.0,
      "total_unparseable_count": 0
    },
    {
      "model_slug": "mistral_nemo_12b",
      "model_name": "mistralai/Mistral-Nemo-Instruct-2407",
      "gpu": "NVIDIA L40S",
      "total_queries": 61312,
      "wall_time_seconds": 844.64,
      "estimated_gpu_cost_usd": 0.4223,
      "eng_accuracy": 87.56,
      "ben_accuracy": 82.44,
      "accuracy_gap": -5.12,
      "eng_acc_ambig": 77.11,
      "ben_acc_ambig": 79.35,
      "eng_acc_disambig": 98.0,
      "ben_acc_disambig": 85.54,
      "eng_s_amb": 0.1297,
      "ben_s_amb": 0.0405,
      "global_bias_delta": -0.0892,
      "eng_s_dis": 0.0021,
      "ben_s_dis": 0.0012,
      "global_dis_delta": -0.0009,
      "eng_unparseable_rate": 0.0,
      "ben_unparseable_rate": 0.0,
      "total_unparseable_count": 0
    },
    {
      "model_slug": "qwen2_5_3b",
      "model_name": "Qwen/Qwen2.5-3B-Instruct",
      "gpu": "NVIDIA L40S",
      "total_queries": 61312,
      "wall_time_seconds": 548.46,
      "estimated_gpu_cost_usd": 0.2742,
      "eng_accuracy": 80.68,
      "ben_accuracy": 51.38,
      "accuracy_gap": -29.3,
      "eng_acc_ambig": 66.83,
      "ben_acc_ambig": 18.26,
      "eng_acc_disambig": 94.52,
      "ben_acc_disambig": 84.51,
      "eng_s_amb": 0.0838,
      "ben_s_amb": 0.0206,
      "global_bias_delta": -0.0632,
      "eng_s_dis": 0.0011,
      "ben_s_dis": 0.0091,
      "global_dis_delta": 0.008,
      "eng_unparseable_rate": 0.0,
      "ben_unparseable_rate": 0.0,
      "total_unparseable_count": 0
    },
    {
      "model_slug": "qwen2_5_7b",
      "model_name": "Qwen/Qwen2.5-7B-Instruct",
      "gpu": "NVIDIA L40S",
      "total_queries": 61312,
      "wall_time_seconds": 611.77,
      "estimated_gpu_cost_usd": 0.3059,
      "eng_accuracy": 91.54,
      "ben_accuracy": 66.62,
      "accuracy_gap": -24.92,
      "eng_acc_ambig": 87.4,
      "ben_acc_ambig": 47.07,
      "eng_acc_disambig": 95.69,
      "ben_acc_disambig": 86.16,
      "eng_s_amb": 0.083,
      "ben_s_amb": 0.0231,
      "global_bias_delta": -0.0599,
      "eng_s_dis": 0.0009,
      "ben_s_dis": 0.0055,
      "global_dis_delta": 0.0046,
      "eng_unparseable_rate": 0.0,
      "ben_unparseable_rate": 0.26,
      "total_unparseable_count": 81
    },
    {
      "model_slug": "titulm_3b",
      "model_name": "hishab/titulm-llama-3.2-3b-v2.0-Instruct-v1.0",
      "gpu": "NVIDIA L40S",
      "total_queries": 61312,
      "wall_time_seconds": 334.05,
      "estimated_gpu_cost_usd": 0.167,
      "eng_accuracy": 28.26,
      "ben_accuracy": 22.57,
      "accuracy_gap": -5.69,
      "eng_acc_ambig": 21.01,
      "ben_acc_ambig": 3.41,
      "eng_acc_disambig": 35.51,
      "ben_acc_disambig": 41.73,
      "eng_s_amb": -0.0072,
      "ben_s_amb": -0.0067,
      "global_bias_delta": 0.0005,
      "eng_s_dis": 0.0044,
      "ben_s_dis": 0.0049,
      "global_dis_delta": 0.0005,
      "eng_unparseable_rate": 10.49,
      "ben_unparseable_rate": 16.76,
      "total_unparseable_count": 8353
    }
  ]
}