{
  "schema_version": "decision-bench-result-v1",
  "benchmark_name": "DecisionBench",
  "benchmark_version": "1.0",
  "dataset_repo": "Hanno-Labs/decision-bench",
  "dataset_revision": "b7c8107e01ecb1aee7c7eaf5caee4a3ba9f59443",
  "task_spec_sha256": "a477586be2d19dc109c9f2479e3c455b76e6311758c05f42b5a0ca15f4c34b44",
  "model": {
    "name": "Mapika/decider-35b-a3b",
    "revision": "d9783c58ad4fbff3a6030c8332d29738ec49d22a",
    "model_type": "decision-model",
    "url": "https://huggingface.co/Mapika/decider-35b-a3b",
    "adapter": "jev-compatible-server-v1",
    "probability_source": "option-slot-logit-softmax",
    "open_weights": true,
    "parameter_count": 34660610688
  },
  "requested_rows": 23900,
  "successful_rows": 18382,
  "unsupported_rows": 0,
  "error_rows": 5518,
  "coverage": 0.7691213389121339,
  "primary_accuracy": 0.5611297071129707,
  "supported_accuracy": 0.7295724077902296,
  "mean_negative_log_likelihood": 0.8213308311369967,
  "expected_calibration_error": 0.044364205985813615,
  "mean_latency_seconds": 3.8200671174657197,
  "views": {
    "candidate_count:10": {
      "rows": 196,
      "accuracy": 0.7908163265306123,
      "mean_negative_log_likelihood": 0.8707006523469906,
      "expected_calibration_error": 0.11842685173594973,
      "mean_latency_seconds": 3.7466086854074834
    },
    "candidate_count:11": {
      "rows": 131,
      "accuracy": 0.7175572519083969,
      "mean_negative_log_likelihood": 1.1316904439079762,
      "expected_calibration_error": 0.1394117827297714,
      "mean_latency_seconds": 3.8295381933820383
    },
    "candidate_count:12": {
      "rows": 94,
      "accuracy": 0.6170212765957447,
      "mean_negative_log_likelihood": 1.4737727628690171,
      "expected_calibration_error": 0.13262334997951367,
      "mean_latency_seconds": 3.8182229727823684
    },
    "candidate_count:13": {
      "rows": 55,
      "accuracy": 0.6363636363636364,
      "mean_negative_log_likelihood": 1.7888240769065806,
      "expected_calibration_error": 0.19019831556414635,
      "mean_latency_seconds": 3.8714997595684095
    },
    "candidate_count:14": {
      "rows": 53,
      "accuracy": 0.5094339622641509,
      "mean_negative_log_likelihood": 2.2257825533158853,
      "expected_calibration_error": 0.3102639991236654,
      "mean_latency_seconds": 3.9083126262181773
    },
    "candidate_count:15": {
      "rows": 47,
      "accuracy": 0.48936170212765956,
      "mean_negative_log_likelihood": 1.7831485784773837,
      "expected_calibration_error": 0.27202376275867046,
      "mean_latency_seconds": 3.8112793912358107
    },
    "candidate_count:16": {
      "rows": 265,
      "accuracy": 0.9169811320754717,
      "mean_negative_log_likelihood": 0.43019382260498573,
      "expected_calibration_error": 0.03917215726520019,
      "mean_latency_seconds": 6.282350300428159
    },
    "candidate_count:17": {
      "rows": 25,
      "accuracy": 0.72,
      "mean_negative_log_likelihood": 1.7489640353387272,
      "expected_calibration_error": 0.24802247053618637,
      "mean_latency_seconds": 4.161734578087926
    },
    "candidate_count:18": {
      "rows": 14,
      "accuracy": 0.6428571428571429,
      "mean_negative_log_likelihood": 2.0149934986925047,
      "expected_calibration_error": 0.27440830069069166,
      "mean_latency_seconds": 4.250699528892126
    },
    "candidate_count:19": {
      "rows": 15,
      "accuracy": 0.4666666666666667,
      "mean_negative_log_likelihood": 2.4956608178506006,
      "expected_calibration_error": 0.4898960223632719,
      "mean_latency_seconds": 3.922792860989769
    },
    "candidate_count:2": {
      "rows": 6604,
      "accuracy": 0.8623561477892187,
      "mean_negative_log_likelihood": 0.3312170915717412,
      "expected_calibration_error": 0.06171282923282132,
      "mean_latency_seconds": 3.412702792524098
    },
    "candidate_count:20": {
      "rows": 22,
      "accuracy": 0.5454545454545454,
      "mean_negative_log_likelihood": 2.6742566421089835,
      "expected_calibration_error": 0.3604791021379095,
      "mean_latency_seconds": 4.0982496692714365
    },
    "candidate_count:21": {
      "rows": 16,
      "accuracy": 0.75,
      "mean_negative_log_likelihood": 1.25738877485923,
      "expected_calibration_error": 0.21211654972807423,
      "mean_latency_seconds": 4.065130981151015
    },
    "candidate_count:22": {
      "rows": 8,
      "accuracy": 0.75,
      "mean_negative_log_likelihood": 1.6082718871219255,
      "expected_calibration_error": 0.3590795724149934,
      "mean_latency_seconds": 4.412038574228063
    },
    "candidate_count:23": {
      "rows": 13,
      "accuracy": 0.5384615384615384,
      "mean_negative_log_likelihood": 2.0445011587203803,
      "expected_calibration_error": 0.26553042714105823,
      "mean_latency_seconds": 4.2313625030219555
    },
    "candidate_count:24": {
      "rows": 11,
      "accuracy": 0.45454545454545453,
      "mean_negative_log_likelihood": 2.6270164745563274,
      "expected_calibration_error": 0.40146292526108335,
      "mean_latency_seconds": 4.303667006167498
    },
    "candidate_count:25": {
      "rows": 5,
      "accuracy": 1.0,
      "mean_negative_log_likelihood": 0.3492657581591178,
      "expected_calibration_error": 0.2366363183753723,
      "mean_latency_seconds": 4.580677562206984
    },
    "candidate_count:26": {
      "rows": 10,
      "accuracy": 0.6,
      "mean_negative_log_likelihood": 2.83337699582335,
      "expected_calibration_error": 0.2848205887486192,
      "mean_latency_seconds": 4.321811843290925
    },
    "candidate_count:27": {
      "rows": 5,
      "accuracy": 0.6,
      "mean_negative_log_likelihood": 2.151872270667959,
      "expected_calibration_error": 0.3652779172588916,
      "mean_latency_seconds": 4.595372003689408
    },
    "candidate_count:28": {
      "rows": 6,
      "accuracy": 0.3333333333333333,
      "mean_negative_log_likelihood": 4.242896189278034,
      "expected_calibration_error": 0.6493857274707516,
      "mean_latency_seconds": 4.691456417553127
    },
    "candidate_count:29": {
      "rows": 4,
      "accuracy": 0.75,
      "mean_negative_log_likelihood": 2.1741394208940976,
      "expected_calibration_error": 0.39870479334150416,
      "mean_latency_seconds": 4.211814020294696
    },
    "candidate_count:3": {
      "rows": 614,
      "accuracy": 0.749185667752443,
      "mean_negative_log_likelihood": 0.745432132126771,
      "expected_calibration_error": 0.1242650146711526,
      "mean_latency_seconds": 3.258104536175752
    },
    "candidate_count:30": {
      "rows": 6,
      "accuracy": 0.8333333333333334,
      "mean_negative_log_likelihood": 0.6997028834899529,
      "expected_calibration_error": 0.19972948885369937,
      "mean_latency_seconds": 4.221733628151317
    },
    "candidate_count:31": {
      "rows": 3,
      "accuracy": 0.6666666666666666,
      "mean_negative_log_likelihood": 2.2788205710732394,
      "expected_calibration_error": 0.40101112780580705,
      "mean_latency_seconds": 4.309758357703686
    },
    "candidate_count:32": {
      "rows": 225,
      "accuracy": 0.9955555555555555,
      "mean_negative_log_likelihood": 0.06894612364248764,
      "expected_calibration_error": 0.054573455076300216,
      "mean_latency_seconds": 7.186713502605756
    },
    "candidate_count:33": {
      "rows": 2,
      "accuracy": 0.5,
      "mean_negative_log_likelihood": 1.9133408479590726,
      "expected_calibration_error": 0.3315551617802823,
      "mean_latency_seconds": 4.1564613012596965
    },
    "candidate_count:34": {
      "rows": 3,
      "accuracy": 0.6666666666666666,
      "mean_negative_log_likelihood": 0.6671581784619819,
      "expected_calibration_error": 0.13817058680397812,
      "mean_latency_seconds": 4.8699061423540115
    },
    "candidate_count:35": {
      "rows": 2,
      "accuracy": 0.0,
      "mean_negative_log_likelihood": 5.6162188551776335,
      "expected_calibration_error": 0.9203318941959807,
      "mean_latency_seconds": 5.092409271746874
    },
    "candidate_count:36": {
      "rows": 5,
      "accuracy": 0.6,
      "mean_negative_log_likelihood": 1.211848738050242,
      "expected_calibration_error": 0.1187360921358633,
      "mean_latency_seconds": 4.615180739760399
    },
    "candidate_count:37": {
      "rows": 4,
      "accuracy": 0.25,
      "mean_negative_log_likelihood": 2.927069772486699,
      "expected_calibration_error": 0.5609861092266138,
      "mean_latency_seconds": 4.908612041268498
    },
    "candidate_count:38": {
      "rows": 6,
      "accuracy": 0.3333333333333333,
      "mean_negative_log_likelihood": 2.789998534504941,
      "expected_calibration_error": 0.34747560857231335,
      "mean_latency_seconds": 4.82584367475162
    },
    "candidate_count:39": {
      "rows": 2,
      "accuracy": 0.0,
      "mean_negative_log_likelihood": 9.269976451286949,
      "expected_calibration_error": 0.8687055745945251,
      "mean_latency_seconds": 4.911202035844326
    },
    "candidate_count:4": {
      "rows": 4558,
      "accuracy": 0.5157964019306713,
      "mean_negative_log_likelihood": 1.487351878647968,
      "expected_calibration_error": 0.18682754310025898,
      "mean_latency_seconds": 3.782695394775578
    },
    "candidate_count:40": {
      "rows": 5,
      "accuracy": 0.4,
      "mean_negative_log_likelihood": 4.453147450349973,
      "expected_calibration_error": 0.42073340031164586,
      "mean_latency_seconds": 5.003193988651037
    },
    "candidate_count:41": {
      "rows": 5,
      "accuracy": 0.6,
      "mean_negative_log_likelihood": 2.8056409142441954,
      "expected_calibration_error": 0.5488785277279029,
      "mean_latency_seconds": 4.960461262241006
    },
    "candidate_count:42": {
      "rows": 2,
      "accuracy": 0.0,
      "mean_negative_log_likelihood": 4.258604270508472,
      "expected_calibration_error": 0.666277990247724,
      "mean_latency_seconds": 4.97329770680517
    },
    "candidate_count:43": {
      "rows": 2,
      "accuracy": 1.0,
      "mean_negative_log_likelihood": 0.4210135318573503,
      "expected_calibration_error": 0.2926587297301128,
      "mean_latency_seconds": 4.875527907162905
    },
    "candidate_count:44": {
      "rows": 2,
      "accuracy": 0.0,
      "mean_negative_log_likelihood": 2.113427358243451,
      "expected_calibration_error": 0.6966918123867987,
      "mean_latency_seconds": 5.057758972980082
    },
    "candidate_count:45": {
      "rows": 6,
      "accuracy": 0.5,
      "mean_negative_log_likelihood": 0.9993621594448686,
      "expected_calibration_error": 0.32012406530874393,
      "mean_latency_seconds": 5.184375576364498
    },
    "candidate_count:46": {
      "rows": 2,
      "accuracy": 1.0,
      "mean_negative_log_likelihood": 0.44421241719560756,
      "expected_calibration_error": 0.32836230437018277,
      "mean_latency_seconds": 4.613212333060801
    },
    "candidate_count:47": {
      "rows": 3,
      "accuracy": 1.0,
      "mean_negative_log_likelihood": 0.2548378224597643,
      "expected_calibration_error": 0.22011748199741824,
      "mean_latency_seconds": 5.226499442011118
    },
    "candidate_count:48": {
      "rows": 11,
      "accuracy": 0.2727272727272727,
      "mean_negative_log_likelihood": 4.673043270625599,
      "expected_calibration_error": 0.45004037787538803,
      "mean_latency_seconds": 4.878420432352207
    },
    "candidate_count:49": {
      "rows": 4,
      "accuracy": 0.5,
      "mean_negative_log_likelihood": 1.205737130061514,
      "expected_calibration_error": 0.18268085445164015,
      "mean_latency_seconds": 5.185674466658384
    },
    "candidate_count:5": {
      "rows": 724,
      "accuracy": 0.8439226519337016,
      "mean_negative_log_likelihood": 0.4823459812715142,
      "expected_calibration_error": 0.05505332098799403,
      "mean_latency_seconds": 4.306698669517382
    },
    "candidate_count:50": {
      "rows": 3,
      "accuracy": 0.3333333333333333,
      "mean_negative_log_likelihood": 1.6289412807199877,
      "expected_calibration_error": 0.33823333053027527,
      "mean_latency_seconds": 4.70363893546164
    },
    "candidate_count:51": {
      "rows": 3,
      "accuracy": 0.0,
      "mean_negative_log_likelihood": 4.740861315768906,
      "expected_calibration_error": 0.586756895612339,
      "mean_latency_seconds": 4.727243279417356
    },
    "candidate_count:52": {
      "rows": 2,
      "accuracy": 0.0,
      "mean_negative_log_likelihood": 4.730110594942598,
      "expected_calibration_error": 0.7088589563252515,
      "mean_latency_seconds": 4.344765578396618
    },
    "candidate_count:53": {
      "rows": 13,
      "accuracy": 0.46153846153846156,
      "mean_negative_log_likelihood": 3.5400624143454595,
      "expected_calibration_error": 0.4849375364119711,
      "mean_latency_seconds": 5.650267466186331
    },
    "candidate_count:54": {
      "rows": 4,
      "accuracy": 0.25,
      "mean_negative_log_likelihood": 5.897408799013645,
      "expected_calibration_error": 0.443260016029575,
      "mean_latency_seconds": 4.804543401580304
    },
    "candidate_count:55": {
      "rows": 6,
      "accuracy": 0.3333333333333333,
      "mean_negative_log_likelihood": 4.413119992685326,
      "expected_calibration_error": 0.4014256355107081,
      "mean_latency_seconds": 5.10850506896774
    },
    "candidate_count:56": {
      "rows": 9,
      "accuracy": 0.4444444444444444,
      "mean_negative_log_likelihood": 4.396309211326352,
      "expected_calibration_error": 0.6150775815131284,
      "mean_latency_seconds": 5.045243215022816
    },
    "candidate_count:57": {
      "rows": 4,
      "accuracy": 0.5,
      "mean_negative_log_likelihood": 2.222650293588862,
      "expected_calibration_error": 0.5441981043005059,
      "mean_latency_seconds": 5.09399897698313
    },
    "candidate_count:58": {
      "rows": 3,
      "accuracy": 0.0,
      "mean_negative_log_likelihood": 8.326382058959892,
      "expected_calibration_error": 0.8548776510780949,
      "mean_latency_seconds": 4.769760572041075
    },
    "candidate_count:59": {
      "rows": 3,
      "accuracy": 0.0,
      "mean_negative_log_likelihood": 8.00044602714391,
      "expected_calibration_error": 0.8672174719867525,
      "mean_latency_seconds": 5.238076192016403
    },
    "candidate_count:6": {
      "rows": 618,
      "accuracy": 0.8980582524271845,
      "mean_negative_log_likelihood": 0.3598170642850692,
      "expected_calibration_error": 0.04803131400867417,
      "mean_latency_seconds": 4.116748208788896
    },
    "candidate_count:60": {
      "rows": 9,
      "accuracy": 0.3333333333333333,
      "mean_negative_log_likelihood": 3.975349242037744,
      "expected_calibration_error": 0.3533095244872255,
      "mean_latency_seconds": 4.889956476373805
    },
    "candidate_count:61": {
      "rows": 2,
      "accuracy": 0.5,
      "mean_negative_log_likelihood": 3.68742409428816,
      "expected_calibration_error": 0.6040594955290393,
      "mean_latency_seconds": 5.003712806850672
    },
    "candidate_count:62": {
      "rows": 1,
      "accuracy": 1.0,
      "mean_negative_log_likelihood": 0.024699489804974287,
      "expected_calibration_error": 0.024396953357029072,
      "mean_latency_seconds": 4.894899610430002
    },
    "candidate_count:63": {
      "rows": 6,
      "accuracy": 0.8333333333333334,
      "mean_negative_log_likelihood": 1.8215084477058279,
      "expected_calibration_error": 0.44610877291984563,
      "mean_latency_seconds": 5.135954747286935
    },
    "candidate_count:64": {
      "rows": 223,
      "accuracy": 0.9910313901345291,
      "mean_negative_log_likelihood": 0.13632306439672107,
      "expected_calibration_error": 0.0887944902389402,
      "mean_latency_seconds": 8.44400437115253
    },
    "candidate_count:7": {
      "rows": 520,
      "accuracy": 0.8730769230769231,
      "mean_negative_log_likelihood": 0.46964709636120294,
      "expected_calibration_error": 0.048500427907145205,
      "mean_latency_seconds": 4.237463441075614
    },
    "candidate_count:8": {
      "rows": 659,
      "accuracy": 0.9119878603945372,
      "mean_negative_log_likelihood": 0.3809312143959437,
      "expected_calibration_error": 0.03818434397670596,
      "mean_latency_seconds": 4.698604852475554
    },
    "candidate_count:9": {
      "rows": 2499,
      "accuracy": 0.579031612645058,
      "mean_negative_log_likelihood": 1.1549378254972376,
      "expected_calibration_error": 0.022751096964877898,
      "mean_latency_seconds": 3.5061517134182103
    },
    "domain:agent_control": {
      "rows": 500,
      "accuracy": 0.694,
      "mean_negative_log_likelihood": 1.3575960205432471,
      "expected_calibration_error": 0.18060128046488283,
      "mean_latency_seconds": 5.662798523083329
    },
    "domain:browser_automation": {
      "rows": 200,
      "accuracy": 0.64,
      "mean_negative_log_likelihood": 1.996171374022716,
      "expected_calibration_error": 0.22589070404483874,
      "mean_latency_seconds": 4.134020175514743
    },
    "domain:code_search": {
      "rows": 200,
      "accuracy": 0.685,
      "mean_negative_log_likelihood": 1.3230041661663454,
      "expected_calibration_error": 0.14286626742745714,
      "mean_latency_seconds": 3.9761996912397444
    },
    "domain:coding_agents": {
      "rows": 300,
      "accuracy": 0.9133333333333333,
      "mean_negative_log_likelihood": 0.2673667523266902,
      "expected_calibration_error": 0.03157083146570982,
      "mean_latency_seconds": 3.2191089891890683
    },
    "domain:databases": {
      "rows": 100,
      "accuracy": 0.49,
      "mean_negative_log_likelihood": 3.287670489418292,
      "expected_calibration_error": 0.40579108448235435,
      "mean_latency_seconds": 3.887372560761869
    },
    "domain:ecommerce": {
      "rows": 2222,
      "accuracy": 0.5067506750675067,
      "mean_negative_log_likelihood": 1.1681927351649195,
      "expected_calibration_error": 0.11397352232607402,
      "mean_latency_seconds": 3.650652985907521
    },
    "domain:evidence_reasoning": {
      "rows": 100,
      "accuracy": 0.71,
      "mean_negative_log_likelihood": 0.8668135798996027,
      "expected_calibration_error": 0.21882203172315717,
      "mean_latency_seconds": 3.130658438038081
    },
    "domain:finance": {
      "rows": 500,
      "accuracy": 0.484,
      "mean_negative_log_likelihood": 0.9650519294166274,
      "expected_calibration_error": 0.09390902702663179,
      "mean_latency_seconds": 3.5107733233347536
    },
    "domain:games": {
      "rows": 296,
      "accuracy": 0.5472972972972973,
      "mean_negative_log_likelihood": 1.5332412814100909,
      "expected_calibration_error": 0.16042599596081616,
      "mean_latency_seconds": 3.5239679991531916
    },
    "domain:home_automation": {
      "rows": 97,
      "accuracy": 0.9278350515463918,
      "mean_negative_log_likelihood": 0.3212363760884467,
      "expected_calibration_error": 0.06374956808021356,
      "mean_latency_seconds": 3.2819226794712937
    },
    "domain:knowledge_graphs": {
      "rows": 100,
      "accuracy": 0.69,
      "mean_negative_log_likelihood": 1.4235450137614085,
      "expected_calibration_error": 0.12287484718260205,
      "mean_latency_seconds": 3.878303200379014
    },
    "domain:legal": {
      "rows": 2222,
      "accuracy": 0.7898289828982898,
      "mean_negative_log_likelihood": 0.49927247287873583,
      "expected_calibration_error": 0.12390259842963341,
      "mean_latency_seconds": 3.180742418712875
    },
    "domain:logic": {
      "rows": 200,
      "accuracy": 0.7,
      "mean_negative_log_likelihood": 0.7480773274650084,
      "expected_calibration_error": 0.0946618418631766,
      "mean_latency_seconds": 3.07926942858845
    },
    "domain:multi_hop_search": {
      "rows": 600,
      "accuracy": 0.6766666666666666,
      "mean_negative_log_likelihood": 0.6208961486397296,
      "expected_calibration_error": 0.06073259121225476,
      "mean_latency_seconds": 6.938704840680584
    },
    "domain:online_safety": {
      "rows": 2222,
      "accuracy": 0.6305130513051305,
      "mean_negative_log_likelihood": 1.6214897979665268,
      "expected_calibration_error": 0.2593494562231875,
      "mean_latency_seconds": 2.918743619096778
    },
    "domain:privacy": {
      "rows": 2221,
      "accuracy": 0.9518235029266097,
      "mean_negative_log_likelihood": 0.17522817217551664,
      "expected_calibration_error": 0.016256765557096778,
      "mean_latency_seconds": 3.7960328164156936
    },
    "domain:public_sector": {
      "rows": 620,
      "accuracy": 1.0,
      "mean_negative_log_likelihood": 0.08504661225525141,
      "expected_calibration_error": 0.07114453816860773,
      "mean_latency_seconds": 9.45890039720182
    },
    "domain:robotics": {
      "rows": 200,
      "accuracy": 0.71,
      "mean_negative_log_likelihood": 1.0596302466283332,
      "expected_calibration_error": 0.10223123414971588,
      "mean_latency_seconds": 3.5991071532852947
    },
    "domain:search": {
      "rows": 100,
      "accuracy": 0.91,
      "mean_negative_log_likelihood": 0.4386649082534544,
      "expected_calibration_error": 0.08325092868688802,
      "mean_latency_seconds": 3.8098051400668917
    },
    "domain:social_media": {
      "rows": 100,
      "accuracy": 0.8,
      "mean_negative_log_likelihood": 1.0583038040252282,
      "expected_calibration_error": 0.12917604923718687,
      "mean_latency_seconds": 3.5517714948765935
    },
    "domain:software_agents": {
      "rows": 637,
      "accuracy": 0.9984301412872841,
      "mean_negative_log_likelihood": 0.008730643320973093,
      "expected_calibration_error": 0.006447168203896263,
      "mean_latency_seconds": 5.020942358130493
    },
    "domain:software_engineering": {
      "rows": 98,
      "accuracy": 0.4897959183673469,
      "mean_negative_log_likelihood": 1.1541571466056402,
      "expected_calibration_error": 0.21526819619940313,
      "mean_latency_seconds": 3.1251419772755127
    },
    "domain:tax_documents": {
      "rows": 2,
      "accuracy": 1.0,
      "mean_negative_log_likelihood": 0.19273335365216956,
      "expected_calibration_error": 0.1697203101636348,
      "mean_latency_seconds": 3.351803509518504
    },
    "domain:technical": {
      "rows": 2223,
      "accuracy": 0.5434098065677013,
      "mean_negative_log_likelihood": 1.2303203168892716,
      "expected_calibration_error": 0.020919832491952745,
      "mean_latency_seconds": 3.4690699621857393
    },
    "domain:web": {
      "rows": 2222,
      "accuracy": 0.8879387938793879,
      "mean_negative_log_likelihood": 0.2966083363051419,
      "expected_calibration_error": 0.057348545294109404,
      "mean_latency_seconds": 3.099703077122931
    },
    "domain:web_content": {
      "rows": 100,
      "accuracy": 1.0,
      "mean_negative_log_likelihood": 0.026671299885487665,
      "expected_calibration_error": 0.024745894514590278,
      "mean_latency_seconds": 2.8887421963363886
    },
    "family:action_selection": {
      "rows": 696,
      "accuracy": 0.6451149425287356,
      "mean_negative_log_likelihood": 1.1952036148298082,
      "expected_calibration_error": 0.1166419602383761,
      "mean_latency_seconds": 3.490999912728031
    },
    "family:bounded_extraction": {
      "rows": 2221,
      "accuracy": 0.9518235029266097,
      "mean_negative_log_likelihood": 0.17522817217551664,
      "expected_calibration_error": 0.016256765557096778,
      "mean_latency_seconds": 3.7960328164156936
    },
    "family:completion_assessment": {
      "rows": 100,
      "accuracy": 0.94,
      "mean_negative_log_likelihood": 0.18025654600959345,
      "expected_calibration_error": 0.04798941581910136,
      "mean_latency_seconds": 3.2682034737616776
    },
    "family:content_classification": {
      "rows": 100,
      "accuracy": 0.8,
      "mean_negative_log_likelihood": 1.0583038040252282,
      "expected_calibration_error": 0.12917604923718687,
      "mean_latency_seconds": 3.5517714948765935
    },
    "family:content_moderation": {
      "rows": 100,
      "accuracy": 1.0,
      "mean_negative_log_likelihood": 0.026671299885487665,
      "expected_calibration_error": 0.024745894514590278,
      "mean_latency_seconds": 2.8887421963363886
    },
    "family:context_management": {
      "rows": 200,
      "accuracy": 0.9,
      "mean_negative_log_likelihood": 0.31092185548523865,
      "expected_calibration_error": 0.04966248188359762,
      "mean_latency_seconds": 3.194561746902764
    },
    "family:document_classification": {
      "rows": 2,
      "accuracy": 1.0,
      "mean_negative_log_likelihood": 0.19273335365216956,
      "expected_calibration_error": 0.1697203101636348,
      "mean_latency_seconds": 3.351803509518504
    },
    "family:document_record_classification": {
      "rows": 2223,
      "accuracy": 0.5434098065677013,
      "mean_negative_log_likelihood": 1.2303203168892716,
      "expected_calibration_error": 0.020919832491952745,
      "mean_latency_seconds": 3.4690699621857393
    },
    "family:document_workflows": {
      "rows": 2222,
      "accuracy": 0.7898289828982898,
      "mean_negative_log_likelihood": 0.49927247287873583,
      "expected_calibration_error": 0.12390259842963341,
      "mean_latency_seconds": 3.180742418712875
    },
    "family:entity_alignment": {
      "rows": 620,
      "accuracy": 1.0,
      "mean_negative_log_likelihood": 0.08504661225525141,
      "expected_calibration_error": 0.07114453816860773,
      "mean_latency_seconds": 9.45890039720182
    },
    "family:function_agent_skill_routing": {
      "rows": 637,
      "accuracy": 0.9984301412872841,
      "mean_negative_log_likelihood": 0.008730643320973093,
      "expected_calibration_error": 0.006447168203896263,
      "mean_latency_seconds": 5.020942358130493
    },
    "family:guardrails_moderation": {
      "rows": 2222,
      "accuracy": 0.6305130513051305,
      "mean_negative_log_likelihood": 1.6214897979665268,
      "expected_calibration_error": 0.2593494562231875,
      "mean_latency_seconds": 2.918743619096778
    },
    "family:navigation": {
      "rows": 100,
      "accuracy": 0.69,
      "mean_negative_log_likelihood": 1.4235450137614085,
      "expected_calibration_error": 0.12287484718260205,
      "mean_latency_seconds": 3.878303200379014
    },
    "family:reasoning": {
      "rows": 1200,
      "accuracy": 0.6041666666666666,
      "mean_negative_log_likelihood": 0.7626218492177804,
      "expected_calibration_error": 0.0529681044240153,
      "mean_latency_seconds": 5.182950757169165
    },
    "family:record_classification": {
      "rows": 100,
      "accuracy": 0.49,
      "mean_negative_log_likelihood": 3.287670489418292,
      "expected_calibration_error": 0.40579108448235435,
      "mean_latency_seconds": 3.887372560761869
    },
    "family:retrieval_ranking": {
      "rows": 100,
      "accuracy": 0.59,
      "mean_negative_log_likelihood": 2.077772881557897,
      "expected_calibration_error": 0.18133321461258572,
      "mean_latency_seconds": 5.051503673512489
    },
    "family:retrieval_verification": {
      "rows": 2222,
      "accuracy": 0.8879387938793879,
      "mean_negative_log_likelihood": 0.2966083363051419,
      "expected_calibration_error": 0.057348545294109404,
      "mean_latency_seconds": 3.099703077122931
    },
    "family:review_scoring": {
      "rows": 98,
      "accuracy": 0.4897959183673469,
      "mean_negative_log_likelihood": 1.1541571466056402,
      "expected_calibration_error": 0.21526819619940313,
      "mean_latency_seconds": 3.1251419772755127
    },
    "family:risk_scoring": {
      "rows": 100,
      "accuracy": 0.68,
      "mean_negative_log_likelihood": 1.6129743938047514,
      "expected_calibration_error": 0.17266451174861952,
      "mean_latency_seconds": 14.342927183862775
    },
    "family:routing": {
      "rows": 300,
      "accuracy": 0.7666666666666667,
      "mean_negative_log_likelihood": 1.200576871005868,
      "expected_calibration_error": 0.13226794931216285,
      "mean_latency_seconds": 3.896309457309544
    },
    "family:rubric_scoring_prioritization": {
      "rows": 2222,
      "accuracy": 0.5067506750675067,
      "mean_negative_log_likelihood": 1.1681927351649195,
      "expected_calibration_error": 0.11397352232607402,
      "mean_latency_seconds": 3.650652985907521
    },
    "family:safety_gating": {
      "rows": 100,
      "accuracy": 0.91,
      "mean_negative_log_likelihood": 0.3033346826316373,
      "expected_calibration_error": 0.0620294738831543,
      "mean_latency_seconds": 2.91929063918069
    },
    "family:semantic_filtering": {
      "rows": 100,
      "accuracy": 0.78,
      "mean_negative_log_likelihood": 0.5682354507747938,
      "expected_calibration_error": 0.11822887763533135,
      "mean_latency_seconds": 2.9008957089670004
    },
    "family:target_selection": {
      "rows": 100,
      "accuracy": 0.46,
      "mean_negative_log_likelihood": 3.2267092782986686,
      "expected_calibration_error": 0.33194767557803595,
      "mean_latency_seconds": 4.749065974410623
    },
    "family:triage": {
      "rows": 97,
      "accuracy": 0.9278350515463918,
      "mean_negative_log_likelihood": 0.3212363760884467,
      "expected_calibration_error": 0.06374956808021356,
      "mean_latency_seconds": 3.2819226794712937
    },
    "family:verification": {
      "rows": 200,
      "accuracy": 0.6,
      "mean_negative_log_likelihood": 1.2877094507076499,
      "expected_calibration_error": 0.27978994847674116,
      "mean_latency_seconds": 3.1516549992747604
    },
    "overall": {
      "rows": 18382,
      "accuracy": 0.7295724077902296,
      "mean_negative_log_likelihood": 0.8213308311369967,
      "expected_calibration_error": 0.044364205985813615,
      "mean_latency_seconds": 3.8200671174657197
    },
    "primitive:binary_classification": {
      "rows": 6388,
      "accuracy": 0.8578584846587352,
      "mean_negative_log_likelihood": 0.34209438991861557,
      "expected_calibration_error": 0.06371461153816178,
      "mean_latency_seconds": 3.3035981744013463
    },
    "primitive:candidate_selection": {
      "rows": 10298,
      "accuracy": 0.7142163526898427,
      "mean_negative_log_likelihood": 0.8448244378933096,
      "expected_calibration_error": 0.04906023846802124,
      "mean_latency_seconds": 4.137352917121264
    },
    "primitive:ordinal_scoring": {
      "rows": 1696,
      "accuracy": 0.33962264150943394,
      "mean_negative_log_likelihood": 2.483727779324795,
      "expected_calibration_error": 0.3902262566659602,
      "mean_latency_seconds": 3.838813841193563
    },
    "task:agent-call-retention": {
      "rows": 100,
      "accuracy": 0.82,
      "mean_negative_log_likelihood": 0.4802335881035724,
      "expected_calibration_error": 0.11531398197641815,
      "mean_latency_seconds": 3.0008405078202487
    },
    "task:agent-readiness-assessment": {
      "rows": 100,
      "accuracy": 0.94,
      "mean_negative_log_likelihood": 0.18025654600959345,
      "expected_calibration_error": 0.04798941581910136,
      "mean_latency_seconds": 3.2682034737616776
    },
    "task:agent-result-retention": {
      "rows": 100,
      "accuracy": 0.98,
      "mean_negative_log_likelihood": 0.1416101228669049,
      "expected_calibration_error": 0.09683098620052823,
      "mean_latency_seconds": 3.388282985985279
    },
    "task:agent-skill-ranking": {
      "rows": 100,
      "accuracy": 0.69,
      "mean_negative_log_likelihood": 2.006527981665857,
      "expected_calibration_error": 0.21804543501161955,
      "mean_latency_seconds": 4.106465211939066
    },
    "task:bounded_value": {
      "rows": 2221,
      "accuracy": 0.9518235029266097,
      "mean_negative_log_likelihood": 0.17522817217551664,
      "expected_calibration_error": 0.016256765557096778,
      "mean_latency_seconds": 3.7960328164156936
    },
    "task:browser-action-selection": {
      "rows": 100,
      "accuracy": 0.82,
      "mean_negative_log_likelihood": 0.765633469746763,
      "expected_calibration_error": 0.11983373251164188,
      "mean_latency_seconds": 3.5189743766188624
    },
    "task:browser-target-selection": {
      "rows": 100,
      "accuracy": 0.46,
      "mean_negative_log_likelihood": 3.2267092782986686,
      "expected_calibration_error": 0.33194767557803595,
      "mean_latency_seconds": 4.749065974410623
    },
    "task:canonical_entity": {
      "rows": 620,
      "accuracy": 1.0,
      "mean_negative_log_likelihood": 0.08504661225525141,
      "expected_calibration_error": 0.07114453816860773,
      "mean_latency_seconds": 9.45890039720182
    },
    "task:claim-evidence-verification": {
      "rows": 100,
      "accuracy": 0.71,
      "mean_negative_log_likelihood": 0.8668135798996027,
      "expected_calibration_error": 0.21882203172315717,
      "mean_latency_seconds": 3.130658438038081
    },
    "task:code-change-risk": {
      "rows": 98,
      "accuracy": 0.4897959183673469,
      "mean_negative_log_likelihood": 1.1541571466056402,
      "expected_calibration_error": 0.21526819619940313,
      "mean_latency_seconds": 3.1251419772755127
    },
    "task:codebase-result-ranking": {
      "rows": 100,
      "accuracy": 0.59,
      "mean_negative_log_likelihood": 2.077772881557897,
      "expected_calibration_error": 0.18133321461258572,
      "mean_latency_seconds": 5.051503673512489
    },
    "task:contains_threat": {
      "rows": 1124,
      "accuracy": 0.99644128113879,
      "mean_negative_log_likelihood": 0.05155331092446189,
      "expected_calibration_error": 0.02195662431364581,
      "mean_latency_seconds": 2.8076875423531966
    },
    "task:database-row-classification": {
      "rows": 100,
      "accuracy": 0.49,
      "mean_negative_log_likelihood": 3.287670489418292,
      "expected_calibration_error": 0.40579108448235435,
      "mean_latency_seconds": 3.887372560761869
    },
    "task:dom-ad-detection": {
      "rows": 100,
      "accuracy": 1.0,
      "mean_negative_log_likelihood": 0.026671299885487665,
      "expected_calibration_error": 0.024745894514590278,
      "mean_latency_seconds": 2.8887421963363886
    },
    "task:drone-tactical-action": {
      "rows": 100,
      "accuracy": 0.82,
      "mean_negative_log_likelihood": 0.5171486196960103,
      "expected_calibration_error": 0.06681085239555422,
      "mean_latency_seconds": 3.5357932955026627
    },
    "task:finqa-numerical-reasoning": {
      "rows": 400,
      "accuracy": 0.4475,
      "mean_negative_log_likelihood": 0.9824826609612427,
      "expected_calibration_error": 0.10690583042984625,
      "mean_latency_seconds": 3.6011602961923925
    },
    "task:folio-logical-inference": {
      "rows": 200,
      "accuracy": 0.7,
      "mean_negative_log_likelihood": 0.7480773274650084,
      "expected_calibration_error": 0.0946618418631766,
      "mean_latency_seconds": 3.07926942858845
    },
    "task:game-goal-selection": {
      "rows": 96,
      "accuracy": 0.71875,
      "mean_negative_log_likelihood": 0.8443452445926137,
      "expected_calibration_error": 0.11050829173822861,
      "mean_latency_seconds": 3.4350416567370607
    },
    "task:graph-edge-selection": {
      "rows": 100,
      "accuracy": 0.69,
      "mean_negative_log_likelihood": 1.4235450137614085,
      "expected_calibration_error": 0.12287484718260205,
      "mean_latency_seconds": 3.878303200379014
    },
    "task:home-alert-triage": {
      "rows": 97,
      "accuracy": 0.9278350515463918,
      "mean_negative_log_likelihood": 0.3212363760884467,
      "expected_calibration_error": 0.06374956808021356,
      "mean_latency_seconds": 3.2819226794712937
    },
    "task:musique-evidence-sufficiency": {
      "rows": 400,
      "accuracy": 0.6025,
      "mean_negative_log_likelihood": 0.6684078337634163,
      "expected_calibration_error": 0.040087483697855086,
      "mean_latency_seconds": 6.872086650100536
    },
    "task:musique-multihop-answer": {
      "rows": 200,
      "accuracy": 0.825,
      "mean_negative_log_likelihood": 0.5258727783923561,
      "expected_calibration_error": 0.14010481886521842,
      "mean_latency_seconds": 7.07194122184068
    },
    "task:patent_section": {
      "rows": 2223,
      "accuracy": 0.5434098065677013,
      "mean_negative_log_likelihood": 1.2303203168892716,
      "expected_calibration_error": 0.020919832491952745,
      "mean_latency_seconds": 3.4690699621857393
    },
    "task:platformer-control-selection": {
      "rows": 100,
      "accuracy": 0.43,
      "mean_negative_log_likelihood": 1.4994180032010163,
      "expected_calibration_error": 0.18695072061396245,
      "mean_latency_seconds": 3.411730334516615
    },
    "task:relevance": {
      "rows": 2222,
      "accuracy": 0.8879387938793879,
      "mean_negative_log_likelihood": 0.2966083363051419,
      "expected_calibration_error": 0.057348545294109404,
      "mean_latency_seconds": 3.099703077122931
    },
    "task:relevance_score": {
      "rows": 2222,
      "accuracy": 0.5067506750675067,
      "mean_negative_log_likelihood": 1.1681927351649195,
      "expected_calibration_error": 0.11397352232607402,
      "mean_latency_seconds": 3.650652985907521
    },
    "task:robot-skill-selection": {
      "rows": 100,
      "accuracy": 0.6,
      "mean_negative_log_likelihood": 1.6021118735606559,
      "expected_calibration_error": 0.16321117409319175,
      "mean_latency_seconds": 3.6624210110679267
    },
    "task:runtime-outcome-verification": {
      "rows": 100,
      "accuracy": 0.49,
      "mean_negative_log_likelihood": 1.7086053215156967,
      "expected_calibration_error": 0.3587011287506007,
      "mean_latency_seconds": 3.17265156051144
    },
    "task:search-plan-routing": {
      "rows": 100,
      "accuracy": 0.91,
      "mean_negative_log_likelihood": 0.4386649082534544,
      "expected_calibration_error": 0.08325092868688802,
      "mean_latency_seconds": 3.8098051400668917
    },
    "task:semantic-line-match": {
      "rows": 100,
      "accuracy": 0.78,
      "mean_negative_log_likelihood": 0.5682354507747938,
      "expected_calibration_error": 0.11822887763533135,
      "mean_latency_seconds": 2.9008957089670004
    },
    "task:social-post-filtering": {
      "rows": 100,
      "accuracy": 0.8,
      "mean_negative_log_likelihood": 1.0583038040252282,
      "expected_calibration_error": 0.12917604923718687,
      "mean_latency_seconds": 3.5517714948765935
    },
    "task:strategy-command-selection": {
      "rows": 100,
      "accuracy": 0.5,
      "mean_negative_log_likelihood": 2.2284047549639436,
      "expected_calibration_error": 0.2584479697331665,
      "mean_latency_seconds": 3.721574952509254
    },
    "task:tax-document-page-classification": {
      "rows": 2,
      "accuracy": 1.0,
      "mean_negative_log_likelihood": 0.19273335365216956,
      "expected_calibration_error": 0.1697203101636348,
      "mean_latency_seconds": 3.351803509518504
    },
    "task:tool-action-impact": {
      "rows": 100,
      "accuracy": 0.68,
      "mean_negative_log_likelihood": 1.6129743938047514,
      "expected_calibration_error": 0.17266451174861952,
      "mean_latency_seconds": 14.342927183862775
    },
    "task:tool-call-routing": {
      "rows": 100,
      "accuracy": 0.7,
      "mean_negative_log_likelihood": 1.1565377230982927,
      "expected_calibration_error": 0.1662287852070862,
      "mean_latency_seconds": 3.7726580199226736
    },
    "task:tool-safety-gate": {
      "rows": 100,
      "accuracy": 0.91,
      "mean_negative_log_likelihood": 0.3033346826316373,
      "expected_calibration_error": 0.0620294738831543,
      "mean_latency_seconds": 2.91929063918069
    },
    "task:tool_route": {
      "rows": 637,
      "accuracy": 0.9984301412872841,
      "mean_negative_log_likelihood": 0.008730643320973093,
      "expected_calibration_error": 0.006447168203896263,
      "mean_latency_seconds": 5.020942358130493
    },
    "task:toxicity_severity": {
      "rows": 1098,
      "accuracy": 0.25591985428051,
      "mean_negative_log_likelihood": 3.228601465940371,
      "expected_calibration_error": 0.5447415971131159,
      "mean_latency_seconds": 3.032429439005508
    },
    "task:trading-action-selection": {
      "rows": 100,
      "accuracy": 0.63,
      "mean_negative_log_likelihood": 0.8953290032381667,
      "expected_calibration_error": 0.1547381492915693,
      "mean_latency_seconds": 3.149225431904197
    },
    "task:workflow_decision": {
      "rows": 2222,
      "accuracy": 0.7898289828982898,
      "mean_negative_log_likelihood": 0.49927247287873583,
      "expected_calibration_error": 0.12390259842963341,
      "mean_latency_seconds": 3.180742418712875
    },
    "suite:DecisionBench(eng, v1)": {
      "rows": 17182,
      "accuracy": 0.738330811314166,
      "mean_negative_log_likelihood": 0.8254310975962598,
      "expected_calibration_error": 0.04503883919198387,
      "mean_latency_seconds": 3.7248826006665037
    }
  },
  "artifact": {
    "uri": "hf://buckets/Hanno-Labs/training/decision-bench/runs/decider-35b-23900-20260922-r1/decider-35b-a3b",
    "manifest_sha256": "598bd3ef55e3bd52342d611075041036a2478274586109364c8f9725b0dd3e7a",
    "summary_sha256": "dcfc452d25eae2364d64e6f7a8b6ea2a0d583ec731b1233ca43f8b7a1c723c49",
    "raw_sha256": "003c55ffc2393264095b9cd628dd772ea99779df5db44fe29b56330886659121"
  },
  "submitted_at": "2026-09-22T21:06:18.154638Z"
}