{
 "manifestSha256": "0d21a64c6b61b6be7b49050dbd89f89f934912a7b586734e475308e9c9422ea2",
 "logicalDigest": "4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768",
 "cases": 1190,
 "evaluationCases": 952,
 "decisions": 1240,
 "seed": 42,
 "datasetVersion": "2.1.0",
 "suites": [
  "agnews",
  "banking77_12",
  "emotion",
  "guardrails",
  "mnli",
  "moderation",
  "multilingual_intent",
  "sst5",
  "triage"
 ],
 "readoutCaveat": {
  "scorings": [
   "per_option_conditional_logprob"
  ],
  "confirmedOn": "lev",
  "confirmedBy": "https://huggingface.co/interfaze-ai/lev/discussions/1",
  "note": "These rows were scored by scoring each declared option independently. For a model trained to answer with label codes restricted in the next-token distribution, that is a different function of the same weights rather than a different implementation of the same one, and it can under-report the model. Confirmed for lev, whose authors measured 0.980 on banking77 in-distribution against our 0.7292 out-of-distribution, and who describe selecting by argmax over one-token label codes with two option orders averaged. We have not confirmed the same expectation for the other rows carrying this flag, so treat them as unqualified rather than known-low. No number here has been changed and no model has been re-run."
 },
 "excluded": [
  [
   "winnow-e4b",
   "GGUF only",
   "A vision-language checkpoint published solely as GGUF. Needs a different runtime."
  ],
  [
   "clm-v0.1-8b",
   "Contrastive reranker",
   "Scores state-answer pairs rather than producing typed decisions."
  ],
  [
   "winnow-12b",
   "GGUF only",
   "22.3 GiB of bf16 weights published solely as GGUF. Fits current hardware but needs a different runtime."
  ],
  [
   "jeff",
   "Artifacts lost",
   "Measured 0.6855 on a DGX Spark, but that run was never mirrored and the host is gone. The row is withdrawn until it is re-run; the broken earlier run is retained under quarantine and is not reported."
  ],
  [
   "seb-9b",
   "Awaiting a GPU host",
   "9.65B, added to scope at the author's request. Not yet measured."
  ]
 ],
 "scopeTotal": 50,
 "runDirectoriesVerified": 52,
 "distinctMeasurements": 49,
 "referenceClosedApi": {
  "runner": "jev",
  "model": "Jev 1.13.0",
  "accuracy": 0.9065,
  "serving": "closed API at api.typesafe.ai",
  "inPanel": false
 },
 "measurements": [
  {
   "runner": "decider-4b",
   "runId": "decider-4b-t4-d0-v210",
   "model": "Mapika/decider-4b",
   "revision": "eb5fbdfc9448473ec25e399882912863afbdb70e",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "vendor-managed",
   "serving": "local",
   "accuracy": 0.8580645161290322,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.925,
    "banking77_12": 0.9166666666666666,
    "emotion": 0.7916666666666666,
    "guardrails": 0.8333333333333334,
    "mnli": 0.8416666666666667,
    "moderation": 0.9444444444444444,
    "multilingual_intent": 1.0,
    "sst5": 0.48333333333333334,
    "triage": 0.9427083333333334
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "decision-lux-9b",
   "runId": "decision-lux-9b-spark-1",
   "model": "llm-semantic-router/Decision-1.0-Lux-9B",
   "revision": "2064c84daf599447f840719ef8e86db152a56851",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cpu",
   "dtype": "vendor-managed",
   "serving": "local",
   "accuracy": 0.8483870967741935,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.84375,
    "banking77_12": 0.8854166666666666,
    "emotion": 0.734375,
    "guardrails": 0.9791666666666666,
    "mnli": 0.9166666666666666,
    "moderation": 0.9305555555555556,
    "multilingual_intent": 0.975,
    "sst5": 0.525,
    "triage": 0.9010416666666666
   },
   "latencyP50": {
    "agnews": 0.1484362236224115,
    "banking77_12": 0.18424990214407444,
    "emotion": 0.1601894369814545,
    "guardrails": 0.16042418300639838,
    "mnli": 0.14797753049060702,
    "moderation": 0.18247860705014318,
    "multilingual_intent": 0.16026931360829622,
    "sst5": 0.1393970079952851,
    "triage": 0.31610619998537004
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "jet",
   "runId": "jet-t4-f0-v210",
   "model": "michaljach/jet",
   "revision": "fbc3d2daa679e0d4bd9f99c9912b6496d5a41f0a",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "vendor-managed",
   "serving": "local",
   "accuracy": 0.8451612903225807,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.89375,
    "banking77_12": 0.90625,
    "emotion": 0.7916666666666666,
    "guardrails": 0.8333333333333334,
    "mnli": 0.825,
    "moderation": 0.9305555555555556,
    "multilingual_intent": 0.9666666666666667,
    "sst5": 0.49166666666666664,
    "triage": 0.9270833333333334
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "kev-4b",
   "runId": "kev-4b-t4-v1-v210",
   "model": "jaredpalmer/kev-4b",
   "revision": "6cfce5c2fa4b4bd64026336ab649c5ca78857d52",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "vendor-managed",
   "serving": "loopback-subprocess",
   "accuracy": 0.8443548387096774,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.88125,
    "banking77_12": 0.9270833333333334,
    "emotion": 0.8072916666666666,
    "guardrails": 0.9166666666666666,
    "mnli": 0.8,
    "moderation": 0.9097222222222222,
    "multilingual_intent": 0.9583333333333334,
    "sst5": 0.45,
    "triage": 0.9270833333333334
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "jpt-9b",
   "runId": "jpt-9b-t4-c1-v210",
   "model": "kirp/jpt-9b",
   "revision": "b447cc7ee105c0a76a22f8fde8ecf05074dc8be0",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "float16",
   "serving": "loopback-subprocess",
   "accuracy": 0.8379032258064516,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.88125,
    "banking77_12": 0.8958333333333334,
    "emotion": 0.7864583333333334,
    "guardrails": 0.9791666666666666,
    "mnli": 0.7583333333333333,
    "moderation": 0.9375,
    "multilingual_intent": 0.8416666666666667,
    "sst5": 0.5166666666666667,
    "triage": 0.9270833333333334
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "intern-decision-4b",
   "runId": "intern-decision-4b-t4-gpu0-v210",
   "model": "internlm/Intern-Decision-4B",
   "revision": "0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "float16",
   "serving": "local",
   "accuracy": 0.8314516129032258,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.875,
    "banking77_12": 0.8958333333333334,
    "emotion": 0.8072916666666666,
    "guardrails": 0.7916666666666666,
    "mnli": 0.7333333333333333,
    "moderation": 0.8888888888888888,
    "multilingual_intent": 1.0,
    "sst5": 0.4583333333333333,
    "triage": 0.953125
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "kev-9b",
   "runId": "kev-9b-spark-1",
   "model": "jaredpalmer/kev-9b",
   "revision": "db029f08b290afd9fee4aa4bbcd9ae48602d1eb0",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "vendor-managed",
   "serving": "loopback-subprocess",
   "accuracy": 0.8290322580645161,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.875,
    "banking77_12": 0.9270833333333334,
    "emotion": 0.8125,
    "guardrails": 0.8020833333333334,
    "mnli": 0.8416666666666667,
    "moderation": 0.8055555555555556,
    "multilingual_intent": 0.9333333333333333,
    "sst5": 0.5166666666666667,
    "triage": 0.9114583333333334
   },
   "latencyP50": {
    "agnews": 0.6277861393755302,
    "banking77_12": 0.6164043990429491,
    "emotion": 0.5426440999144688,
    "guardrails": 0.566511366982013,
    "mnli": 0.6240064374869689,
    "moderation": 0.5688687224173918,
    "multilingual_intent": 0.564832657459192,
    "sst5": 0.49538842984475195,
    "triage": 0.7586066676303744
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "jevk5",
   "runId": "jevk5-t4-c0-v210",
   "model": "alibiserikbay/JevK5",
   "revision": "c4f7fdb3aeab5582336406e78d3bef11bf98833d",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "vendor-calibrated",
   "serving": "local",
   "accuracy": 0.8282258064516129,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.78125,
    "banking77_12": 0.875,
    "emotion": 0.8125,
    "guardrails": 0.8333333333333334,
    "mnli": 0.7666666666666667,
    "moderation": 0.9375,
    "multilingual_intent": 1.0,
    "sst5": 0.475,
    "triage": 0.9270833333333334
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "neohorse-4b",
   "runId": "neohorse-4b-t4-d1-v210",
   "model": "TokenRhythm/NeoHorse-Jev-4B",
   "revision": "434cb21d3a994a953d3ae5788405fcb2c4970554",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "vendor-managed",
   "serving": "local",
   "accuracy": 0.8282258064516129,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.89375,
    "banking77_12": 0.9166666666666666,
    "emotion": 0.78125,
    "guardrails": 0.9895833333333334,
    "mnli": 0.85,
    "moderation": 0.9027777777777778,
    "multilingual_intent": 0.7166666666666667,
    "sst5": 0.475,
    "triage": 0.9166666666666666
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "jpt-4b",
   "runId": "jpt-4b-spark-1",
   "model": "kirp/jpt-4b",
   "revision": "78312f855b8bebf83ae7e9e8f05b5a0f73f519a9",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "auto",
   "serving": "loopback-subprocess",
   "accuracy": 0.8274193548387097,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.8875,
    "banking77_12": 0.84375,
    "emotion": 0.796875,
    "guardrails": 0.9479166666666666,
    "mnli": 0.7416666666666667,
    "moderation": 0.9097222222222222,
    "multilingual_intent": 0.8083333333333333,
    "sst5": 0.5416666666666666,
    "triage": 0.921875
   },
   "latencyP50": {
    "agnews": 0.08500509895384312,
    "banking77_12": 0.09944389399606735,
    "emotion": 0.08382662606891245,
    "guardrails": 0.13266246696002781,
    "mnli": 0.09435782744549215,
    "moderation": 0.19778597203548998,
    "multilingual_intent": 0.09760918642859906,
    "sst5": 0.08461063308641315,
    "triage": 0.2765486428979784
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "tev1-4b",
   "runId": "tev1-4b-t4-20261002-v210",
   "model": "togethercomputer/Tev1-4B-experimental",
   "revision": "0b7becf017daa0e5eb222f8ce7483c8c8259c52f",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "float16",
   "serving": "local",
   "accuracy": 0.8241935483870968,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.84375,
    "banking77_12": 0.8958333333333334,
    "emotion": 0.796875,
    "guardrails": 0.9479166666666666,
    "mnli": 0.825,
    "moderation": 0.9375,
    "multilingual_intent": 0.8416666666666667,
    "sst5": 0.4583333333333333,
    "triage": 0.8697916666666666
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "decision-nox",
   "runId": "decision-nox-t4-gpu0-v210",
   "model": "llm-semantic-router/Decision-1.0-Nox",
   "revision": "7f65e1db2c55a11ab8b558a8027c8883752f192d",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "vendor-managed",
   "serving": "local",
   "accuracy": 0.8,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.84375,
    "banking77_12": 0.875,
    "emotion": 0.734375,
    "guardrails": 0.84375,
    "mnli": 0.7833333333333333,
    "moderation": 0.9027777777777778,
    "multilingual_intent": 0.9333333333333333,
    "sst5": 0.375,
    "triage": 0.8854166666666666
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "decider-2b",
   "runId": "decider-2b-t4-d0-v210",
   "model": "Mapika/decider-2b",
   "revision": "533964dae8be954c5b5e19fa4948e48408094c1e",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "vendor-managed",
   "serving": "local",
   "accuracy": 0.7975806451612903,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.89375,
    "banking77_12": 0.90625,
    "emotion": 0.7083333333333334,
    "guardrails": 0.6354166666666666,
    "mnli": 0.7916666666666666,
    "moderation": 0.8819444444444444,
    "multilingual_intent": 0.9166666666666666,
    "sst5": 0.4666666666666667,
    "triage": 0.90625
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "this-that-12",
   "runId": "this-that-12-20261002-v210",
   "model": "flock-io/this-that-model-1.2",
   "revision": "c4d1c30b8d512d278726de439b8cc81fccc70c8f",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cpu",
   "dtype": null,
   "serving": "local",
   "accuracy": 0.7870967741935484,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.80625,
    "banking77_12": 0.8854166666666666,
    "emotion": 0.703125,
    "guardrails": 0.5833333333333334,
    "mnli": 0.7833333333333333,
    "moderation": 0.9097222222222222,
    "multilingual_intent": 0.9416666666666667,
    "sst5": 0.525,
    "triage": 0.8854166666666666
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "decision-eos",
   "runId": "decision-eos-20261002-v210",
   "model": "llm-semantic-router/Decision-1.0-Eos-0.8B",
   "revision": "bbdc2221d0ecb1a7929be9f5822bfc4394a4a1fb",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cpu",
   "dtype": null,
   "serving": "local",
   "accuracy": 0.7814516129032258,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.8625,
    "banking77_12": 0.8229166666666666,
    "emotion": 0.65625,
    "guardrails": 0.8645833333333334,
    "mnli": 0.75,
    "moderation": 0.8611111111111112,
    "multilingual_intent": 0.9083333333333333,
    "sst5": 0.4083333333333333,
    "triage": 0.890625
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "hopper-g",
   "runId": "hopper-g-t4-q0-v210",
   "model": "HopitAI/hopper-g",
   "revision": "d60a1d6ca3f3fa25e5daddea16b8ba2b431ebcee",
   "baseModel": "Qwen/Qwen3.5-4B",
   "scoring": "option_letter_logits",
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "float16",
   "serving": "local",
   "accuracy": 0.7798387096774193,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.825,
    "banking77_12": 0.8125,
    "emotion": 0.8125,
    "guardrails": 0.7708333333333334,
    "mnli": 0.5916666666666667,
    "moderation": 0.8888888888888888,
    "multilingual_intent": 0.8916666666666667,
    "sst5": 0.39166666666666666,
    "triage": 0.90625
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "lumma-fev-4b",
   "runId": "lumma-fev-4b-spark-1",
   "model": "FrontiersMind/Lumma-fev-4b",
   "revision": "6fea60ae613c832b41c65a74095ae20428ca7d7b",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "auto",
   "serving": "local",
   "accuracy": 0.7758064516129032,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.95625,
    "banking77_12": 0.9791666666666666,
    "emotion": 0.609375,
    "guardrails": 0.5833333333333334,
    "mnli": 0.7333333333333333,
    "moderation": 0.9444444444444444,
    "multilingual_intent": 0.8166666666666667,
    "sst5": 0.3333333333333333,
    "triage": 0.9375
   },
   "latencyP50": {
    "agnews": 0.058921876014210284,
    "banking77_12": 0.07409136556088924,
    "emotion": 0.05369020195212215,
    "guardrails": 0.0524854640243575,
    "mnli": 0.05912290851119906,
    "moderation": 0.06479918805416673,
    "multilingual_intent": 0.059731881017796695,
    "sst5": 0.04460139153525233,
    "triage": 0.13117618148680776
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "lumma-fev-9b",
   "runId": "lumma-fev-9b-spark-1",
   "model": "FrontiersMind/Lumma-fev-9b",
   "revision": "51c4b084f40bcc45073f5c92d6119c72a58210b2",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "auto",
   "serving": "local",
   "accuracy": 0.7741935483870968,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.95625,
    "banking77_12": 0.9895833333333334,
    "emotion": 0.6197916666666666,
    "guardrails": 0.78125,
    "mnli": 0.7666666666666667,
    "moderation": 0.9236111111111112,
    "multilingual_intent": 0.525,
    "sst5": 0.4083333333333333,
    "triage": 0.9427083333333334
   },
   "latencyP50": {
    "agnews": 0.10865770990494639,
    "banking77_12": 0.1381475500529632,
    "emotion": 0.09069016703870147,
    "guardrails": 0.0889944100053981,
    "mnli": 0.10840501845814288,
    "moderation": 0.1136881901184097,
    "multilingual_intent": 0.10862690710928291,
    "sst5": 0.07988677697721869,
    "triage": 0.1991460684221238
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "kev-08b",
   "runId": "kev-08b-t4-c0-v210",
   "model": "jaredpalmer/kev-0.8b",
   "revision": "bf75a6a8848ea6960ff2ed108d9ed44c2941174f",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "vendor-managed",
   "serving": "loopback-subprocess",
   "accuracy": 0.7725806451612903,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.86875,
    "banking77_12": 0.9270833333333334,
    "emotion": 0.6822916666666666,
    "guardrails": 0.90625,
    "mnli": 0.5083333333333333,
    "moderation": 0.875,
    "multilingual_intent": 0.9,
    "sst5": 0.4166666666666667,
    "triage": 0.8697916666666666
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "intern-decision-2b",
   "runId": "intern-decision-2b-t4-20261002-v210",
   "model": "internlm/Intern-Decision-2B",
   "revision": "8797836c65fc91a2435b1fb6850b5f0aabd75cc3",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "float16",
   "serving": "local",
   "accuracy": 0.771774193548387,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.86875,
    "banking77_12": 0.8020833333333334,
    "emotion": 0.765625,
    "guardrails": 0.7708333333333334,
    "mnli": 0.6666666666666666,
    "moderation": 0.7986111111111112,
    "multilingual_intent": 0.975,
    "sst5": 0.3333333333333333,
    "triage": 0.875
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "nimble-v2",
   "runId": "nimble-v2-t4-s1-v210",
   "model": "bespokelabs/Bespoke-Nimble-9B-v2",
   "revision": "4b8c04d1ac2cea3e41e5e3c4d2130bcead2c0abe",
   "baseModel": "Qwen/Qwen3.5-9B",
   "scoring": "option_letter_logits",
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": "device_map=auto across all visible GPUs",
   "singleDevice": false,
   "device": "cpu",
   "dtype": "auto",
   "serving": "local",
   "accuracy": 0.7701612903225806,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.80625,
    "banking77_12": 0.4479166666666667,
    "emotion": 0.7604166666666666,
    "guardrails": 0.9791666666666666,
    "mnli": 0.55,
    "moderation": 0.9305555555555556,
    "multilingual_intent": 0.925,
    "sst5": 0.44166666666666665,
    "triage": 0.9322916666666666
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "decision-sol",
   "runId": "decision-sol-20261002-v210",
   "model": "llm-semantic-router/Decision-1.0-Sol",
   "revision": "fc210c8fcc7f1b86519aac34be810950b582e35e",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cpu",
   "dtype": null,
   "serving": "local",
   "accuracy": 0.7693548387096775,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.79375,
    "banking77_12": 0.84375,
    "emotion": 0.6822916666666666,
    "guardrails": 0.7291666666666666,
    "mnli": 0.8,
    "moderation": 0.9097222222222222,
    "multilingual_intent": 0.8416666666666667,
    "sst5": 0.4,
    "triage": 0.8802083333333334
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "jpt-08b",
   "runId": "jpt-08b-t4-gpu1b-v210",
   "model": "kirp/jpt-0.8b",
   "revision": "1431c0509bbc10772cd59964cc7af5835c8720d6",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "float16",
   "serving": "loopback-subprocess",
   "accuracy": 0.7645161290322581,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.83125,
    "banking77_12": 0.8125,
    "emotion": 0.7239583333333334,
    "guardrails": 0.9270833333333334,
    "mnli": 0.6,
    "moderation": 0.875,
    "multilingual_intent": 0.7416666666666667,
    "sst5": 0.425,
    "triage": 0.890625
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "tev1-08b",
   "runId": "tev1-08b-t4-g1-v210",
   "model": "togethercomputer/Tev1-0.8B-experimental",
   "revision": "6bb2dff14b38fea90ddb14d870166ccaf77374e9",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "float16",
   "serving": "local",
   "accuracy": 0.7637096774193548,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.85,
    "banking77_12": 0.8125,
    "emotion": 0.6666666666666666,
    "guardrails": 0.6145833333333334,
    "mnli": 0.675,
    "moderation": 0.875,
    "multilingual_intent": 1.0,
    "sst5": 0.49166666666666664,
    "triage": 0.8333333333333334
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "mini-jev",
   "runId": "mini-jev-t4-t0-v210",
   "model": "Qwen/Qwen3-4B-Instruct-2507",
   "revision": "cdbee75f17c01a7cc42f958dc650907174af0554",
   "baseModel": "Qwen/Qwen3-4B-Instruct-2507",
   "scoring": "per_option_conditional_logprob",
   "techniqueReimplementation": true,
   "vendorCodeExecuted": false,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "float16",
   "serving": "local",
   "accuracy": 0.7620967741935484,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.8625,
    "banking77_12": 0.71875,
    "emotion": 0.7291666666666666,
    "guardrails": 0.84375,
    "mnli": 0.5833333333333334,
    "moderation": 0.9444444444444444,
    "multilingual_intent": 0.8,
    "sst5": 0.45,
    "triage": 0.8385416666666666
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": true,
   "deferrals": null
  },
  {
   "runner": "bosun-17b",
   "runId": "bosun-17b-t4-gpu1-v210",
   "model": "Hanno-Labs/bosun-v3.1-1.7b",
   "revision": "1d8dc82a20e4a32ed60927a47272d6efff48eed2",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "float16",
   "serving": "local",
   "accuracy": 0.7451612903225806,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.83125,
    "banking77_12": 0.9791666666666666,
    "emotion": 0.5520833333333334,
    "guardrails": 0.6979166666666666,
    "mnli": 0.6583333333333333,
    "moderation": 0.8611111111111112,
    "multilingual_intent": 0.9916666666666667,
    "sst5": 0.26666666666666666,
    "triage": 0.8854166666666666
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "metask",
   "runId": "metask-t4-m1-v210",
   "model": "wayfind/metask-jev-4b-policy-mix",
   "revision": "ea20fe85b28733b1522721dec119bec50947a869",
   "baseModel": null,
   "scoring": "per_option_conditional_logprob",
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "float16",
   "serving": "local",
   "accuracy": 0.739516129032258,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.88125,
    "banking77_12": 0.8020833333333334,
    "emotion": 0.78125,
    "guardrails": 0.5104166666666666,
    "mnli": 0.5333333333333333,
    "moderation": 0.8888888888888888,
    "multilingual_intent": 0.7916666666666666,
    "sst5": 0.39166666666666666,
    "triage": 0.8645833333333334
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": true,
   "deferrals": null
  },
  {
   "runner": "intern-decision-08b",
   "runId": "intern-decision-08b-t4-gpu1-v210",
   "model": "internlm/Intern-Decision-0.8B",
   "revision": "85a0cc5a99d67ea8d56dfe98115689212867171d",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "float16",
   "serving": "local",
   "accuracy": 0.7241935483870968,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.85625,
    "banking77_12": 0.7916666666666666,
    "emotion": 0.71875,
    "guardrails": 0.34375,
    "mnli": 0.5166666666666667,
    "moderation": 0.8819444444444444,
    "multilingual_intent": 0.975,
    "sst5": 0.3333333333333333,
    "triage": 0.875
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "lavoir",
   "runId": "lavoir-t4-gpu0-v210",
   "model": "moganai/lavoir",
   "revision": "4c5eaeb99b2368f1416b9893c7aacc58235aa09f",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "vendor-managed",
   "serving": "local",
   "accuracy": 0.7104838709677419,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.88125,
    "banking77_12": 0.8229166666666666,
    "emotion": 0.6979166666666666,
    "guardrails": 0.7291666666666666,
    "mnli": 0.43333333333333335,
    "moderation": 0.7916666666666666,
    "multilingual_intent": 0.5666666666666667,
    "sst5": 0.38333333333333336,
    "triage": 0.921875
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "decision-kai",
   "runId": "decision-kai-20261002-v210",
   "model": "llm-semantic-router/Decision-1.0-Kai",
   "revision": "69aef4060cafad7e8bf812e0014da8097b30bf69",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cpu",
   "dtype": null,
   "serving": "local",
   "accuracy": 0.7088709677419355,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.75625,
    "banking77_12": 0.71875,
    "emotion": 0.6354166666666666,
    "guardrails": 0.8854166666666666,
    "mnli": 0.6333333333333333,
    "moderation": 0.8194444444444444,
    "multilingual_intent": 0.7833333333333333,
    "sst5": 0.2,
    "triage": 0.8854166666666666
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "bosun-06b",
   "runId": "bosun-06b-20261002-v210",
   "model": "Hanno-Labs/bosun-v3.1-0.6b",
   "revision": "1d8b6f9611f9b64b514ce8b57cd86398fbc31a3b",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cpu",
   "dtype": null,
   "serving": "local",
   "accuracy": 0.7,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.78125,
    "banking77_12": 0.8958333333333334,
    "emotion": 0.5364583333333334,
    "guardrails": 0.5104166666666666,
    "mnli": 0.6,
    "moderation": 0.8125,
    "multilingual_intent": 1.0,
    "sst5": 0.25,
    "triage": 0.8645833333333334
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "laya",
   "runId": "laya-8cpu-smoke-v210",
   "model": "convaiinnovations/laya",
   "revision": "55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cpu",
   "dtype": null,
   "serving": "local",
   "accuracy": 0.6975806451612904,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.93125,
    "banking77_12": 0.7604166666666666,
    "emotion": 0.6770833333333334,
    "guardrails": 0.7604166666666666,
    "mnli": 0.6166666666666667,
    "moderation": 0.7569444444444444,
    "multilingual_intent": 0.45,
    "sst5": 0.2916666666666667,
    "triage": 0.875
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "gliner-decide",
   "runId": "probe-gliner-decide-v210",
   "model": "fastino/GLiNER2.5-Decide",
   "revision": "5a7adf72a23b4d311abae6ce050d7f0012bb3416",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cpu",
   "dtype": null,
   "serving": "local",
   "accuracy": 0.692741935483871,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.7375,
    "banking77_12": 0.875,
    "emotion": 0.6666666666666666,
    "guardrails": 0.78125,
    "mnli": 0.36666666666666664,
    "moderation": 0.7430555555555556,
    "multilingual_intent": 0.6333333333333333,
    "sst5": 0.49166666666666664,
    "triage": 0.875
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "mimir",
   "runId": "mimir-1.2.0-20261009-v210",
   "model": "Mythologic/MIMIR-1",
   "revision": "d2f5a66ea111adb8d3cd5f6c59d66974f073ab80",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cpu",
   "dtype": null,
   "serving": "local",
   "accuracy": 0.6717741935483871,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.9,
    "banking77_12": 0.8020833333333334,
    "emotion": 0.625,
    "guardrails": 0.84375,
    "mnli": 0.7166666666666667,
    "moderation": 0.7777777777777778,
    "multilingual_intent": 0.11666666666666667,
    "sst5": 0.425,
    "triage": 0.7708333333333334
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": {
    "deferred": 1040,
    "deferred_share": 0.8387096774193549,
    "evaluation_decisions": 1240,
    "projected_to_argmax": 169
   }
  },
  {
   "runner": "gliner-base",
   "runId": "gliner-base-20261002-v210",
   "model": "fastino/gliner2.5-base-v1",
   "revision": "ca906247640776a07753514055be9726f9080ead",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cpu",
   "dtype": null,
   "serving": "local",
   "accuracy": 0.635483870967742,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.7375,
    "banking77_12": 0.78125,
    "emotion": 0.6979166666666666,
    "guardrails": 0.59375,
    "mnli": 0.6,
    "moderation": 0.6180555555555556,
    "multilingual_intent": 0.6166666666666667,
    "sst5": 0.38333333333333336,
    "triage": 0.640625
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "decision-lex",
   "runId": "decision-lex-20261002-v210",
   "model": "llm-semantic-router/decision-1.0-lex",
   "revision": "1f9750a75ae9d32825e7f29ae44c1e1406676884",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cpu",
   "dtype": null,
   "serving": "local",
   "accuracy": 0.5967741935483871,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.71875,
    "banking77_12": 0.6666666666666666,
    "emotion": 0.5729166666666666,
    "guardrails": 0.3020833333333333,
    "mnli": 0.39166666666666666,
    "moderation": 0.7291666666666666,
    "multilingual_intent": 0.7416666666666667,
    "sst5": 0.26666666666666666,
    "triage": 0.7760416666666666
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "gliner-multi",
   "runId": "gliner-multi-20261002-v210",
   "model": "fastino/gliner2.5-multi-v1",
   "revision": "2ca71aafb3446d9014e1c55c7ff51c9bc7209c47",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cpu",
   "dtype": null,
   "serving": "local",
   "accuracy": 0.5717741935483871,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.725,
    "banking77_12": 0.8125,
    "emotion": 0.578125,
    "guardrails": 0.6145833333333334,
    "mnli": 0.5666666666666667,
    "moderation": 0.2222222222222222,
    "multilingual_intent": 0.8416666666666667,
    "sst5": 0.30833333333333335,
    "triage": 0.5572916666666666
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "lev",
   "runId": "lev-t4-s1-v210",
   "model": "interfaze-ai/lev",
   "revision": "7bdc748dffebd85b57ee0dbea8f994c6354fed31",
   "baseModel": "Qwen/Qwen3.5-4B",
   "scoring": "option_letter_logits",
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "float16",
   "serving": "local",
   "accuracy": 0.5637096774193548,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.55625,
    "banking77_12": 0.7916666666666666,
    "emotion": 0.296875,
    "guardrails": 0.9375,
    "mnli": 0.3333333333333333,
    "moderation": 0.8055555555555556,
    "multilingual_intent": 0.225,
    "sst5": 0.5333333333333333,
    "triage": 0.7291666666666666
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "jobe",
   "runId": "jobe-t4-t0-v210",
   "model": "Qwen/Qwen3.5-4B-Base",
   "revision": "1001bb4d826a52d1f399e183466143f4da7b741b",
   "baseModel": "Qwen/Qwen3.5-4B-Base",
   "scoring": "per_option_conditional_logprob",
   "techniqueReimplementation": true,
   "vendorCodeExecuted": false,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "float16",
   "serving": "local",
   "accuracy": 0.5516129032258065,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.84375,
    "banking77_12": 0.3958333333333333,
    "emotion": 0.7239583333333334,
    "guardrails": 0.9166666666666666,
    "mnli": 0.3333333333333333,
    "moderation": 0.4861111111111111,
    "multilingual_intent": 0.3333333333333333,
    "sst5": 0.36666666666666664,
    "triage": 0.46875
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": true,
   "deferrals": null
  },
  {
   "runner": "verdict",
   "runId": "verdict-t4-j0-v210",
   "model": "heman10x/rlcd-modernbert-151m",
   "revision": "8af2496eb63c7fa66d7d234e1f62629380030eb4",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "float32",
   "serving": "local",
   "accuracy": 0.5516129032258065,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.61875,
    "banking77_12": 0.7708333333333334,
    "emotion": 0.5104166666666666,
    "guardrails": 0.4583333333333333,
    "mnli": 0.4083333333333333,
    "moderation": 0.6111111111111112,
    "multilingual_intent": 0.5083333333333333,
    "sst5": 0.325,
    "triage": 0.6875
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "mojev",
   "runId": "mojev-20261002-v210",
   "model": "MoLeMo-Lab/mojev",
   "revision": "0c8695b6252f4205907433d4e196a94f032e60c3",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cpu",
   "dtype": null,
   "serving": "local",
   "accuracy": 0.5435483870967742,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.63125,
    "banking77_12": 0.7708333333333334,
    "emotion": 0.3541666666666667,
    "guardrails": 0.2916666666666667,
    "mnli": 0.4666666666666667,
    "moderation": 0.7291666666666666,
    "multilingual_intent": 0.4583333333333333,
    "sst5": 0.30833333333333335,
    "triage": 0.78125
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "gliner-small",
   "runId": "gliner-small-20261002-v210",
   "model": "fastino/gliner2.5-small-v1",
   "revision": "7132dc4561c3f94563c6147e75ffa8ef34c4964a",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cpu",
   "dtype": null,
   "serving": "local",
   "accuracy": 0.5362903225806451,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.76875,
    "banking77_12": 0.75,
    "emotion": 0.609375,
    "guardrails": 0.5625,
    "mnli": 0.6333333333333333,
    "moderation": 0.2847222222222222,
    "multilingual_intent": 0.5333333333333333,
    "sst5": 0.2916666666666667,
    "triage": 0.4322916666666667
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "julia-1",
   "runId": "julia-1-t4-gpu1-v210",
   "model": "SupersonicLabs/Julia-1",
   "revision": "a85b127321d580d65176c89ced8273f305745d85",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "float16",
   "serving": "local",
   "accuracy": 0.5258064516129032,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.54375,
    "banking77_12": 0.84375,
    "emotion": 0.6145833333333334,
    "guardrails": 0.3020833333333333,
    "mnli": 0.3333333333333333,
    "moderation": 0.3819444444444444,
    "multilingual_intent": 0.75,
    "sst5": 0.3,
    "triage": 0.6041666666666666
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "lumma-fev-06b",
   "runId": "lumma-fev-06b-20261002-v210",
   "model": "FrontiersMind/Lumma-fev-0.6b",
   "revision": "59272e2c10506d99a464c999dcaa91b5108c3640",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cpu",
   "dtype": null,
   "serving": "local",
   "accuracy": 0.5145161290322581,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.95625,
    "banking77_12": 0.9375,
    "emotion": 0.625,
    "guardrails": 0.65625,
    "mnli": 0.325,
    "moderation": 0.2777777777777778,
    "multilingual_intent": 0.06666666666666667,
    "sst5": 0.25833333333333336,
    "triage": 0.4895833333333333
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "lumma-fev-01b",
   "runId": "lumma-fev-01b-20261002-v210",
   "model": "FrontiersMind/Lumma-fev-0.1b",
   "revision": "085f4705aa860a6404d6cc3ff17de8a2969ac0f4",
   "baseModel": null,
   "scoring": null,
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cpu",
   "dtype": null,
   "serving": "local",
   "accuracy": 0.4290322580645161,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.90625,
    "banking77_12": 0.6875,
    "emotion": 0.5416666666666666,
    "guardrails": 0.2708333333333333,
    "mnli": 0.35833333333333334,
    "moderation": 0.2847222222222222,
    "multilingual_intent": 0.0,
    "sst5": 0.2,
    "triage": 0.4322916666666667
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  },
  {
   "runner": "harsha",
   "runId": "harsha-t4-u1-v210",
   "model": "Qwen/Qwen2.5-1.5B",
   "revision": "8faed761d45a263340a0528343f099c05c9a4323",
   "baseModel": "Qwen/Qwen2.5-1.5B",
   "scoring": "per_option_conditional_logprob",
   "techniqueReimplementation": true,
   "vendorCodeExecuted": false,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "float16",
   "serving": "local",
   "accuracy": 0.4290322580645161,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.55625,
    "banking77_12": 0.2604166666666667,
    "emotion": 0.6458333333333334,
    "guardrails": 0.7083333333333334,
    "mnli": 0.3333333333333333,
    "moderation": 0.2638888888888889,
    "multilingual_intent": 0.2833333333333333,
    "sst5": 0.3416666666666667,
    "triage": 0.3802083333333333
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": true,
   "deferrals": null
  },
  {
   "runner": "lfm2600",
   "runId": "lfm2600-t4-n0-v210",
   "model": "monotykamary/LFM2.5-2.6B-RLCD",
   "revision": "31455458983bdbdc41b69ebbcedabd0d5de299c9",
   "baseModel": null,
   "scoring": "per_option_conditional_logprob",
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "float16",
   "serving": "local",
   "accuracy": 0.3758064516129032,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.25,
    "banking77_12": 0.3020833333333333,
    "emotion": 0.21875,
    "guardrails": 0.5416666666666666,
    "mnli": 0.3416666666666667,
    "moderation": 0.6875,
    "multilingual_intent": 0.16666666666666666,
    "sst5": 0.36666666666666664,
    "triage": 0.515625
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": true,
   "deferrals": null
  },
  {
   "runner": "lfm350",
   "runId": "lfm350-t4-m0-v210",
   "model": "notnotsamuel/LFM2.5-350M-RLCD",
   "revision": "deb589d803d141cabd158ef55f6617b128529f36",
   "baseModel": null,
   "scoring": "per_option_conditional_logprob",
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "float16",
   "serving": "local",
   "accuracy": 0.34919354838709676,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.4375,
    "banking77_12": 0.125,
    "emotion": 0.5260416666666666,
    "guardrails": 0.7083333333333334,
    "mnli": 0.425,
    "moderation": 0.2708333333333333,
    "multilingual_intent": 0.16666666666666666,
    "sst5": 0.20833333333333334,
    "triage": 0.24479166666666666
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": true,
   "deferrals": null
  },
  {
   "runner": "pngwn",
   "runId": "pngwn-t4-x0-v210",
   "model": "pngwn/system-one-qwen3.5-4b-scorer-v2b",
   "revision": "3ec7785bf6aa1e8b0ade992c85206d46711d86c2",
   "baseModel": "Qwen/Qwen3.5-4B-Base",
   "scoring": "per_option_scalar_head_softmax",
   "techniqueReimplementation": false,
   "vendorCodeExecuted": null,
   "sharding": null,
   "singleDevice": null,
   "device": "cuda",
   "dtype": "float16",
   "serving": "local",
   "accuracy": 0.25967741935483873,
   "cases": 952,
   "decisions": 1240,
   "suites": {
    "agnews": 0.3375,
    "banking77_12": 0.2604166666666667,
    "emotion": 0.078125,
    "guardrails": 0.7083333333333334,
    "mnli": 0.38333333333333336,
    "moderation": 0.2708333333333333,
    "multilingual_intent": 0.16666666666666666,
    "sst5": 0.09166666666666666,
    "triage": 0.22916666666666666
   },
   "latencyP50": {
    "agnews": null,
    "banking77_12": null,
    "emotion": null,
    "guardrails": null,
    "mnli": null,
    "moderation": null,
    "multilingual_intent": null,
    "sst5": null,
    "triage": null
   },
   "readoutCaveat": false,
   "deferrals": null
  }
 ],
 "duplicates": {
  "repeatedRuns": {
   "decider-2b": [
    "decider-2b-t4-c0-v210",
    "decider-2b-t4-d0-v210"
   ],
   "tev1-08b": [
    "tev1-08b-20261002-v210",
    "tev1-08b-t4-g1-v210"
   ]
  },
  "identicalPredictions": {
   "952:544fc02392c51e9070d6f4945333125d": [
    "mini-jev",
    "openvons"
   ]
  },
  "runnersCountedOnce": [
   "openvons"
  ]
 }
}