{"calibration":[{"checksum":"baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","metric":"ece_raw","metric_definition":"Expected calibration error for choice using raw evaluation probabilities","model":"laya","primitive":"choice","run_id":"laya-assisted-20260925-5","sample_count":736,"split":"evaluation","unit":"fraction","value":0.0021294756618364784},{"checksum":"baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","metric":"ece_fitted","metric_definition":"Expected calibration error for choice using calibration-fitted evaluation probabilities","model":"laya","primitive":"choice","run_id":"laya-assisted-20260925-5","sample_count":736,"split":"evaluation","unit":"fraction","value":0.0027817282546448367},{"checksum":"baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","metric":"ece_raw","metric_definition":"Expected calibration error for noul using raw evaluation probabilities","model":"laya","primitive":"noul","run_id":"laya-assisted-20260925-5","sample_count":384,"split":"evaluation","unit":"fraction","value":0.004774482314627559},{"checksum":"baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","metric":"ece_fitted","metric_definition":"Expected calibration error for noul using calibration-fitted evaluation probabilities","model":"laya","primitive":"noul","run_id":"laya-assisted-20260925-5","sample_count":384,"split":"evaluation","unit":"fraction","value":0.0021577786089748757},{"checksum":"1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e","metric":"ece_raw","metric_definition":"Expected calibration error for choice using raw evaluation probabilities","model":"jev","primitive":"choice","run_id":"jev-assisted-20260925","sample_count":736,"split":"evaluation","unit":"fraction","value":0.0009158587783783721},{"checksum":"1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e","metric":"ece_fitted","metric_definition":"Expected calibration error for choice using calibration-fitted evaluation probabilities","model":"jev","primitive":"choice","run_id":"jev-assisted-20260925","sample_count":736,"split":"evaluation","unit":"fraction","value":0.0009160711013483203},{"checksum":"1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e","metric":"ece_raw","metric_definition":"Expected calibration error for noul using raw evaluation probabilities","model":"jev","primitive":"noul","run_id":"jev-assisted-20260925","sample_count":384,"split":"evaluation","unit":"fraction","value":0.004668501442628624},{"checksum":"1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e","metric":"ece_fitted","metric_definition":"Expected calibration error for noul using calibration-fitted evaluation probabilities","model":"jev","primitive":"noul","run_id":"jev-assisted-20260925","sample_count":384,"split":"evaluation","unit":"fraction","value":0.005830057395979381},{"checksum":"cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb","metric":"ece_raw","metric_definition":"Expected calibration error for choice using raw evaluation probabilities","model":"qwen-pcd","primitive":"choice","run_id":"qwen-assisted-20260925","sample_count":736,"split":"evaluation","unit":"fraction","value":0.002332767192563974},{"checksum":"cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb","metric":"ece_fitted","metric_definition":"Expected calibration error for choice using calibration-fitted evaluation probabilities","model":"qwen-pcd","primitive":"choice","run_id":"qwen-assisted-20260925","sample_count":736,"split":"evaluation","unit":"fraction","value":0.0009155183712336363},{"checksum":"cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb","metric":"ece_raw","metric_definition":"Expected calibration error for noul using raw evaluation probabilities","model":"qwen-pcd","primitive":"noul","run_id":"qwen-assisted-20260925","sample_count":384,"split":"evaluation","unit":"fraction","value":0.005771060758291521},{"checksum":"cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb","metric":"ece_fitted","metric_definition":"Expected calibration error for noul using calibration-fitted evaluation probabilities","model":"qwen-pcd","primitive":"noul","run_id":"qwen-assisted-20260925","sample_count":384,"split":"evaluation","unit":"fraction","value":0.0017397064060351338}],"dataset":{"evaluation_cases":952,"evaluation_decisions":1240,"manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","version":"2.0.0"},"efficiency":{"jev":{"p50_ms":313.542,"p95_ms":386.704},"laya":{"p50_ms":588.359,"p95_ms":1364.391},"qwen-pcd":{"p50_ms":3947.5,"p95_ms":13470.742}},"multilingual":{"ar":{"jev":1.0,"laya":0.16666666666666666,"qwen-pcd":0.8333333333333334},"de":{"jev":1.0,"laya":0.625,"qwen-pcd":0.6666666666666666},"es":{"jev":1.0,"laya":0.7083333333333334,"qwen-pcd":0.6666666666666666},"fr":{"jev":1.0,"laya":0.75,"qwen-pcd":0.625},"hi":{"jev":1.0,"laya":0.0,"qwen-pcd":0.625}},"runs":{"jev":{"checksum":"1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e","model":"jev-1.13.0","revision":"unknown","run_id":"jev-assisted-20260925","serving":"api"},"laya":{"checksum":"baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","model":"convaiinnovations/laya","revision":"55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851","run_id":"laya-assisted-20260925-5","serving":"local"},"qwen-pcd":{"checksum":"cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb","model":"Qwen/Qwen2.5-1.5B-Instruct","revision":"989aa7980e4cf806f80c7fef2b1adb7bc71aa306","run_id":"qwen-assisted-20260925","serving":"local"}},"schema_version":2,"suites":{"agnews":{"ci_high":0.19375,"ci_low":0.08125,"decisions":160,"delta":0.13749999999999996,"jev":0.9875,"laya":0.85,"p_value":4.999750012499375e-05,"p_value_holm":0.00044997750112494374,"qwen-pcd":0.9125},"banking77_12":{"ci_high":0.21875,"ci_low":0.0625,"decisions":96,"delta":0.13541666666666663,"jev":0.9479166666666666,"laya":0.8125,"p_value":0.0025498725063746812,"p_value_holm":0.0050997450127493625,"qwen-pcd":0.5833333333333334},"emotion":{"ci_high":0.2552083333333333,"ci_low":0.11979166666666666,"decisions":192,"delta":0.1875,"jev":0.84375,"laya":0.65625,"p_value":4.999750012499375e-05,"p_value_holm":0.00044997750112494374,"qwen-pcd":0.4114583333333333},"guardrails":{"ci_high":0.32291666666666663,"ci_low":0.15625,"decisions":96,"delta":0.23958333333333337,"jev":1.0,"laya":0.7604166666666666,"p_value":4.999750012499375e-05,"p_value_holm":0.00044997750112494374,"qwen-pcd":0.2916666666666667},"mnli":{"ci_high":0.4083333333333333,"ci_low":0.20833333333333334,"decisions":120,"delta":0.30833333333333335,"jev":0.8666666666666667,"laya":0.5583333333333333,"p_value":4.999750012499375e-05,"p_value_holm":0.00044997750112494374,"qwen-pcd":0.3416666666666667},"moderation":{"ci_high":0.25,"ci_low":0.125,"decisions":144,"delta":0.1875,"jev":0.9444444444444444,"laya":0.7569444444444444,"p_value":4.999750012499375e-05,"p_value_holm":0.00044997750112494374,"qwen-pcd":0.7291666666666666},"multilingual_intent":{"ci_high":0.6416666666666666,"ci_low":0.4583333333333333,"decisions":120,"delta":0.55,"jev":1.0,"laya":0.45,"p_value":4.999750012499375e-05,"p_value_holm":0.00044997750112494374,"qwen-pcd":0.6833333333333333},"sst5":{"ci_high":0.425,"ci_low":0.20833333333333334,"decisions":120,"delta":0.3166666666666667,"jev":0.65,"laya":0.3333333333333333,"p_value":4.999750012499375e-05,"p_value_holm":0.00044997750112494374,"qwen-pcd":0.525},"triage":{"ci_high":0.109375,"ci_low":0.005208333333333333,"decisions":192,"delta":0.05729166666666663,"jev":0.9322916666666666,"laya":0.875,"p_value":0.06269686515674217,"p_value_holm":0.06269686515674217,"qwen-pcd":0.78125}}}