{"accuracy":[{"checksum":"baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"laya","plotted_values":{"value":0.85},"run_id":"laya-assisted-20260925-5","sample_count":160,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5"],"split":"evaluation","suite_id":"agnews","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.85},{"checksum":"1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"jev","plotted_values":{"value":0.9875},"run_id":"jev-assisted-20260925","sample_count":160,"source_checksums":["1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["jev-assisted-20260925"],"split":"evaluation","suite_id":"agnews","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.9875},{"checksum":"cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"qwen-pcd","plotted_values":{"value":0.9125},"run_id":"qwen-assisted-20260925","sample_count":160,"source_checksums":["cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb"],"source_kind":"base","source_run_ids":["qwen-assisted-20260925"],"split":"evaluation","suite_id":"agnews","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.9125},{"checksum":"baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"laya","plotted_values":{"value":0.8125},"run_id":"laya-assisted-20260925-5","sample_count":96,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5"],"split":"evaluation","suite_id":"banking77_12","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.8125},{"checksum":"1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"jev","plotted_values":{"value":0.9479166666666666},"run_id":"jev-assisted-20260925","sample_count":96,"source_checksums":["1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["jev-assisted-20260925"],"split":"evaluation","suite_id":"banking77_12","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.9479166666666666},{"checksum":"cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"qwen-pcd","plotted_values":{"value":0.5833333333333334},"run_id":"qwen-assisted-20260925","sample_count":96,"source_checksums":["cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb"],"source_kind":"base","source_run_ids":["qwen-assisted-20260925"],"split":"evaluation","suite_id":"banking77_12","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.5833333333333334},{"checksum":"baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"laya","plotted_values":{"value":0.65625},"run_id":"laya-assisted-20260925-5","sample_count":192,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5"],"split":"evaluation","suite_id":"emotion","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.65625},{"checksum":"1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"jev","plotted_values":{"value":0.84375},"run_id":"jev-assisted-20260925","sample_count":192,"source_checksums":["1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["jev-assisted-20260925"],"split":"evaluation","suite_id":"emotion","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.84375},{"checksum":"cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"qwen-pcd","plotted_values":{"value":0.4114583333333333},"run_id":"qwen-assisted-20260925","sample_count":192,"source_checksums":["cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb"],"source_kind":"base","source_run_ids":["qwen-assisted-20260925"],"split":"evaluation","suite_id":"emotion","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.4114583333333333},{"checksum":"baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"laya","plotted_values":{"value":0.7604166666666666},"run_id":"laya-assisted-20260925-5","sample_count":96,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5"],"split":"evaluation","suite_id":"guardrails","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.7604166666666666},{"checksum":"1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"jev","plotted_values":{"value":1.0},"run_id":"jev-assisted-20260925","sample_count":96,"source_checksums":["1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["jev-assisted-20260925"],"split":"evaluation","suite_id":"guardrails","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":1.0},{"checksum":"cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"qwen-pcd","plotted_values":{"value":0.2916666666666667},"run_id":"qwen-assisted-20260925","sample_count":96,"source_checksums":["cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb"],"source_kind":"base","source_run_ids":["qwen-assisted-20260925"],"split":"evaluation","suite_id":"guardrails","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.2916666666666667},{"checksum":"baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"laya","plotted_values":{"value":0.5583333333333333},"run_id":"laya-assisted-20260925-5","sample_count":120,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5"],"split":"evaluation","suite_id":"mnli","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.5583333333333333},{"checksum":"1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"jev","plotted_values":{"value":0.8666666666666667},"run_id":"jev-assisted-20260925","sample_count":120,"source_checksums":["1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["jev-assisted-20260925"],"split":"evaluation","suite_id":"mnli","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.8666666666666667},{"checksum":"cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"qwen-pcd","plotted_values":{"value":0.3416666666666667},"run_id":"qwen-assisted-20260925","sample_count":120,"source_checksums":["cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb"],"source_kind":"base","source_run_ids":["qwen-assisted-20260925"],"split":"evaluation","suite_id":"mnli","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.3416666666666667},{"checksum":"baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"laya","plotted_values":{"value":0.7569444444444444},"run_id":"laya-assisted-20260925-5","sample_count":144,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5"],"split":"evaluation","suite_id":"moderation","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.7569444444444444},{"checksum":"1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"jev","plotted_values":{"value":0.9444444444444444},"run_id":"jev-assisted-20260925","sample_count":144,"source_checksums":["1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["jev-assisted-20260925"],"split":"evaluation","suite_id":"moderation","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.9444444444444444},{"checksum":"cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"qwen-pcd","plotted_values":{"value":0.7291666666666666},"run_id":"qwen-assisted-20260925","sample_count":144,"source_checksums":["cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb"],"source_kind":"base","source_run_ids":["qwen-assisted-20260925"],"split":"evaluation","suite_id":"moderation","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.7291666666666666},{"checksum":"baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"laya","plotted_values":{"value":0.45},"run_id":"laya-assisted-20260925-5","sample_count":120,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5"],"split":"evaluation","suite_id":"multilingual_intent","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.45},{"checksum":"1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"jev","plotted_values":{"value":1.0},"run_id":"jev-assisted-20260925","sample_count":120,"source_checksums":["1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["jev-assisted-20260925"],"split":"evaluation","suite_id":"multilingual_intent","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":1.0},{"checksum":"cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"qwen-pcd","plotted_values":{"value":0.6833333333333333},"run_id":"qwen-assisted-20260925","sample_count":120,"source_checksums":["cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb"],"source_kind":"base","source_run_ids":["qwen-assisted-20260925"],"split":"evaluation","suite_id":"multilingual_intent","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.6833333333333333},{"checksum":"baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"laya","plotted_values":{"value":0.3333333333333333},"run_id":"laya-assisted-20260925-5","sample_count":120,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5"],"split":"evaluation","suite_id":"sst5","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.3333333333333333},{"checksum":"1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"jev","plotted_values":{"value":0.65},"run_id":"jev-assisted-20260925","sample_count":120,"source_checksums":["1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["jev-assisted-20260925"],"split":"evaluation","suite_id":"sst5","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.65},{"checksum":"cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"qwen-pcd","plotted_values":{"value":0.525},"run_id":"qwen-assisted-20260925","sample_count":120,"source_checksums":["cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb"],"source_kind":"base","source_run_ids":["qwen-assisted-20260925"],"split":"evaluation","suite_id":"sst5","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.525},{"checksum":"baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"laya","plotted_values":{"value":0.875},"run_id":"laya-assisted-20260925-5","sample_count":192,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5"],"split":"evaluation","suite_id":"triage","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.875},{"checksum":"1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"jev","plotted_values":{"value":0.9322916666666666},"run_id":"jev-assisted-20260925","sample_count":192,"source_checksums":["1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["jev-assisted-20260925"],"split":"evaluation","suite_id":"triage","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.9322916666666666},{"checksum":"cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Micro accuracy across reviewed decisions in the evaluation split","model":"qwen-pcd","plotted_values":{"value":0.78125},"run_id":"qwen-assisted-20260925","sample_count":192,"source_checksums":["cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb"],"source_kind":"base","source_run_ids":["qwen-assisted-20260925"],"split":"evaluation","suite_id":"triage","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.78125}],"calibration":[{"checksum":"baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"ece_fitted","metric_definition":"Expected calibration error for choice using calibration-fitted evaluation probabilities","model":"laya","plotted_values":{"value":0.0027817282546448367},"primitive":"choice","run_id":"laya-assisted-20260925-5","sample_count":736,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5"],"split":"evaluation","suite_id":"comparison","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.0027817282546448367},{"checksum":"baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"ece_raw","metric_definition":"Expected calibration error for choice using raw evaluation probabilities","model":"laya","plotted_values":{"value":0.0021294756618364784},"primitive":"choice","run_id":"laya-assisted-20260925-5","sample_count":736,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5"],"split":"evaluation","suite_id":"comparison","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.0021294756618364784},{"checksum":"1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"ece_fitted","metric_definition":"Expected calibration error for choice using calibration-fitted evaluation probabilities","model":"jev","plotted_values":{"value":0.0009160711013483203},"primitive":"choice","run_id":"jev-assisted-20260925","sample_count":736,"source_checksums":["1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["jev-assisted-20260925"],"split":"evaluation","suite_id":"comparison","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.0009160711013483203},{"checksum":"1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"ece_raw","metric_definition":"Expected calibration error for choice using raw evaluation probabilities","model":"jev","plotted_values":{"value":0.0009158587783783721},"primitive":"choice","run_id":"jev-assisted-20260925","sample_count":736,"source_checksums":["1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["jev-assisted-20260925"],"split":"evaluation","suite_id":"comparison","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.0009158587783783721},{"checksum":"cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"ece_fitted","metric_definition":"Expected calibration error for choice using calibration-fitted evaluation probabilities","model":"qwen-pcd","plotted_values":{"value":0.0009155183712336363},"primitive":"choice","run_id":"qwen-assisted-20260925","sample_count":736,"source_checksums":["cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb"],"source_kind":"base","source_run_ids":["qwen-assisted-20260925"],"split":"evaluation","suite_id":"comparison","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.0009155183712336363},{"checksum":"cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"ece_raw","metric_definition":"Expected calibration error for choice using raw evaluation probabilities","model":"qwen-pcd","plotted_values":{"value":0.002332767192563974},"primitive":"choice","run_id":"qwen-assisted-20260925","sample_count":736,"source_checksums":["cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb"],"source_kind":"base","source_run_ids":["qwen-assisted-20260925"],"split":"evaluation","suite_id":"comparison","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.002332767192563974},{"checksum":"baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"ece_fitted","metric_definition":"Expected calibration error for noul using calibration-fitted evaluation probabilities","model":"laya","plotted_values":{"value":0.0021577786089748757},"primitive":"noul","run_id":"laya-assisted-20260925-5","sample_count":384,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5"],"split":"evaluation","suite_id":"comparison","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.0021577786089748757},{"checksum":"baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"ece_raw","metric_definition":"Expected calibration error for noul using raw evaluation probabilities","model":"laya","plotted_values":{"value":0.004774482314627559},"primitive":"noul","run_id":"laya-assisted-20260925-5","sample_count":384,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5"],"split":"evaluation","suite_id":"comparison","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.004774482314627559},{"checksum":"1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"ece_fitted","metric_definition":"Expected calibration error for noul using calibration-fitted evaluation probabilities","model":"jev","plotted_values":{"value":0.005830057395979381},"primitive":"noul","run_id":"jev-assisted-20260925","sample_count":384,"source_checksums":["1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["jev-assisted-20260925"],"split":"evaluation","suite_id":"comparison","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.005830057395979381},{"checksum":"1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"ece_raw","metric_definition":"Expected calibration error for noul using raw evaluation probabilities","model":"jev","plotted_values":{"value":0.004668501442628624},"primitive":"noul","run_id":"jev-assisted-20260925","sample_count":384,"source_checksums":["1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["jev-assisted-20260925"],"split":"evaluation","suite_id":"comparison","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.004668501442628624},{"checksum":"cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"ece_fitted","metric_definition":"Expected calibration error for noul using calibration-fitted evaluation probabilities","model":"qwen-pcd","plotted_values":{"value":0.0017397064060351338},"primitive":"noul","run_id":"qwen-assisted-20260925","sample_count":384,"source_checksums":["cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb"],"source_kind":"base","source_run_ids":["qwen-assisted-20260925"],"split":"evaluation","suite_id":"comparison","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.0017397064060351338},{"checksum":"cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"ece_raw","metric_definition":"Expected calibration error for noul using raw evaluation probabilities","model":"qwen-pcd","plotted_values":{"value":0.005771060758291521},"primitive":"noul","run_id":"qwen-assisted-20260925","sample_count":384,"source_checksums":["cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb"],"source_kind":"base","source_run_ids":["qwen-assisted-20260925"],"split":"evaluation","suite_id":"comparison","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.005771060758291521}],"efficiency":[{"checksum":"baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"p50_ms","metric_definition":"Median end-to-end latency in milliseconds","model":"laya","plotted_values":{"value":588.359},"run_id":"laya-assisted-20260925-5","sample_count":952,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5"],"split":"evaluation","suite_id":"comparison","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"milliseconds","value":588.359},{"checksum":"baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"p95_ms","metric_definition":"95th-percentile end-to-end latency in milliseconds","model":"laya","plotted_values":{"value":1364.391},"run_id":"laya-assisted-20260925-5","sample_count":952,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5"],"split":"evaluation","suite_id":"comparison","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"milliseconds","value":1364.391},{"checksum":"1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"p50_ms","metric_definition":"Median end-to-end latency in milliseconds","model":"jev","plotted_values":{"value":313.542},"run_id":"jev-assisted-20260925","sample_count":952,"source_checksums":["1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["jev-assisted-20260925"],"split":"evaluation","suite_id":"comparison","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"milliseconds","value":313.542},{"checksum":"1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"p95_ms","metric_definition":"95th-percentile end-to-end latency in milliseconds","model":"jev","plotted_values":{"value":386.704},"run_id":"jev-assisted-20260925","sample_count":952,"source_checksums":["1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["jev-assisted-20260925"],"split":"evaluation","suite_id":"comparison","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"milliseconds","value":386.704},{"checksum":"cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"p50_ms","metric_definition":"Median end-to-end latency in milliseconds","model":"qwen-pcd","plotted_values":{"value":3947.5},"run_id":"qwen-assisted-20260925","sample_count":952,"source_checksums":["cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb"],"source_kind":"base","source_run_ids":["qwen-assisted-20260925"],"split":"evaluation","suite_id":"comparison","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"milliseconds","value":3947.5},{"checksum":"cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"p95_ms","metric_definition":"95th-percentile end-to-end latency in milliseconds","model":"qwen-pcd","plotted_values":{"value":13470.742},"run_id":"qwen-assisted-20260925","sample_count":952,"source_checksums":["cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb"],"source_kind":"base","source_run_ids":["qwen-assisted-20260925"],"split":"evaluation","suite_id":"comparison","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"milliseconds","value":13470.742}],"multilingual":[{"checksum":"baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","dataset_version":"2.0.0","language":"ar","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Classification accuracy for the stated language in the evaluation split","model":"laya","plotted_values":{"value":0.16666666666666666},"run_id":"laya-assisted-20260925-5","sample_count":null,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5"],"split":"evaluation","suite_id":"multilingual_intent","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.16666666666666666},{"checksum":"1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e","dataset_version":"2.0.0","language":"ar","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Classification accuracy for the stated language in the evaluation split","model":"jev","plotted_values":{"value":1.0},"run_id":"jev-assisted-20260925","sample_count":null,"source_checksums":["1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["jev-assisted-20260925"],"split":"evaluation","suite_id":"multilingual_intent","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":1.0},{"checksum":"cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb","dataset_version":"2.0.0","language":"ar","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Classification accuracy for the stated language in the evaluation split","model":"qwen-pcd","plotted_values":{"value":0.8333333333333334},"run_id":"qwen-assisted-20260925","sample_count":null,"source_checksums":["cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb"],"source_kind":"base","source_run_ids":["qwen-assisted-20260925"],"split":"evaluation","suite_id":"multilingual_intent","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.8333333333333334},{"checksum":"baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","dataset_version":"2.0.0","language":"de","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Classification accuracy for the stated language in the evaluation split","model":"laya","plotted_values":{"value":0.625},"run_id":"laya-assisted-20260925-5","sample_count":null,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5"],"split":"evaluation","suite_id":"multilingual_intent","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.625},{"checksum":"1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e","dataset_version":"2.0.0","language":"de","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Classification accuracy for the stated language in the evaluation split","model":"jev","plotted_values":{"value":1.0},"run_id":"jev-assisted-20260925","sample_count":null,"source_checksums":["1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["jev-assisted-20260925"],"split":"evaluation","suite_id":"multilingual_intent","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":1.0},{"checksum":"cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb","dataset_version":"2.0.0","language":"de","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Classification accuracy for the stated language in the evaluation split","model":"qwen-pcd","plotted_values":{"value":0.6666666666666666},"run_id":"qwen-assisted-20260925","sample_count":null,"source_checksums":["cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb"],"source_kind":"base","source_run_ids":["qwen-assisted-20260925"],"split":"evaluation","suite_id":"multilingual_intent","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.6666666666666666},{"checksum":"baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","dataset_version":"2.0.0","language":"es","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Classification accuracy for the stated language in the evaluation split","model":"laya","plotted_values":{"value":0.7083333333333334},"run_id":"laya-assisted-20260925-5","sample_count":null,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5"],"split":"evaluation","suite_id":"multilingual_intent","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.7083333333333334},{"checksum":"1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e","dataset_version":"2.0.0","language":"es","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Classification accuracy for the stated language in the evaluation split","model":"jev","plotted_values":{"value":1.0},"run_id":"jev-assisted-20260925","sample_count":null,"source_checksums":["1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["jev-assisted-20260925"],"split":"evaluation","suite_id":"multilingual_intent","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":1.0},{"checksum":"cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb","dataset_version":"2.0.0","language":"es","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Classification accuracy for the stated language in the evaluation split","model":"qwen-pcd","plotted_values":{"value":0.6666666666666666},"run_id":"qwen-assisted-20260925","sample_count":null,"source_checksums":["cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb"],"source_kind":"base","source_run_ids":["qwen-assisted-20260925"],"split":"evaluation","suite_id":"multilingual_intent","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.6666666666666666},{"checksum":"baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","dataset_version":"2.0.0","language":"fr","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Classification accuracy for the stated language in the evaluation split","model":"laya","plotted_values":{"value":0.75},"run_id":"laya-assisted-20260925-5","sample_count":null,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5"],"split":"evaluation","suite_id":"multilingual_intent","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.75},{"checksum":"1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e","dataset_version":"2.0.0","language":"fr","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Classification accuracy for the stated language in the evaluation split","model":"jev","plotted_values":{"value":1.0},"run_id":"jev-assisted-20260925","sample_count":null,"source_checksums":["1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["jev-assisted-20260925"],"split":"evaluation","suite_id":"multilingual_intent","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":1.0},{"checksum":"cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb","dataset_version":"2.0.0","language":"fr","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Classification accuracy for the stated language in the evaluation split","model":"qwen-pcd","plotted_values":{"value":0.625},"run_id":"qwen-assisted-20260925","sample_count":null,"source_checksums":["cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb"],"source_kind":"base","source_run_ids":["qwen-assisted-20260925"],"split":"evaluation","suite_id":"multilingual_intent","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.625},{"checksum":"baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","dataset_version":"2.0.0","language":"hi","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Classification accuracy for the stated language in the evaluation split","model":"laya","plotted_values":{"value":0.0},"run_id":"laya-assisted-20260925-5","sample_count":null,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5"],"split":"evaluation","suite_id":"multilingual_intent","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.0},{"checksum":"1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e","dataset_version":"2.0.0","language":"hi","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Classification accuracy for the stated language in the evaluation split","model":"jev","plotted_values":{"value":1.0},"run_id":"jev-assisted-20260925","sample_count":null,"source_checksums":["1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["jev-assisted-20260925"],"split":"evaluation","suite_id":"multilingual_intent","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":1.0},{"checksum":"cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb","dataset_version":"2.0.0","language":"hi","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"accuracy","metric_definition":"Classification accuracy for the stated language in the evaluation split","model":"qwen-pcd","plotted_values":{"value":0.625},"run_id":"qwen-assisted-20260925","sample_count":null,"source_checksums":["cd8c9a301e9f54c0dbff87a23c33ee786996caa21ffd6be956074c40d53ebfcb"],"source_kind":"base","source_run_ids":["qwen-assisted-20260925"],"split":"evaluation","suite_id":"multilingual_intent","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.625}],"paired_effects":[{"ci_high":0.19375,"ci_low":0.08125,"contrast":"jev_minus_laya","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"delta","metric_definition":"Jev evaluation accuracy minus Laya evaluation accuracy","model":"jev_minus_laya","p_value":4.999750012499375e-05,"p_value_holm":0.00044997750112494374,"plotted_values":{"ci_high":0.19375,"ci_low":0.08125,"estimate":0.13749999999999996,"p_value":4.999750012499375e-05,"p_value_holm":0.00044997750112494374},"sample_count":160,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5","jev-assisted-20260925"],"split":"evaluation","suite_id":"agnews","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.13749999999999996},{"ci_high":0.21875,"ci_low":0.0625,"contrast":"jev_minus_laya","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"delta","metric_definition":"Jev evaluation accuracy minus Laya evaluation accuracy","model":"jev_minus_laya","p_value":0.0025498725063746812,"p_value_holm":0.0050997450127493625,"plotted_values":{"ci_high":0.21875,"ci_low":0.0625,"estimate":0.13541666666666663,"p_value":0.0025498725063746812,"p_value_holm":0.0050997450127493625},"sample_count":96,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5","jev-assisted-20260925"],"split":"evaluation","suite_id":"banking77_12","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.13541666666666663},{"ci_high":0.2552083333333333,"ci_low":0.11979166666666666,"contrast":"jev_minus_laya","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"delta","metric_definition":"Jev evaluation accuracy minus Laya evaluation accuracy","model":"jev_minus_laya","p_value":4.999750012499375e-05,"p_value_holm":0.00044997750112494374,"plotted_values":{"ci_high":0.2552083333333333,"ci_low":0.11979166666666666,"estimate":0.1875,"p_value":4.999750012499375e-05,"p_value_holm":0.00044997750112494374},"sample_count":192,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5","jev-assisted-20260925"],"split":"evaluation","suite_id":"emotion","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.1875},{"ci_high":0.32291666666666663,"ci_low":0.15625,"contrast":"jev_minus_laya","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"delta","metric_definition":"Jev evaluation accuracy minus Laya evaluation accuracy","model":"jev_minus_laya","p_value":4.999750012499375e-05,"p_value_holm":0.00044997750112494374,"plotted_values":{"ci_high":0.32291666666666663,"ci_low":0.15625,"estimate":0.23958333333333337,"p_value":4.999750012499375e-05,"p_value_holm":0.00044997750112494374},"sample_count":96,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5","jev-assisted-20260925"],"split":"evaluation","suite_id":"guardrails","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.23958333333333337},{"ci_high":0.4083333333333333,"ci_low":0.20833333333333334,"contrast":"jev_minus_laya","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"delta","metric_definition":"Jev evaluation accuracy minus Laya evaluation accuracy","model":"jev_minus_laya","p_value":4.999750012499375e-05,"p_value_holm":0.00044997750112494374,"plotted_values":{"ci_high":0.4083333333333333,"ci_low":0.20833333333333334,"estimate":0.30833333333333335,"p_value":4.999750012499375e-05,"p_value_holm":0.00044997750112494374},"sample_count":120,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5","jev-assisted-20260925"],"split":"evaluation","suite_id":"mnli","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.30833333333333335},{"ci_high":0.25,"ci_low":0.125,"contrast":"jev_minus_laya","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"delta","metric_definition":"Jev evaluation accuracy minus Laya evaluation accuracy","model":"jev_minus_laya","p_value":4.999750012499375e-05,"p_value_holm":0.00044997750112494374,"plotted_values":{"ci_high":0.25,"ci_low":0.125,"estimate":0.1875,"p_value":4.999750012499375e-05,"p_value_holm":0.00044997750112494374},"sample_count":144,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5","jev-assisted-20260925"],"split":"evaluation","suite_id":"moderation","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.1875},{"ci_high":0.6416666666666666,"ci_low":0.4583333333333333,"contrast":"jev_minus_laya","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"delta","metric_definition":"Jev evaluation accuracy minus Laya evaluation accuracy","model":"jev_minus_laya","p_value":4.999750012499375e-05,"p_value_holm":0.00044997750112494374,"plotted_values":{"ci_high":0.6416666666666666,"ci_low":0.4583333333333333,"estimate":0.55,"p_value":4.999750012499375e-05,"p_value_holm":0.00044997750112494374},"sample_count":120,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5","jev-assisted-20260925"],"split":"evaluation","suite_id":"multilingual_intent","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.55},{"ci_high":0.425,"ci_low":0.20833333333333334,"contrast":"jev_minus_laya","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"delta","metric_definition":"Jev evaluation accuracy minus Laya evaluation accuracy","model":"jev_minus_laya","p_value":4.999750012499375e-05,"p_value_holm":0.00044997750112494374,"plotted_values":{"ci_high":0.425,"ci_low":0.20833333333333334,"estimate":0.3166666666666667,"p_value":4.999750012499375e-05,"p_value_holm":0.00044997750112494374},"sample_count":120,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5","jev-assisted-20260925"],"split":"evaluation","suite_id":"sst5","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.3166666666666667},{"ci_high":0.109375,"ci_low":0.005208333333333333,"contrast":"jev_minus_laya","dataset_version":"2.0.0","manifest_digest":"4272a7a25ebcb324235696ad808544e157a31e3f9c04421da731a08dfb9d5768","metric":"delta","metric_definition":"Jev evaluation accuracy minus Laya evaluation accuracy","model":"jev_minus_laya","p_value":0.06269686515674217,"p_value_holm":0.06269686515674217,"plotted_values":{"ci_high":0.109375,"ci_low":0.005208333333333333,"estimate":0.05729166666666663,"p_value":0.06269686515674217,"p_value_holm":0.06269686515674217},"sample_count":192,"source_checksums":["baaf56782282c0b45a1a515c0a188052d55b0fb67245c2ae011344ed1ea82fc7","1b40cc74031daf1dea731c33433f91c77c62547596fc2c46ce559ec2f002c58e"],"source_kind":"base","source_run_ids":["laya-assisted-20260925-5","jev-assisted-20260925"],"split":"evaluation","suite_id":"triage","uncertainty_method":"paired_state_cluster_bootstrap_20000_seed_42","unit":"fraction","value":0.05729166666666663}],"risk_coverage":[],"router":[]}