{ "buckets": [ { "fid": "F1", "key": "content_msg", "label": "Content wrong / insufficient -- message" }, { "fid": "F2", "key": "content_file", "label": "Content wrong / insufficient -- written file" }, { "fid": "F3", "key": "grounding", "label": "Grounding gap -- required asset never opened" }, { "fid": "F4", "key": "anchor", "label": "Missing evidence anchor (literal substring in a file)" }, { "fid": "F5", "key": "side_effect", "label": "Required terminal action absent" }, { "fid": "F6", "key": "window", "label": "Acted outside the required time window" }, { "fid": "--", "key": "other", "label": "UNBUCKETED (guard -- must stay empty)" } ], "cells": { "high": { "head_to_head": { "dsv4": { "flash": { "loss": 0, "tie": 0, "win": 60 }, "flash36": { "loss": 31, "tie": 1, "win": 28 }, "flash37": { "loss": 29, "tie": 0, "win": 31 }, "glm": { "loss": 9, "tie": 1, "win": 50 }, "kimi": { "loss": 27, "tie": 0, "win": 33 }, "luna": { "loss": 36, "tie": 1, "win": 23 }, "opus": { "loss": 41, "tie": 0, "win": 19 }, "opus5": { "loss": 45, "tie": 1, "win": 14 }, "pro": { "loss": 11, "tie": 0, "win": 49 }, "sol": { "loss": 39, "tie": 1, "win": 20 }, "sonnet": { "loss": 34, "tie": 2, "win": 24 }, "terra": { "loss": 34, "tie": 0, "win": 26 } }, "flash": { "dsv4": { "loss": 60, "tie": 0, "win": 0 }, "flash36": { "loss": 60, "tie": 0, "win": 0 }, "flash37": { "loss": 60, "tie": 0, "win": 0 }, "glm": { "loss": 57, "tie": 0, "win": 3 }, "kimi": { "loss": 58, "tie": 0, "win": 2 }, "luna": { "loss": 59, "tie": 0, "win": 1 }, "opus": { "loss": 59, "tie": 0, "win": 1 }, "opus5": { "loss": 59, "tie": 0, "win": 1 }, "pro": { "loss": 56, "tie": 0, "win": 4 }, "sol": { "loss": 60, "tie": 0, "win": 0 }, "sonnet": { "loss": 58, "tie": 0, "win": 2 }, "terra": { "loss": 60, "tie": 0, "win": 0 } }, "flash36": { "dsv4": { "loss": 28, "tie": 1, "win": 31 }, "flash": { "loss": 0, "tie": 0, "win": 60 }, "flash37": { "loss": 26, "tie": 2, "win": 32 }, "glm": { "loss": 11, "tie": 0, "win": 49 }, "kimi": { "loss": 29, "tie": 0, "win": 31 }, "luna": { "loss": 31, "tie": 1, "win": 28 }, "opus": { "loss": 41, "tie": 1, "win": 18 }, "opus5": { "loss": 41, "tie": 0, "win": 19 }, "pro": { "loss": 5, "tie": 1, "win": 54 }, "sol": { "loss": 44, "tie": 1, "win": 15 }, "sonnet": { "loss": 29, "tie": 2, "win": 29 }, "terra": { "loss": 36, "tie": 0, "win": 24 } }, "flash37": { "dsv4": { "loss": 31, "tie": 0, "win": 29 }, "flash": { "loss": 0, "tie": 0, "win": 60 }, "flash36": { "loss": 32, "tie": 2, "win": 26 }, "glm": { "loss": 17, "tie": 0, "win": 43 }, "kimi": { "loss": 32, "tie": 1, "win": 27 }, "luna": { "loss": 37, "tie": 1, "win": 22 }, "opus": { "loss": 44, "tie": 1, "win": 15 }, "opus5": { "loss": 40, "tie": 3, "win": 17 }, "pro": { "loss": 8, "tie": 0, "win": 52 }, "sol": { "loss": 44, "tie": 3, "win": 13 }, "sonnet": { "loss": 32, "tie": 1, "win": 27 }, "terra": { "loss": 36, "tie": 1, "win": 23 } }, "glm": { "dsv4": { "loss": 50, "tie": 1, "win": 9 }, "flash": { "loss": 3, "tie": 0, "win": 57 }, "flash36": { "loss": 49, "tie": 0, "win": 11 }, "flash37": { "loss": 43, "tie": 0, "win": 17 }, "kimi": { "loss": 40, "tie": 1, "win": 19 }, "luna": { "loss": 48, "tie": 2, "win": 10 }, "opus": { "loss": 51, "tie": 0, "win": 9 }, "opus5": { "loss": 53, "tie": 1, "win": 6 }, "pro": { "loss": 29, "tie": 0, "win": 31 }, "sol": { "loss": 50, "tie": 1, "win": 9 }, "sonnet": { "loss": 51, "tie": 0, "win": 9 }, "terra": { "loss": 49, "tie": 0, "win": 11 } }, "kimi": { "dsv4": { "loss": 33, "tie": 0, "win": 27 }, "flash": { "loss": 2, "tie": 0, "win": 58 }, "flash36": { "loss": 31, "tie": 0, "win": 29 }, "flash37": { "loss": 27, "tie": 1, "win": 32 }, "glm": { "loss": 19, "tie": 1, "win": 40 }, "luna": { "loss": 37, "tie": 0, "win": 23 }, "opus": { "loss": 37, "tie": 0, "win": 23 }, "opus5": { "loss": 44, "tie": 0, "win": 16 }, "pro": { "loss": 15, "tie": 0, "win": 45 }, "sol": { "loss": 40, "tie": 0, "win": 20 }, "sonnet": { "loss": 31, "tie": 0, "win": 29 }, "terra": { "loss": 34, "tie": 0, "win": 26 } }, "luna": { "dsv4": { "loss": 23, "tie": 1, "win": 36 }, "flash": { "loss": 1, "tie": 0, "win": 59 }, "flash36": { "loss": 28, "tie": 1, "win": 31 }, "flash37": { "loss": 22, "tie": 1, "win": 37 }, "glm": { "loss": 10, "tie": 2, "win": 48 }, "kimi": { "loss": 23, "tie": 0, "win": 37 }, "opus": { "loss": 37, "tie": 1, "win": 22 }, "opus5": { "loss": 39, "tie": 1, "win": 20 }, "pro": { "loss": 4, "tie": 1, "win": 55 }, "sol": { "loss": 39, "tie": 1, "win": 20 }, "sonnet": { "loss": 29, "tie": 1, "win": 30 }, "terra": { "loss": 27, "tie": 2, "win": 31 } }, "opus": { "dsv4": { "loss": 19, "tie": 0, "win": 41 }, "flash": { "loss": 1, "tie": 0, "win": 59 }, "flash36": { "loss": 18, "tie": 1, "win": 41 }, "flash37": { "loss": 15, "tie": 1, "win": 44 }, "glm": { "loss": 9, "tie": 0, "win": 51 }, "kimi": { "loss": 23, "tie": 0, "win": 37 }, "luna": { "loss": 22, "tie": 1, "win": 37 }, "opus5": { "loss": 33, "tie": 3, "win": 24 }, "pro": { "loss": 3, "tie": 0, "win": 57 }, "sol": { "loss": 33, "tie": 0, "win": 27 }, "sonnet": { "loss": 21, "tie": 3, "win": 36 }, "terra": { "loss": 22, "tie": 2, "win": 36 } }, "opus5": { "dsv4": { "loss": 14, "tie": 1, "win": 45 }, "flash": { "loss": 1, "tie": 0, "win": 59 }, "flash36": { "loss": 19, "tie": 0, "win": 41 }, "flash37": { "loss": 17, "tie": 3, "win": 40 }, "glm": { "loss": 6, "tie": 1, "win": 53 }, "kimi": { "loss": 16, "tie": 0, "win": 44 }, "luna": { "loss": 20, "tie": 1, "win": 39 }, "opus": { "loss": 24, "tie": 3, "win": 33 }, "pro": { "loss": 6, "tie": 0, "win": 54 }, "sol": { "loss": 24, "tie": 2, "win": 34 }, "sonnet": { "loss": 16, "tie": 1, "win": 43 }, "terra": { "loss": 19, "tie": 0, "win": 41 } }, "pro": { "dsv4": { "loss": 49, "tie": 0, "win": 11 }, "flash": { "loss": 4, "tie": 0, "win": 56 }, "flash36": { "loss": 54, "tie": 1, "win": 5 }, "flash37": { "loss": 52, "tie": 0, "win": 8 }, "glm": { "loss": 31, "tie": 0, "win": 29 }, "kimi": { "loss": 45, "tie": 0, "win": 15 }, "luna": { "loss": 55, "tie": 1, "win": 4 }, "opus": { "loss": 57, "tie": 0, "win": 3 }, "opus5": { "loss": 54, "tie": 0, "win": 6 }, "sol": { "loss": 58, "tie": 0, "win": 2 }, "sonnet": { "loss": 53, "tie": 0, "win": 7 }, "terra": { "loss": 55, "tie": 1, "win": 4 } }, "sol": { "dsv4": { "loss": 20, "tie": 1, "win": 39 }, "flash": { "loss": 0, "tie": 0, "win": 60 }, "flash36": { "loss": 15, "tie": 1, "win": 44 }, "flash37": { "loss": 13, "tie": 3, "win": 44 }, "glm": { "loss": 9, "tie": 1, "win": 50 }, "kimi": { "loss": 20, "tie": 0, "win": 40 }, "luna": { "loss": 20, "tie": 1, "win": 39 }, "opus": { "loss": 27, "tie": 0, "win": 33 }, "opus5": { "loss": 34, "tie": 2, "win": 24 }, "pro": { "loss": 2, "tie": 0, "win": 58 }, "sonnet": { "loss": 16, "tie": 3, "win": 41 }, "terra": { "loss": 13, "tie": 7, "win": 40 } }, "sonnet": { "dsv4": { "loss": 24, "tie": 2, "win": 34 }, "flash": { "loss": 2, "tie": 0, "win": 58 }, "flash36": { "loss": 29, "tie": 2, "win": 29 }, "flash37": { "loss": 27, "tie": 1, "win": 32 }, "glm": { "loss": 9, "tie": 0, "win": 51 }, "kimi": { "loss": 29, "tie": 0, "win": 31 }, "luna": { "loss": 30, "tie": 1, "win": 29 }, "opus": { "loss": 36, "tie": 3, "win": 21 }, "opus5": { "loss": 43, "tie": 1, "win": 16 }, "pro": { "loss": 7, "tie": 0, "win": 53 }, "sol": { "loss": 41, "tie": 3, "win": 16 }, "terra": { "loss": 31, "tie": 1, "win": 28 } }, "terra": { "dsv4": { "loss": 26, "tie": 0, "win": 34 }, "flash": { "loss": 0, "tie": 0, "win": 60 }, "flash36": { "loss": 24, "tie": 0, "win": 36 }, "flash37": { "loss": 23, "tie": 1, "win": 36 }, "glm": { "loss": 11, "tie": 0, "win": 49 }, "kimi": { "loss": 26, "tie": 0, "win": 34 }, "luna": { "loss": 31, "tie": 2, "win": 27 }, "opus": { "loss": 36, "tie": 2, "win": 22 }, "opus5": { "loss": 41, "tie": 0, "win": 19 }, "pro": { "loss": 4, "tie": 1, "win": 55 }, "sol": { "loss": 40, "tie": 7, "win": 13 }, "sonnet": { "loss": 28, "tie": 1, "win": 31 } } }, "models": { "dsv4": { "avg_score": 0.5811, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 81, "checks_total": 3760, "display": "deepseek-v4-pro", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'REAPP-BP-2026-0842-AO': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r04" }, { "message": "files_created.content contains 'POL-SEC-RECON-3.4': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r03" }, { "message": "files_created.content contains 'SEC-RUNBOOK-REL-8842': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r09" }, { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" } ], "content_file": [ { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The report finds that the request failed reconciliation and permit cannot be issued, contradicting the requirement to state that all reconciliation checks passed. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "LLM semantic match: The report explicitly states that the permit cannot be issued and access is not granted, failing to satisfy the criterion to proceed with the access grant. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" } ], "content_msg": [ { "message": "LLM semantic match: No message was sent to Sana_Security_Reviewer in #iam-approvals-sec containing the required permit PERMIT-2026-BP-9982 or report path /reports/access_grant_decision.md. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The message requested re-approval for WRITE scope on billing-prod from Ravi_Application_Owner in #iam-approvals-sec, but cited AUDIT-APP-2026-ARCH instead of AUDIT-SEC-2026-READ. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: No message was sent to Sana_Security_Reviewer in #iam-approvals-sec informing her of the Application Owner's TARGET-MISMATCH (billing-archive) status on REQ-2026-0842. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" }, { "message": "LLM semantic match: The message to Sana_Security_Reviewer in #iam-approvals-sec confirmed the corrected WRITE approval for billing-prod, but omitted the tracking code REAPP-BP-2026-0842-AO. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r04" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r07': False; normalized fallback disabled ('iam_console_access_scope_matrix_r07' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20241024_085500': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20241024_085500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v02_r07" }, { "message": "assets_accessed.id contains 'audit_dashboard_sync_telemetry_20250218_110000': False; normalized fallback disabled ('audit_dashboard_sync_telemetry_20250218_110000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r09" }, { "message": "assets_accessed.id contains 'ops_console_canary_cutover_telemetry_20260330_141500': False; normalized fallback disabled ('ops_console_canary_cutover_telemetry_20260330_141500' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r06" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r04" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 80 }, "content_file": { "diagnostic": 0, "required": 263 }, "content_msg": { "diagnostic": 0, "required": 652 }, "grounding": { "diagnostic": 61, "required": 30 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 337 }, "window": { "diagnostic": 0, "required": 27 } }, "failed_total": 1450, "gating": { "content_involved": 430, "grounding_only": 0, "grounding_only_ids": [], "n_fail": 481, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 7, "score": 0.375, "task_id": "conflict_vlm_access_request_approval_v01_r01", "verb": "asset open" }, { "count": 10, "score": 0.2, "task_id": "conflict_vlm_access_request_approval_v02_r03", "verb": "message search" }, { "count": 12, "score": 0.6666666666666666, "task_id": "conflict_vlm_access_request_approval_v02_r09", "verb": "message search" }, { "count": 6, "score": 0.5, "task_id": "conflict_vlm_barrier_sync_deployment_v02_r07", "verb": "agent status" } ], "loop_verbs": { "agent status": 1, "asset open": 43, "file read": 3, "message search": 27, "message send": 1 }, "loopy_capped": 75, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 265, "ok_false": 0, "pass_pct": 19.8, "passed": 119, "rank": { "avg_score": 9, "capped": 7, "failed_total": 7, "loopy_capped": 7, "near_miss": 8, "pass_pct": 8, "step_mean": 6, "step_p50": 6, "zero_open_tasks": 1, "zero_score": 8, "zero_write_tasks": 7 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.6099, "n": 10, "passed": 1 }, "access_request_approval_v02": { "mean_score": 0.6825, "n": 10, "passed": 1 }, "access_request_approval_v03": { "mean_score": 0.6774, "n": 10, "passed": 3 }, "barrier_sync_deployment_v01": { "mean_score": 0.6153, "n": 10, "passed": 2 }, "barrier_sync_deployment_v02": { "mean_score": 0.6088, "n": 10, "passed": 2 }, "barrier_sync_deployment_v03": { "mean_score": 0.69, "n": 10, "passed": 2 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.3967, "n": 10, "passed": 2 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.6024, "n": 10, "passed": 4 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.4485, "n": 10, "passed": 1 }, "collaborative_contract_review_v01": { "mean_score": 0.6492, "n": 10, "passed": 3 }, "collaborative_contract_review_v02": { "mean_score": 0.7267, "n": 10, "passed": 3 }, "collaborative_contract_review_v03": { "mean_score": 0.6301, "n": 10, "passed": 1 }, "coordinated_containment_v01": { "mean_score": 0.3565, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.6037, "n": 10, "passed": 1 }, "coordinated_containment_v03": { "mean_score": 0.691, "n": 10, "passed": 3 }, "deadlock_multi_approval_v01": { "mean_score": 0.6267, "n": 10, "passed": 0 }, "deadlock_multi_approval_v02": { "mean_score": 0.7467, "n": 10, "passed": 4 }, "deadlock_multi_approval_v03": { "mean_score": 0.695, "n": 10, "passed": 3 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.6414, "n": 10, "passed": 3 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.3777, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.3792, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v01": { "mean_score": 0.5033, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v02": { "mean_score": 0.5745, "n": 10, "passed": 2 }, "employee_onboarding_workflow_v03": { "mean_score": 0.4942, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.725, "n": 10, "passed": 5 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.64, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.7, "n": 10, "passed": 2 }, "expiring_resource_hold_v01": { "mean_score": 0.6571, "n": 10, "passed": 2 }, "expiring_resource_hold_v02": { "mean_score": 0.5298, "n": 10, "passed": 1 }, "expiring_resource_hold_v03": { "mean_score": 0.8107, "n": 10, "passed": 1 }, "incident_commander_selection_v01": { "mean_score": 0.4583, "n": 10, "passed": 2 }, "incident_commander_selection_v02": { "mean_score": 0.545, "n": 10, "passed": 1 }, "incident_commander_selection_v03": { "mean_score": 0.465, "n": 10, "passed": 1 }, "it_deployment_gate_v01": { "mean_score": 0.6595, "n": 10, "passed": 4 }, "it_deployment_gate_v02": { "mean_score": 0.6618, "n": 10, "passed": 4 }, "it_deployment_gate_v03": { "mean_score": 0.7082, "n": 10, "passed": 3 }, "multi_device_pairing_v01": { "mean_score": 0.5806, "n": 10, "passed": 1 }, "multi_device_pairing_v02": { "mean_score": 0.6559, "n": 10, "passed": 2 }, "multi_device_pairing_v03": { "mean_score": 0.608, "n": 10, "passed": 3 }, "multi_party_access_permit_v01": { "mean_score": 0.6117, "n": 10, "passed": 1 }, "multi_party_access_permit_v02": { "mean_score": 0.8262, "n": 10, "passed": 6 }, "multi_party_access_permit_v03": { "mean_score": 0.6295, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.585, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.3754, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.5671, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.3433, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.34, "n": 10, "passed": 0 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.35, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v01": { "mean_score": 0.9083, "n": 10, "passed": 7 }, "payment_chargeback_dispute_v02": { "mean_score": 0.4917, "n": 10, "passed": 2 }, "payment_chargeback_dispute_v03": { "mean_score": 0.4333, "n": 10, "passed": 2 }, "satellite_ground_station_handover_v01": { "mean_score": 0.4833, "n": 10, "passed": 2 }, "satellite_ground_station_handover_v02": { "mean_score": 0.6667, "n": 10, "passed": 3 }, "satellite_ground_station_handover_v03": { "mean_score": 0.5417, "n": 10, "passed": 4 }, "sla_support_response_v01": { "mean_score": 0.5695, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.6722, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.4501, "n": 10, "passed": 0 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.6417, "n": 10, "passed": 4 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.5833, "n": 10, "passed": 3 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.3667, "n": 10, "passed": 1 } }, "semantic": { "fail_closed": 99, "fail_closed_pct": 4.6, "n": 2137 }, "side_effect_fields": { "approval_escalated": 7, "approval_granted": 3, "approval_recorded": 17, "approval_requested": 21, "assets_accessed": 21, "booking_extended": 4, "booking_held": 22, "booking_released": 8, "containment_action": 20, "deployment_approved": 6, "deployment_cutover": 1, "deployment_gate_set": 11, "deployment_requested": 31, "document_signed": 2, "files_created": 1, "lock_acquired": 14, "messages_sent": 4, "pairing_code_entered": 3, "pairing_code_shown": 1, "pairing_completed": 1, "pairing_started": 11, "review_assigned": 5, "review_submitted": 4 }, "step_mean": 13.42, "step_p50": 13, "stops": { "incomplete_required_submission": 182, "missing_required_side_effect": 218, "none": 119, "reached max_steps=20": 81 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 10": 37, "wait 15": 8, "wait 20": 1, "wait 30": 7, "wait 300": 4, "wait 5": 106, "wait 60": 1 }, "max": 300.0, "mean": 15.3, "median": 5.0, "n": 164, "sub_30": 152, "total": 2510.0 }, "zero_open_tasks": 0, "zero_score": 41, "zero_write_tasks": 43 }, "flash": { "avg_score": 0.2594, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 535, "checks_total": 3760, "display": "gemini-3.5-flash", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'AUDIT-SEC-2026-READ': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "files_created.content contains 'SEC-TELEMETRY-9914-BP': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "files_created.content contains 'PERMIT-2026-BP-9982': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r08" }, { "message": "files_created.content contains 'ERR-SCOPE-MISMATCH-8842-SEC': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r02" } ], "content_file": [ { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "content_msg": [ { "message": "LLM semantic match: no 'message' content in messages_sent to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'message' content in messages_sent to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'message' content in messages_sent to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: A message was sent to Sana_Security_Reviewer in #iam-approvals-sec citing AUDIT-SEC-2026-READ regarding WRITE access on billing-prod, but it failed to include the required passcode SEC-PASS-8842-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r07': False; normalized fallback disabled ('iam_console_access_scope_matrix_r07' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "assets_accessed.id contains 'security_dashboard_telemetry_20260330_100000': False; normalized fallback disabled ('security_dashboard_telemetry_20260330_100000' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r08" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20241024_085500': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20241024_085500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v02_r07" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20250218_103000': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20250218_103000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r07" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r04" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 211 }, "content_file": { "diagnostic": 0, "required": 765 }, "content_msg": { "diagnostic": 0, "required": 955 }, "grounding": { "diagnostic": 72, "required": 32 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 371 }, "window": { "diagnostic": 0, "required": 28 } }, "failed_total": 2434, "gating": { "content_involved": 567, "grounding_only": 0, "grounding_only_ids": [], "n_fail": 582, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 7, "score": 0.125, "task_id": "conflict_vlm_access_request_approval_v01_r01", "verb": "asset open" }, { "count": 6, "score": 0.2857142857142857, "task_id": "conflict_vlm_access_request_approval_v01_r02", "verb": "file read" }, { "count": 7, "score": 0.42857142857142855, "task_id": "conflict_vlm_access_request_approval_v01_r03", "verb": "file read" }, { "count": 8, "score": 0.14285714285714285, "task_id": "conflict_vlm_access_request_approval_v01_r05", "verb": "file read" } ], "loop_verbs": { "asset info": 4, "asset open": 99, "booking status": 3, "file read": 241, "message search": 164, "message send": 2 }, "loopy_capped": 517, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 100, "ok_false": 0, "pass_pct": 3.0, "passed": 18, "rank": { "avg_score": 13, "capped": 13, "failed_total": 13, "loopy_capped": 13, "near_miss": 1, "pass_pct": 13, "step_mean": 13, "step_p50": 13, "zero_open_tasks": 11, "zero_score": 13, "zero_write_tasks": 13 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.3182, "n": 10, "passed": 1 }, "access_request_approval_v02": { "mean_score": 0.275, "n": 10, "passed": 0 }, "access_request_approval_v03": { "mean_score": 0.5623, "n": 10, "passed": 2 }, "barrier_sync_deployment_v01": { "mean_score": 0.327, "n": 10, "passed": 1 }, "barrier_sync_deployment_v02": { "mean_score": 0.3152, "n": 10, "passed": 0 }, "barrier_sync_deployment_v03": { "mean_score": 0.3017, "n": 10, "passed": 0 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.07, "n": 10, "passed": 0 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.0, "n": 10, "passed": 0 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.1944, "n": 10, "passed": 0 }, "collaborative_contract_review_v01": { "mean_score": 0.292, "n": 10, "passed": 1 }, "collaborative_contract_review_v02": { "mean_score": 0.5233, "n": 10, "passed": 1 }, "collaborative_contract_review_v03": { "mean_score": 0.3011, "n": 10, "passed": 0 }, "coordinated_containment_v01": { "mean_score": 0.1236, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.1386, "n": 10, "passed": 0 }, "coordinated_containment_v03": { "mean_score": 0.5133, "n": 10, "passed": 0 }, "deadlock_multi_approval_v01": { "mean_score": 0.4658, "n": 10, "passed": 2 }, "deadlock_multi_approval_v02": { "mean_score": 0.473, "n": 10, "passed": 1 }, "deadlock_multi_approval_v03": { "mean_score": 0.3879, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.2662, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.1152, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.325, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v01": { "mean_score": 0.2338, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v02": { "mean_score": 0.3233, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v03": { "mean_score": 0.445, "n": 10, "passed": 0 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.22, "n": 10, "passed": 0 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.32, "n": 10, "passed": 0 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.3138, "n": 10, "passed": 0 }, "expiring_resource_hold_v01": { "mean_score": 0.2526, "n": 10, "passed": 0 }, "expiring_resource_hold_v02": { "mean_score": 0.3917, "n": 10, "passed": 0 }, "expiring_resource_hold_v03": { "mean_score": 0.2379, "n": 10, "passed": 0 }, "incident_commander_selection_v01": { "mean_score": 0.0, "n": 10, "passed": 0 }, "incident_commander_selection_v02": { "mean_score": 0.3417, "n": 10, "passed": 1 }, "incident_commander_selection_v03": { "mean_score": 0.09, "n": 10, "passed": 0 }, "it_deployment_gate_v01": { "mean_score": 0.3504, "n": 10, "passed": 0 }, "it_deployment_gate_v02": { "mean_score": 0.489, "n": 10, "passed": 2 }, "it_deployment_gate_v03": { "mean_score": 0.2967, "n": 10, "passed": 0 }, "multi_device_pairing_v01": { "mean_score": 0.3333, "n": 10, "passed": 0 }, "multi_device_pairing_v02": { "mean_score": 0.2455, "n": 10, "passed": 0 }, "multi_device_pairing_v03": { "mean_score": 0.3701, "n": 10, "passed": 1 }, "multi_party_access_permit_v01": { "mean_score": 0.2317, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.4512, "n": 10, "passed": 1 }, "multi_party_access_permit_v03": { "mean_score": 0.261, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.1616, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.1721, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.2596, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.21, "n": 10, "passed": 0 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.0583, "n": 10, "passed": 0 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.2433, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v01": { "mean_score": 0.275, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v02": { "mean_score": 0.075, "n": 10, "passed": 0 }, "payment_chargeback_dispute_v03": { "mean_score": 0.0333, "n": 10, "passed": 0 }, "satellite_ground_station_handover_v01": { "mean_score": 0.0333, "n": 10, "passed": 0 }, "satellite_ground_station_handover_v02": { "mean_score": 0.0, "n": 10, "passed": 0 }, "satellite_ground_station_handover_v03": { "mean_score": 0.0667, "n": 10, "passed": 0 }, "sla_support_response_v01": { "mean_score": 0.389, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.3802, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.3135, "n": 10, "passed": 0 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.2417, "n": 10, "passed": 1 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.1, "n": 10, "passed": 0 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.0667, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 1222, "fail_closed_pct": 57.2, "n": 2137 }, "side_effect_fields": { "approval_granted": 1, "approval_requested": 2, "assets_accessed": 3, "booking_held": 1, "booking_released": 2, "containment_action": 1, "deployment_approved": 1, "deployment_gate_set": 3, "deployment_requested": 6, "lock_acquired": 1, "pairing_code_entered": 2, "pairing_started": 1, "review_assigned": 2 }, "step_mean": 19.7, "step_p50": 20, "stops": { "generation_stall_empty_reply": 1, "incomplete_required_submission": 19, "missing_required_side_effect": 26, "none": 18, "reached max_steps=20": 535, "stopped after 3 identical no-progress transitions": 1 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 10": 130, "wait 120": 1, "wait 20": 2, "wait 30": 27, "wait 300": 3, "wait 5": 99, "wait 60": 5 }, "max": 300.0, "mean": 14.9, "median": 10.0, "n": 267, "sub_30": 231, "total": 3965.0 }, "zero_open_tasks": 18, "zero_score": 188, "zero_write_tasks": 440 }, "flash36": { "avg_score": 0.5857, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 148, "checks_total": 3760, "display": "gemini-3.6-flash", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'PERMIT-2026-BP-9982': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r08" }, { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'PERMIT-BILLING-PROD-9982': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r07" }, { "message": "files_created.content contains 'ERR-DATA-BACKFILL-409': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r03" } ], "content_file": [ { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r08" }, { "message": "LLM semantic match: The report references transaction ID TXN-APP-WRITE-9912 instead of the expected TXN-APP-WRITE-9908. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r04" } ], "content_msg": [ { "message": "LLM semantic match: No message was sent to Sana_Security_Reviewer in #iam-approvals-sec citing permit PERMIT-2026-BP-9982 and report path /reports/access_grant_decision.md. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The message sent to Sana_Security_Reviewer in #iam-approvals-sec requested updated WRITE authorization citing AUDIT-SEC-2026-READ, but omitted the passcode SEC-PASS-8842-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: The message in #iam-approvals-sec addressed Ravi_Application_Owner to request re-approval for billing-prod, but cited reference code AUDIT-APP-2026-ARCH instead of AUDIT-SEC-2026-READ. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: A message was sent to Quinn in #proj-billing-migration regarding the dashboard audit and WRITE scope on billing-prod, but it reported the status as 'VERIFIED with 5/5 reconciliation layers cleared' rather than 'ALL RECONCILIATION CHECKS PAS", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r01': False; normalized fallback disabled ('iam_console_access_scope_matrix_r01' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20241024_085500': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20241024_085500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v02_r07" }, { "message": "assets_accessed.id contains 'audit_dashboard_passcode_telemetry_20250218_103000': False; normalized fallback disabled ('audit_dashboard_passcode_telemetry_20250218_103000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r06" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20250218_103000': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20250218_103000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r07" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r04" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 79 }, "content_file": { "diagnostic": 0, "required": 288 }, "content_msg": { "diagnostic": 0, "required": 647 }, "grounding": { "diagnostic": 69, "required": 31 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 308 }, "window": { "diagnostic": 0, "required": 27 } }, "failed_total": 1449, "gating": { "content_involved": 422, "grounding_only": 3, "grounding_only_ids": [ "conflict_vlm_barrier_sync_deployment_v01_r07", "conflict_vlm_distributed_lock_shared_resource_v03_r06", "conflict_vlm_it_deployment_gate_v03_r10" ], "n_fail": 489, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 5, "score": 0.8571428571428571, "task_id": "conflict_vlm_access_request_approval_v01_r05", "verb": "asset open" }, { "count": 6, "score": 0.1, "task_id": "conflict_vlm_access_request_approval_v01_r08", "verb": "file read" }, { "count": 6, "score": 0.6, "task_id": "conflict_vlm_access_request_approval_v02_r03", "verb": "asset open" }, { "count": 6, "score": 0.6, "task_id": "conflict_vlm_access_request_approval_v02_r06", "verb": "file read" } ], "loop_verbs": { "asset info": 2, "asset open": 38, "deploy status": 2, "file read": 71, "message search": 5, "message send": 7 }, "loopy_capped": 127, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 272, "ok_false": 0, "pass_pct": 18.5, "passed": 111, "rank": { "avg_score": 7, "capped": 8, "failed_total": 6, "loopy_capped": 9, "near_miss": 10, "pass_pct": 10, "step_mean": 11, "step_p50": 11, "zero_open_tasks": 9, "zero_score": 6, "zero_write_tasks": 9 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.7036, "n": 10, "passed": 2 }, "access_request_approval_v02": { "mean_score": 0.71, "n": 10, "passed": 1 }, "access_request_approval_v03": { "mean_score": 0.6957, "n": 10, "passed": 2 }, "barrier_sync_deployment_v01": { "mean_score": 0.6879, "n": 10, "passed": 2 }, "barrier_sync_deployment_v02": { "mean_score": 0.5969, "n": 10, "passed": 3 }, "barrier_sync_deployment_v03": { "mean_score": 0.6667, "n": 10, "passed": 3 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.38, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.3131, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.3946, "n": 10, "passed": 1 }, "collaborative_contract_review_v01": { "mean_score": 0.6932, "n": 10, "passed": 3 }, "collaborative_contract_review_v02": { "mean_score": 0.795, "n": 10, "passed": 6 }, "collaborative_contract_review_v03": { "mean_score": 0.7419, "n": 10, "passed": 3 }, "coordinated_containment_v01": { "mean_score": 0.3552, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.646, "n": 10, "passed": 1 }, "coordinated_containment_v03": { "mean_score": 0.6543, "n": 10, "passed": 1 }, "deadlock_multi_approval_v01": { "mean_score": 0.6675, "n": 10, "passed": 3 }, "deadlock_multi_approval_v02": { "mean_score": 0.8592, "n": 10, "passed": 3 }, "deadlock_multi_approval_v03": { "mean_score": 0.7121, "n": 10, "passed": 2 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.6848, "n": 10, "passed": 3 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.3498, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.4286, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v01": { "mean_score": 0.5502, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v02": { "mean_score": 0.5923, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v03": { "mean_score": 0.5735, "n": 10, "passed": 2 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.6867, "n": 10, "passed": 4 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.71, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.6893, "n": 10, "passed": 1 }, "expiring_resource_hold_v01": { "mean_score": 0.6955, "n": 10, "passed": 4 }, "expiring_resource_hold_v02": { "mean_score": 0.6012, "n": 10, "passed": 1 }, "expiring_resource_hold_v03": { "mean_score": 0.6512, "n": 10, "passed": 0 }, "incident_commander_selection_v01": { "mean_score": 0.4583, "n": 10, "passed": 1 }, "incident_commander_selection_v02": { "mean_score": 0.6433, "n": 10, "passed": 2 }, "incident_commander_selection_v03": { "mean_score": 0.5517, "n": 10, "passed": 2 }, "it_deployment_gate_v01": { "mean_score": 0.5844, "n": 10, "passed": 2 }, "it_deployment_gate_v02": { "mean_score": 0.6271, "n": 10, "passed": 2 }, "it_deployment_gate_v03": { "mean_score": 0.6921, "n": 10, "passed": 2 }, "multi_device_pairing_v01": { "mean_score": 0.6278, "n": 10, "passed": 2 }, "multi_device_pairing_v02": { "mean_score": 0.4881, "n": 10, "passed": 0 }, "multi_device_pairing_v03": { "mean_score": 0.5677, "n": 10, "passed": 2 }, "multi_party_access_permit_v01": { "mean_score": 0.505, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.7262, "n": 10, "passed": 4 }, "multi_party_access_permit_v03": { "mean_score": 0.6895, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.4477, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.471, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.5518, "n": 10, "passed": 0 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.42, "n": 10, "passed": 2 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.4983, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.47, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v01": { "mean_score": 0.8333, "n": 10, "passed": 7 }, "payment_chargeback_dispute_v02": { "mean_score": 0.4833, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v03": { "mean_score": 0.5333, "n": 10, "passed": 2 }, "satellite_ground_station_handover_v01": { "mean_score": 0.475, "n": 10, "passed": 3 }, "satellite_ground_station_handover_v02": { "mean_score": 0.7667, "n": 10, "passed": 6 }, "satellite_ground_station_handover_v03": { "mean_score": 0.6, "n": 10, "passed": 3 }, "sla_support_response_v01": { "mean_score": 0.5329, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.6667, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.4835, "n": 10, "passed": 1 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.5683, "n": 10, "passed": 3 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.4833, "n": 10, "passed": 4 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.2117, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 185, "fail_closed_pct": 8.7, "n": 2137 }, "side_effect_fields": { "approval_escalated": 5, "approval_granted": 2, "approval_recorded": 17, "approval_requested": 15, "assets_accessed": 21, "booking_extended": 4, "booking_held": 16, "booking_released": 6, "containment_action": 13, "deployment_approved": 4, "deployment_gate_set": 5, "deployment_requested": 25, "lock_acquired": 14, "lock_released": 1, "messages_sent": 5, "pairing_code_entered": 3, "pairing_started": 7, "review_assigned": 3, "review_submitted": 5 }, "step_mean": 16.31, "step_p50": 17, "stops": { "incomplete_required_submission": 170, "missing_required_side_effect": 171, "none": 111, "reached max_steps=20": 148 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 10": 29, "wait 20": 1, "wait 300": 4, "wait 5": 23 }, "max": 300.0, "mean": 28.5, "median": 10.0, "n": 57, "sub_30": 53, "total": 1625.0 }, "zero_open_tasks": 5, "zero_score": 35, "zero_write_tasks": 74 }, "flash37": { "avg_score": 0.5814, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 244, "checks_total": 3760, "display": "gemini-3.7-flash", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'PERMIT-2026-BP-9982': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r08" }, { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'PERMIT-BILLING-PROD-9982': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r07" }, { "message": "files_created.content contains 'FULLY_RECONCILED_AND_GRANTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r10" } ], "content_file": [ { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r08" }, { "message": "LLM semantic match: The report states transaction ID TXN-APP-WRITE-9912 instead of TXN-APP-WRITE-9908. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r04" } ], "content_msg": [ { "message": "LLM semantic match: No message was sent to Sana_Security_Reviewer containing permit PERMIT-2026-BP-9982, report path /reports/access_grant_decision.md, or authorizing the release of the runbook to Tara_IAM_Provisioning_Agent. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The message was sent to Sana in #iam-approvals-sec citing AUDIT-SEC-2026-READ for updated WRITE authorization on billing-prod, but it omitted the required passcode SEC-PASS-8842-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: The message was sent to Ravi in #iam-approvals-sec requesting re-approval for WRITE on billing-prod, but cited AUDIT-APP-2026-ARCH instead of AUDIT-SEC-2026-READ. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: The message was sent to Quinn in #proj-billing-migration confirming verification with Sana, but cited passcode SEC-TELEMETRY-9840-BP instead of SEC-TELEMETRY-9914-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r01': False; normalized fallback disabled ('iam_console_access_scope_matrix_r01' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20241024_085500': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20241024_085500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v02_r07" }, { "message": "assets_accessed.id contains 'audit_dashboard_passcode_telemetry_20250218_103000': False; normalized fallback disabled ('audit_dashboard_passcode_telemetry_20250218_103000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r06" }, { "message": "assets_accessed.id contains 'ops_console_gate_controller_metrics_20260330_143000': False; normalized fallback disabled ('ops_console_gate_controller_metrics_20260330_143000' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r04" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r04" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r05" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 84 }, "content_file": { "diagnostic": 0, "required": 324 }, "content_msg": { "diagnostic": 0, "required": 627 }, "grounding": { "diagnostic": 89, "required": 35 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 299 }, "window": { "diagnostic": 0, "required": 26 } }, "failed_total": 1484, "gating": { "content_involved": 418, "grounding_only": 0, "grounding_only_ids": [], "n_fail": 478, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 7, "score": 0.8, "task_id": "conflict_vlm_access_request_approval_v01_r09", "verb": "file read" }, { "count": 8, "score": 0.6, "task_id": "conflict_vlm_access_request_approval_v02_r03", "verb": "file read" }, { "count": 5, "score": 0.6, "task_id": "conflict_vlm_access_request_approval_v02_r05", "verb": "file read" }, { "count": 5, "score": 0.8333333333333334, "task_id": "conflict_vlm_access_request_approval_v02_r09", "verb": "file read" } ], "loop_verbs": { "agent status": 1, "asset info": 3, "asset open": 33, "file read": 150, "message search": 3, "message send": 12 }, "loopy_capped": 202, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 254, "ok_false": 0, "pass_pct": 20.3, "passed": 122, "rank": { "avg_score": 8, "capped": 12, "failed_total": 8, "loopy_capped": 11, "near_miss": 5, "pass_pct": 6, "step_mean": 12, "step_p50": 12, "zero_open_tasks": 12, "zero_score": 9, "zero_write_tasks": 10 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.715, "n": 10, "passed": 3 }, "access_request_approval_v02": { "mean_score": 0.6867, "n": 10, "passed": 0 }, "access_request_approval_v03": { "mean_score": 0.6235, "n": 10, "passed": 1 }, "barrier_sync_deployment_v01": { "mean_score": 0.6429, "n": 10, "passed": 2 }, "barrier_sync_deployment_v02": { "mean_score": 0.6288, "n": 10, "passed": 2 }, "barrier_sync_deployment_v03": { "mean_score": 0.6, "n": 10, "passed": 2 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.4333, "n": 10, "passed": 2 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.4869, "n": 10, "passed": 4 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.437, "n": 10, "passed": 0 }, "collaborative_contract_review_v01": { "mean_score": 0.5611, "n": 10, "passed": 1 }, "collaborative_contract_review_v02": { "mean_score": 0.7067, "n": 10, "passed": 6 }, "collaborative_contract_review_v03": { "mean_score": 0.7019, "n": 10, "passed": 2 }, "coordinated_containment_v01": { "mean_score": 0.4173, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.5813, "n": 10, "passed": 1 }, "coordinated_containment_v03": { "mean_score": 0.7638, "n": 10, "passed": 2 }, "deadlock_multi_approval_v01": { "mean_score": 0.6183, "n": 10, "passed": 2 }, "deadlock_multi_approval_v02": { "mean_score": 0.82, "n": 10, "passed": 3 }, "deadlock_multi_approval_v03": { "mean_score": 0.755, "n": 10, "passed": 3 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.6483, "n": 10, "passed": 5 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.364, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.3827, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v01": { "mean_score": 0.5697, "n": 10, "passed": 2 }, "employee_onboarding_workflow_v02": { "mean_score": 0.507, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v03": { "mean_score": 0.6229, "n": 10, "passed": 3 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.5283, "n": 10, "passed": 2 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.705, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.7331, "n": 10, "passed": 3 }, "expiring_resource_hold_v01": { "mean_score": 0.6202, "n": 10, "passed": 1 }, "expiring_resource_hold_v02": { "mean_score": 0.6036, "n": 10, "passed": 2 }, "expiring_resource_hold_v03": { "mean_score": 0.7521, "n": 10, "passed": 1 }, "incident_commander_selection_v01": { "mean_score": 0.4417, "n": 10, "passed": 1 }, "incident_commander_selection_v02": { "mean_score": 0.6967, "n": 10, "passed": 3 }, "incident_commander_selection_v03": { "mean_score": 0.665, "n": 10, "passed": 2 }, "it_deployment_gate_v01": { "mean_score": 0.5727, "n": 10, "passed": 3 }, "it_deployment_gate_v02": { "mean_score": 0.6582, "n": 10, "passed": 4 }, "it_deployment_gate_v03": { "mean_score": 0.635, "n": 10, "passed": 2 }, "multi_device_pairing_v01": { "mean_score": 0.5944, "n": 10, "passed": 2 }, "multi_device_pairing_v02": { "mean_score": 0.6092, "n": 10, "passed": 0 }, "multi_device_pairing_v03": { "mean_score": 0.6162, "n": 10, "passed": 3 }, "multi_party_access_permit_v01": { "mean_score": 0.5408, "n": 10, "passed": 1 }, "multi_party_access_permit_v02": { "mean_score": 0.7214, "n": 10, "passed": 4 }, "multi_party_access_permit_v03": { "mean_score": 0.6695, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.4121, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.5076, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.4896, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.4, "n": 10, "passed": 2 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.3483, "n": 10, "passed": 0 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.55, "n": 10, "passed": 3 }, "payment_chargeback_dispute_v01": { "mean_score": 0.8333, "n": 10, "passed": 7 }, "payment_chargeback_dispute_v02": { "mean_score": 0.425, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v03": { "mean_score": 0.5667, "n": 10, "passed": 3 }, "satellite_ground_station_handover_v01": { "mean_score": 0.375, "n": 10, "passed": 2 }, "satellite_ground_station_handover_v02": { "mean_score": 0.7333, "n": 10, "passed": 7 }, "satellite_ground_station_handover_v03": { "mean_score": 0.725, "n": 10, "passed": 5 }, "sla_support_response_v01": { "mean_score": 0.5452, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.6294, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.4277, "n": 10, "passed": 1 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.55, "n": 10, "passed": 3 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.5167, "n": 10, "passed": 3 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.2117, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 231, "fail_closed_pct": 10.8, "n": 2137 }, "side_effect_fields": { "approval_escalated": 6, "approval_granted": 3, "approval_recorded": 16, "approval_requested": 12, "assets_accessed": 15, "booking_extended": 4, "booking_held": 10, "booking_released": 4, "containment_action": 11, "deployment_approved": 5, "deployment_gate_set": 3, "deployment_requested": 10, "lock_acquired": 6, "lock_released": 1, "messages_sent": 5, "pairing_code_entered": 1, "pairing_started": 9, "review_assigned": 1, "review_submitted": 6 }, "step_mean": 17.79, "step_p50": 19, "stops": { "incomplete_required_submission": 106, "missing_required_side_effect": 128, "none": 122, "reached max_steps=20": 244 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 10": 13, "wait 15": 5, "wait 180": 2, "wait 30": 5, "wait 300": 4, "wait 5": 46 }, "max": 300.0, "mean": 28.6, "median": 5.0, "n": 75, "sub_30": 64, "total": 2145.0 }, "zero_open_tasks": 60, "zero_score": 42, "zero_write_tasks": 103 }, "glm": { "avg_score": 0.4867, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 207, "checks_total": 3760, "display": "glm-5.2", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'REAPP-BP-2026-0842-AO': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r04" }, { "message": "files_created.content contains 'SEC-TELEMETRY-9914-BP': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "files_created.content contains 'POL-SEC-RECON-3.4': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r03" }, { "message": "files_created.content contains 'SEC-RUNBOOK-REL-8842': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r09" } ], "content_file": [ { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" } ], "content_msg": [ { "message": "LLM semantic match: No message was sent to Sana_Security_Reviewer in #iam-approvals-sec citing permit PERMIT-2026-BP-9982 and report path /reports/access_grant_decision.md to authorize runbook release. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The message sent in #iam-approvals-sec requested updated WRITE authorization citing AUDIT-SEC-2026-READ, but failed to include the passcode SEC-PASS-8842-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: The message to Ravi in #iam-approvals-sec informs him of the TARGET-MISMATCH for billing-archive, but fails to explicitly request a corrected sign-off with WRITE scope for billing-prod. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" }, { "message": "LLM semantic match: The required message confirming that the corrected WRITE approval was registered under tracking code REAPP-BP-2026-0842-AO was not sent to Sana_Security_Reviewer. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r04" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20241024_085500': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20241024_085500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v02_r07" }, { "message": "assets_accessed.id contains 'deploy_console_candidates_20250214_083400': False; normalized fallback disabled ('deploy_console_candidates_20250214_083400' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v02_r02" }, { "message": "assets_accessed.id contains 'telemetry_dashboard_staging_ticket_status_20231104_082200': False; normalized fallback disabled ('telemetry_dashboard_staging_ticket_status_20231104_082200' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v03_r10" }, { "message": "assets_accessed.id contains 'pv_dashboard_interim_model_verification_20231112_110000': False; normalized fallback disabled ('pv_dashboard_interim_model_verification_20231112_110000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_clinical_trial_dosage_conflict_v01_r09" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r04" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 124 }, "content_file": { "diagnostic": 0, "required": 397 }, "content_msg": { "diagnostic": 0, "required": 772 }, "grounding": { "diagnostic": 40, "required": 16 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 361 }, "window": { "diagnostic": 0, "required": 30 } }, "failed_total": 1740, "gating": { "content_involved": 468, "grounding_only": 0, "grounding_only_ids": [], "n_fail": 512, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 7, "score": 0.375, "task_id": "conflict_vlm_access_request_approval_v01_r01", "verb": "asset open" }, { "count": 17, "score": 0.14285714285714285, "task_id": "conflict_vlm_access_request_approval_v01_r05", "verb": "asset open" }, { "count": 19, "score": 0.2, "task_id": "conflict_vlm_access_request_approval_v01_r07", "verb": "asset open" }, { "count": 10, "score": 0.6, "task_id": "conflict_vlm_access_request_approval_v01_r08", "verb": "asset open" } ], "loop_verbs": { "asset open": 165, "booking search": 1, "file read": 10, "message search": 29, "message send": 1 }, "loopy_capped": 206, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 208, "ok_false": 0, "pass_pct": 14.7, "passed": 88, "rank": { "avg_score": 12, "capped": 11, "failed_total": 11, "loopy_capped": 12, "near_miss": 2, "pass_pct": 11, "step_mean": 9, "step_p50": 9, "zero_open_tasks": 1, "zero_score": 11, "zero_write_tasks": 12 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.5813, "n": 10, "passed": 1 }, "access_request_approval_v02": { "mean_score": 0.44, "n": 10, "passed": 0 }, "access_request_approval_v03": { "mean_score": 0.5004, "n": 10, "passed": 1 }, "barrier_sync_deployment_v01": { "mean_score": 0.5862, "n": 10, "passed": 1 }, "barrier_sync_deployment_v02": { "mean_score": 0.5236, "n": 10, "passed": 2 }, "barrier_sync_deployment_v03": { "mean_score": 0.62, "n": 10, "passed": 2 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.3, "n": 10, "passed": 0 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.3012, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.4799, "n": 10, "passed": 2 }, "collaborative_contract_review_v01": { "mean_score": 0.3896, "n": 10, "passed": 2 }, "collaborative_contract_review_v02": { "mean_score": 0.7717, "n": 10, "passed": 6 }, "collaborative_contract_review_v03": { "mean_score": 0.436, "n": 10, "passed": 1 }, "coordinated_containment_v01": { "mean_score": 0.2433, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.5279, "n": 10, "passed": 2 }, "coordinated_containment_v03": { "mean_score": 0.4471, "n": 10, "passed": 1 }, "deadlock_multi_approval_v01": { "mean_score": 0.605, "n": 10, "passed": 2 }, "deadlock_multi_approval_v02": { "mean_score": 0.6758, "n": 10, "passed": 2 }, "deadlock_multi_approval_v03": { "mean_score": 0.65, "n": 10, "passed": 1 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.7281, "n": 10, "passed": 2 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.3583, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.3202, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v01": { "mean_score": 0.4953, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v02": { "mean_score": 0.5917, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v03": { "mean_score": 0.5168, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.645, "n": 10, "passed": 4 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.485, "n": 10, "passed": 0 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.6167, "n": 10, "passed": 1 }, "expiring_resource_hold_v01": { "mean_score": 0.5419, "n": 10, "passed": 3 }, "expiring_resource_hold_v02": { "mean_score": 0.6518, "n": 10, "passed": 2 }, "expiring_resource_hold_v03": { "mean_score": 0.4626, "n": 10, "passed": 1 }, "incident_commander_selection_v01": { "mean_score": 0.3833, "n": 10, "passed": 2 }, "incident_commander_selection_v02": { "mean_score": 0.4683, "n": 10, "passed": 0 }, "incident_commander_selection_v03": { "mean_score": 0.205, "n": 10, "passed": 0 }, "it_deployment_gate_v01": { "mean_score": 0.5734, "n": 10, "passed": 1 }, "it_deployment_gate_v02": { "mean_score": 0.6368, "n": 10, "passed": 3 }, "it_deployment_gate_v03": { "mean_score": 0.6092, "n": 10, "passed": 3 }, "multi_device_pairing_v01": { "mean_score": 0.5378, "n": 10, "passed": 1 }, "multi_device_pairing_v02": { "mean_score": 0.6247, "n": 10, "passed": 2 }, "multi_device_pairing_v03": { "mean_score": 0.454, "n": 10, "passed": 2 }, "multi_party_access_permit_v01": { "mean_score": 0.5583, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.7548, "n": 10, "passed": 3 }, "multi_party_access_permit_v03": { "mean_score": 0.4595, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.2573, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.2545, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.5075, "n": 10, "passed": 2 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.3433, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.3817, "n": 10, "passed": 0 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.3617, "n": 10, "passed": 2 }, "payment_chargeback_dispute_v01": { "mean_score": 0.7167, "n": 10, "passed": 5 }, "payment_chargeback_dispute_v02": { "mean_score": 0.4583, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v03": { "mean_score": 0.3, "n": 10, "passed": 2 }, "satellite_ground_station_handover_v01": { "mean_score": 0.5167, "n": 10, "passed": 3 }, "satellite_ground_station_handover_v02": { "mean_score": 0.4667, "n": 10, "passed": 2 }, "satellite_ground_station_handover_v03": { "mean_score": 0.2, "n": 10, "passed": 2 }, "sla_support_response_v01": { "mean_score": 0.5043, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.5506, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.3444, "n": 10, "passed": 0 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.4417, "n": 10, "passed": 3 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.5417, "n": 10, "passed": 3 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.2967, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 494, "fail_closed_pct": 23.1, "n": 2137 }, "side_effect_fields": { "approval_escalated": 6, "approval_granted": 4, "approval_recorded": 9, "approval_requested": 17, "assets_accessed": 13, "booking_extended": 3, "booking_held": 9, "booking_released": 7, "containment_action": 11, "deployment_approved": 6, "deployment_cutover": 1, "deployment_gate_set": 10, "deployment_requested": 27, "document_signed": 1, "lock_acquired": 13, "lock_released": 1, "messages_sent": 3, "pairing_code_entered": 3, "pairing_completed": 1, "pairing_started": 6, "review_assigned": 2, "review_submitted": 3 }, "step_mean": 14.89, "step_p50": 15, "stops": { "generation_stall_empty_reply": 4, "incomplete_required_submission": 139, "missing_required_side_effect": 156, "none": 88, "reached max_steps=20": 207, "stopped after 3 identical no-progress transitions": 6 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 10": 20, "wait 15": 31, "wait 20": 5, "wait 30": 7, "wait 300": 2, "wait 5": 20 }, "max": 300.0, "mean": 19.7, "median": 15.0, "n": 85, "sub_30": 76, "total": 1675.0 }, "zero_open_tasks": 0, "zero_score": 69, "zero_write_tasks": 157 }, "kimi": { "avg_score": 0.5591, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 170, "checks_total": 3760, "display": "kimi-k3", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'SEC-TELEMETRY-9914-BP': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'SEC-VERIFY-4109': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r06" }, { "message": "files_created.content contains 'FULLY_RECONCILED_AND_GRANTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r10" } ], "content_file": [ { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r06" } ], "content_msg": [ { "message": "LLM semantic match: No message was sent in #iam-approvals-sec citing permit PERMIT-2026-BP-9982 and report path /reports/access_grant_decision.md. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The message to Sana_Security_Reviewer in #iam-approvals-sec requested updated WRITE authorization and cited reference code AUDIT-SEC-2026-READ, but omitted the required passcode SEC-PASS-8842-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: The confirmation message containing tracking code REAPP-BP-2026-0842-AO in #iam-approvals-sec was addressed to Ravi_Application_Owner instead of Sana_Security_Reviewer. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r04" }, { "message": "LLM semantic match: no 'message' content in messages_sent to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20241024_085500': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20241024_085500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v02_r07" }, { "message": "assets_accessed.id contains 'ops_console_gate_controller_metrics_20260330_143000': False; normalized fallback disabled ('ops_console_gate_controller_metrics_20260330_143000' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r04" }, { "message": "assets_accessed.id contains 'ops_console_gate_board_20260330_141000': False; normalized fallback disabled ('ops_console_gate_board_20260330_141000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r06" }, { "message": "assets_accessed.id contains 'ops_console_canary_cutover_telemetry_20260330_141500': False; normalized fallback disabled ('ops_console_canary_cutover_telemetry_20260330_141500' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r06" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r04" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r05" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r06" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 101 }, "content_file": { "diagnostic": 0, "required": 334 }, "content_msg": { "diagnostic": 0, "required": 686 }, "grounding": { "diagnostic": 45, "required": 17 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 291 }, "window": { "diagnostic": 0, "required": 22 } }, "failed_total": 1496, "gating": { "content_involved": 419, "grounding_only": 0, "grounding_only_ids": [], "n_fail": 469, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 7, "score": 0.375, "task_id": "conflict_vlm_access_request_approval_v01_r01", "verb": "asset open" }, { "count": 14, "score": 0.14285714285714285, "task_id": "conflict_vlm_access_request_approval_v01_r05", "verb": "asset open" }, { "count": 19, "score": 0.2, "task_id": "conflict_vlm_access_request_approval_v01_r06", "verb": "asset open" }, { "count": 19, "score": 0.2, "task_id": "conflict_vlm_access_request_approval_v01_r07", "verb": "asset open" } ], "loop_verbs": { "agent status": 1, "asset open": 136, "file read": 15, "message search": 14, "pairing status": 1 }, "loopy_capped": 167, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 223, "ok_false": 0, "pass_pct": 21.8, "passed": 131, "rank": { "avg_score": 10, "capped": 10, "failed_total": 9, "loopy_capped": 10, "near_miss": 3, "pass_pct": 4, "step_mean": 10, "step_p50": 10, "zero_open_tasks": 8, "zero_score": 10, "zero_write_tasks": 11 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.4923, "n": 10, "passed": 1 }, "access_request_approval_v02": { "mean_score": 0.5767, "n": 10, "passed": 1 }, "access_request_approval_v03": { "mean_score": 0.5369, "n": 10, "passed": 1 }, "barrier_sync_deployment_v01": { "mean_score": 0.6753, "n": 10, "passed": 3 }, "barrier_sync_deployment_v02": { "mean_score": 0.6221, "n": 10, "passed": 2 }, "barrier_sync_deployment_v03": { "mean_score": 0.6, "n": 10, "passed": 2 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.2933, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.3833, "n": 10, "passed": 3 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.2685, "n": 10, "passed": 1 }, "collaborative_contract_review_v01": { "mean_score": 0.6099, "n": 10, "passed": 3 }, "collaborative_contract_review_v02": { "mean_score": 0.7483, "n": 10, "passed": 6 }, "collaborative_contract_review_v03": { "mean_score": 0.6193, "n": 10, "passed": 3 }, "coordinated_containment_v01": { "mean_score": 0.4526, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.4837, "n": 10, "passed": 0 }, "coordinated_containment_v03": { "mean_score": 0.6729, "n": 10, "passed": 2 }, "deadlock_multi_approval_v01": { "mean_score": 0.6375, "n": 10, "passed": 5 }, "deadlock_multi_approval_v02": { "mean_score": 0.8401, "n": 10, "passed": 5 }, "deadlock_multi_approval_v03": { "mean_score": 0.69, "n": 10, "passed": 4 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.7883, "n": 10, "passed": 5 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.4617, "n": 10, "passed": 1 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.4554, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v01": { "mean_score": 0.5533, "n": 10, "passed": 2 }, "employee_onboarding_workflow_v02": { "mean_score": 0.634, "n": 10, "passed": 2 }, "employee_onboarding_workflow_v03": { "mean_score": 0.4388, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.62, "n": 10, "passed": 4 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.79, "n": 10, "passed": 3 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.771, "n": 10, "passed": 3 }, "expiring_resource_hold_v01": { "mean_score": 0.6076, "n": 10, "passed": 4 }, "expiring_resource_hold_v02": { "mean_score": 0.5369, "n": 10, "passed": 1 }, "expiring_resource_hold_v03": { "mean_score": 0.7171, "n": 10, "passed": 1 }, "incident_commander_selection_v01": { "mean_score": 0.35, "n": 10, "passed": 0 }, "incident_commander_selection_v02": { "mean_score": 0.4017, "n": 10, "passed": 1 }, "incident_commander_selection_v03": { "mean_score": 0.5617, "n": 10, "passed": 2 }, "it_deployment_gate_v01": { "mean_score": 0.5759, "n": 10, "passed": 2 }, "it_deployment_gate_v02": { "mean_score": 0.6915, "n": 10, "passed": 5 }, "it_deployment_gate_v03": { "mean_score": 0.6369, "n": 10, "passed": 0 }, "multi_device_pairing_v01": { "mean_score": 0.6556, "n": 10, "passed": 2 }, "multi_device_pairing_v02": { "mean_score": 0.6159, "n": 10, "passed": 2 }, "multi_device_pairing_v03": { "mean_score": 0.4919, "n": 10, "passed": 1 }, "multi_party_access_permit_v01": { "mean_score": 0.4108, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.7929, "n": 10, "passed": 5 }, "multi_party_access_permit_v03": { "mean_score": 0.4545, "n": 10, "passed": 1 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.5412, "n": 10, "passed": 1 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.5092, "n": 10, "passed": 1 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.6221, "n": 10, "passed": 2 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.4433, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.5933, "n": 10, "passed": 2 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.5883, "n": 10, "passed": 3 }, "payment_chargeback_dispute_v01": { "mean_score": 0.5, "n": 10, "passed": 4 }, "payment_chargeback_dispute_v02": { "mean_score": 0.4333, "n": 10, "passed": 2 }, "payment_chargeback_dispute_v03": { "mean_score": 0.2333, "n": 10, "passed": 1 }, "satellite_ground_station_handover_v01": { "mean_score": 0.4917, "n": 10, "passed": 4 }, "satellite_ground_station_handover_v02": { "mean_score": 0.6, "n": 10, "passed": 3 }, "satellite_ground_station_handover_v03": { "mean_score": 0.625, "n": 10, "passed": 6 }, "sla_support_response_v01": { "mean_score": 0.5948, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.6333, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.5192, "n": 10, "passed": 1 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.4417, "n": 10, "passed": 3 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.6833, "n": 10, "passed": 5 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.2783, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 437, "fail_closed_pct": 20.4, "n": 2137 }, "side_effect_fields": { "approval_escalated": 5, "approval_granted": 1, "approval_recorded": 10, "approval_requested": 14, "assets_accessed": 12, "booking_extended": 3, "booking_held": 14, "booking_released": 10, "containment_action": 15, "deployment_approved": 9, "deployment_cutover": 1, "deployment_gate_set": 6, "deployment_requested": 25, "files_created": 3, "lock_acquired": 11, "lock_released": 1, "messages_sent": 1, "pairing_code_entered": 5, "pairing_code_shown": 2, "pairing_started": 10, "review_assigned": 3, "review_submitted": 4 }, "step_mean": 15.42, "step_p50": 16, "stops": { "generation_stall_empty_reply": 2, "incomplete_required_submission": 132, "missing_required_side_effect": 165, "none": 131, "reached max_steps=20": 170 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 10": 23, "wait 120": 3, "wait 15": 6, "wait 30": 70, "wait 300": 6, "wait 45": 7, "wait 5": 3, "wait 60": 11 }, "max": 300.0, "mean": 44.9, "median": 30.0, "n": 133, "sub_30": 33, "total": 5970.0 }, "zero_open_tasks": 3, "zero_score": 54, "zero_write_tasks": 128 }, "luna": { "avg_score": 0.6035, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 55, "checks_total": 3760, "display": "gpt-5.6-luna", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'SEC-TELEMETRY-9914-BP': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'FULLY_RECONCILED_AND_GRANTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r10" }, { "message": "files_created.content contains 'ERR-DATA-BACKFILL-409': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r03" } ], "content_file": [ { "message": "LLM semantic match: The report's final decisive finding is GRANTED (WRITE) rather than UNRECONCILED, and the initial analysis noted one scope mismatch and one resource mismatch rather than two scope mismatches. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The report finds that reconciliation failed (UNRECONCILED), contradicting the required finding that all checks passed. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "LLM semantic match: The report's verdict explicitly denies the access grant and states not to proceed with permit issuance. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r04" } ], "content_msg": [ { "message": "LLM semantic match: The message requested Sana's scope upgrade to WRITE on billing-prod in #iam-approvals-sec, but provided the incorrect passcode SEC-PASS-8842-BP instead of SEC-TELEMETRY-9914-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "LLM semantic match: No message was sent to Quinn_Engineering_Manager in #proj-billing-migration confirming that passcode SEC-TELEMETRY-9914-BP was submitted to Sana. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "LLM semantic match: No message was sent to Ravi_Application_Owner in #iam-approvals-sec confirming that passcode SEC-TELEMETRY-9914-BP was verified. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "LLM semantic match: The sent message in #proj-billing-migration reported the audit status as UNRECONCILED rather than ALL RECONCILIATION CHECKS PASSED. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r07': False; normalized fallback disabled ('iam_console_access_scope_matrix_r07' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "assets_accessed.id contains 'security_dashboard_telemetry_20260330_100000': False; normalized fallback disabled ('security_dashboard_telemetry_20260330_100000' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r08" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20250218_103000': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20250218_103000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r07" }, { "message": "assets_accessed.id contains 'audit_dashboard_sync_telemetry_20250218_110000': False; normalized fallback disabled ('audit_dashboard_sync_telemetry_20250218_110000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r09" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r04" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 62 }, "content_file": { "diagnostic": 0, "required": 223 }, "content_msg": { "diagnostic": 0, "required": 644 }, "grounding": { "diagnostic": 77, "required": 36 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 329 }, "window": { "diagnostic": 0, "required": 27 } }, "failed_total": 1398, "gating": { "content_involved": 411, "grounding_only": 4, "grounding_only_ids": [ "conflict_vlm_distributed_lock_shared_resource_v03_r06", "conflict_vlm_employee_onboarding_workflow_v02_r06", "conflict_vlm_employee_onboarding_workflow_v02_r08", "conflict_vlm_multi_tab_slot_monitoring_v02_r06" ], "n_fail": 481, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 9, "score": 0.75, "task_id": "conflict_vlm_access_request_approval_v02_r01", "verb": "message search" }, { "count": 10, "score": 0.6, "task_id": "conflict_vlm_access_request_approval_v02_r02", "verb": "message search" }, { "count": 8, "score": 0.25, "task_id": "conflict_vlm_access_request_approval_v02_r04", "verb": "message search" }, { "count": 6, "score": 0.6666666666666666, "task_id": "conflict_vlm_barrier_sync_deployment_v01_r09", "verb": "asset open" } ], "loop_verbs": { "agent status": 2, "asset open": 22, "file create": 1, "file read": 7, "message search": 16, "message send": 4 }, "loopy_capped": 52, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 286, "ok_false": 0, "pass_pct": 19.8, "passed": 119, "rank": { "avg_score": 4, "capped": 6, "failed_total": 4, "loopy_capped": 6, "near_miss": 13, "pass_pct": 8, "step_mean": 7, "step_p50": 6, "zero_open_tasks": 1, "zero_score": 4, "zero_write_tasks": 6 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.6725, "n": 10, "passed": 2 }, "access_request_approval_v02": { "mean_score": 0.7017, "n": 10, "passed": 1 }, "access_request_approval_v03": { "mean_score": 0.5401, "n": 10, "passed": 1 }, "barrier_sync_deployment_v01": { "mean_score": 0.672, "n": 10, "passed": 3 }, "barrier_sync_deployment_v02": { "mean_score": 0.7555, "n": 10, "passed": 4 }, "barrier_sync_deployment_v03": { "mean_score": 0.7067, "n": 10, "passed": 2 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.26, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.4429, "n": 10, "passed": 2 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.437, "n": 10, "passed": 1 }, "collaborative_contract_review_v01": { "mean_score": 0.6123, "n": 10, "passed": 1 }, "collaborative_contract_review_v02": { "mean_score": 0.7167, "n": 10, "passed": 4 }, "collaborative_contract_review_v03": { "mean_score": 0.7498, "n": 10, "passed": 4 }, "coordinated_containment_v01": { "mean_score": 0.4395, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.6879, "n": 10, "passed": 2 }, "coordinated_containment_v03": { "mean_score": 0.7495, "n": 10, "passed": 2 }, "deadlock_multi_approval_v01": { "mean_score": 0.6717, "n": 10, "passed": 3 }, "deadlock_multi_approval_v02": { "mean_score": 0.7933, "n": 10, "passed": 4 }, "deadlock_multi_approval_v03": { "mean_score": 0.74, "n": 10, "passed": 4 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.6214, "n": 10, "passed": 2 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.2964, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.4095, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v01": { "mean_score": 0.5831, "n": 10, "passed": 3 }, "employee_onboarding_workflow_v02": { "mean_score": 0.5917, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v03": { "mean_score": 0.5585, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.8417, "n": 10, "passed": 6 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.725, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.6931, "n": 10, "passed": 2 }, "expiring_resource_hold_v01": { "mean_score": 0.6419, "n": 10, "passed": 3 }, "expiring_resource_hold_v02": { "mean_score": 0.4887, "n": 10, "passed": 1 }, "expiring_resource_hold_v03": { "mean_score": 0.7357, "n": 10, "passed": 1 }, "incident_commander_selection_v01": { "mean_score": 0.5333, "n": 10, "passed": 1 }, "incident_commander_selection_v02": { "mean_score": 0.6683, "n": 10, "passed": 4 }, "incident_commander_selection_v03": { "mean_score": 0.6683, "n": 10, "passed": 2 }, "it_deployment_gate_v01": { "mean_score": 0.6959, "n": 10, "passed": 3 }, "it_deployment_gate_v02": { "mean_score": 0.6071, "n": 10, "passed": 2 }, "it_deployment_gate_v03": { "mean_score": 0.6496, "n": 10, "passed": 1 }, "multi_device_pairing_v01": { "mean_score": 0.5739, "n": 10, "passed": 0 }, "multi_device_pairing_v02": { "mean_score": 0.7159, "n": 10, "passed": 3 }, "multi_device_pairing_v03": { "mean_score": 0.6548, "n": 10, "passed": 3 }, "multi_party_access_permit_v01": { "mean_score": 0.5683, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.8429, "n": 10, "passed": 6 }, "multi_party_access_permit_v03": { "mean_score": 0.6862, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.5363, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.4476, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.5489, "n": 10, "passed": 0 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.33, "n": 10, "passed": 0 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.3617, "n": 10, "passed": 0 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.5433, "n": 10, "passed": 2 }, "payment_chargeback_dispute_v01": { "mean_score": 0.7833, "n": 10, "passed": 5 }, "payment_chargeback_dispute_v02": { "mean_score": 0.4583, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v03": { "mean_score": 0.4667, "n": 10, "passed": 1 }, "satellite_ground_station_handover_v01": { "mean_score": 0.575, "n": 10, "passed": 4 }, "satellite_ground_station_handover_v02": { "mean_score": 0.7667, "n": 10, "passed": 4 }, "satellite_ground_station_handover_v03": { "mean_score": 0.7417, "n": 10, "passed": 5 }, "sla_support_response_v01": { "mean_score": 0.5695, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.681, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.4811, "n": 10, "passed": 1 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.5, "n": 10, "passed": 2 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.75, "n": 10, "passed": 5 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.27, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 51, "fail_closed_pct": 2.4, "n": 2137 }, "side_effect_fields": { "approval_escalated": 7, "approval_granted": 3, "approval_recorded": 17, "approval_requested": 20, "assets_accessed": 25, "booking_extended": 4, "booking_held": 25, "booking_released": 8, "containment_action": 20, "deployment_approved": 10, "deployment_gate_set": 8, "deployment_requested": 32, "lock_acquired": 13, "lock_released": 1, "messages_sent": 3, "pairing_code_entered": 5, "pairing_started": 13, "review_assigned": 5, "review_submitted": 7 }, "step_mean": 13.54, "step_p50": 13, "stops": { "incomplete_required_submission": 194, "missing_required_side_effect": 226, "none": 119, "reached max_steps=20": 55, "stopped after 3 identical no-progress transitions": 6 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 1": 46, "wait 10": 15, "wait 2": 38, "wait 3": 40, "wait 30": 2, "wait 300": 4, "wait 4": 4, "wait 5": 48 }, "max": 300.0, "mean": 10.0, "median": 3.0, "n": 200, "sub_30": 193, "total": 1996.0 }, "zero_open_tasks": 0, "zero_score": 31, "zero_write_tasks": 15 }, "opus": { "avg_score": 0.6207, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 37, "checks_total": 3760, "display": "claude-opus-4.8", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'SEC-TELEMETRY-9914-BP': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'ERR-DATA-BACKFILL-409': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r03" }, { "message": "files_created.content contains 'AUTH-GATE-9942-SYNC': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v02_r03" } ], "content_file": [ { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "LLM semantic match: The report finds that reconciliation failed due to scope and resource mismatches, directly contradicting the requirement that all reconciliation checks passed. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "LLM semantic match: The report concludes that the request is not reconciled and that the access permit must not be issued, rather than deciding to proceed with the access grant. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "LLM semantic match: The report references transaction ID TXN-APP-WRITE-9912 instead of TXN-APP-WRITE-9908. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r04" } ], "content_msg": [ { "message": "LLM semantic match: The required message citing permit PERMIT-2026-BP-9982 and report path /reports/access_grant_decision.md to Sana_Security_Reviewer was not sent. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The message to Quinn in #proj-billing-migration explains the portal discrepancies and delays, but fails to include the specific reference code AUDIT-SEC-2026-READ. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: The message to Ravi in #iam-approvals-sec requested re-approval for WRITE scope on billing-prod, but cited AUDIT-APP-2026-ARCH instead of the requested reference code AUDIT-SEC-2026-READ. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: No message was sent to Ravi_Application_Owner in #iam-approvals-sec confirming that passcode SEC-TELEMETRY-9914-BP was verified. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20241024_085500': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20241024_085500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v02_r07" }, { "message": "assets_accessed.id contains 'ops_console_gate_controller_metrics_20260330_143000': False; normalized fallback disabled ('ops_console_gate_controller_metrics_20260330_143000' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r04" }, { "message": "assets_accessed.id contains 'ops_console_canary_cutover_telemetry_20260330_141500': False; normalized fallback disabled ('ops_console_canary_cutover_telemetry_20260330_141500' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r06" }, { "message": "assets_accessed.id contains 'telemetry_dashboard_worker_batch_rate_20231104_081115': False; normalized fallback disabled ('telemetry_dashboard_worker_batch_rate_20231104_081115' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v03_r04" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r05" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 56 }, "content_file": { "diagnostic": 0, "required": 225 }, "content_msg": { "diagnostic": 0, "required": 633 }, "grounding": { "diagnostic": 87, "required": 38 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 305 }, "window": { "diagnostic": 0, "required": 26 } }, "failed_total": 1370, "gating": { "content_involved": 402, "grounding_only": 0, "grounding_only_ids": [], "n_fail": 463, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 6, "score": 0.42857142857142855, "task_id": "conflict_vlm_access_request_approval_v01_r05", "verb": "file read" }, { "count": 5, "score": 0.4166666666666667, "task_id": "conflict_vlm_barrier_sync_deployment_v03_r08", "verb": "file read" }, { "count": 11, "score": 0.3333333333333333, "task_id": "conflict_vlm_clinical_trial_dosage_conflict_v02_r10", "verb": "file read" }, { "count": 9, "score": 0.5, "task_id": "conflict_vlm_coordinated_containment_v01_r10", "verb": "file read" } ], "loop_verbs": { "asset open": 8, "booking status": 1, "file read": 13, "message search": 7 }, "loopy_capped": 29, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 262, "ok_false": 0, "pass_pct": 22.8, "passed": 137, "rank": { "avg_score": 3, "capped": 5, "failed_total": 3, "loopy_capped": 5, "near_miss": 6, "pass_pct": 3, "step_mean": 4, "step_p50": 4, "zero_open_tasks": 5, "zero_score": 2, "zero_write_tasks": 5 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.6982, "n": 10, "passed": 2 }, "access_request_approval_v02": { "mean_score": 0.8042, "n": 10, "passed": 3 }, "access_request_approval_v03": { "mean_score": 0.4477, "n": 10, "passed": 1 }, "barrier_sync_deployment_v01": { "mean_score": 0.6686, "n": 10, "passed": 2 }, "barrier_sync_deployment_v02": { "mean_score": 0.7302, "n": 10, "passed": 4 }, "barrier_sync_deployment_v03": { "mean_score": 0.5933, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.5067, "n": 10, "passed": 2 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.3226, "n": 10, "passed": 0 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.4627, "n": 10, "passed": 1 }, "collaborative_contract_review_v01": { "mean_score": 0.5337, "n": 10, "passed": 1 }, "collaborative_contract_review_v02": { "mean_score": 0.8283, "n": 10, "passed": 6 }, "collaborative_contract_review_v03": { "mean_score": 0.8036, "n": 10, "passed": 5 }, "coordinated_containment_v01": { "mean_score": 0.452, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.6513, "n": 10, "passed": 2 }, "coordinated_containment_v03": { "mean_score": 0.7071, "n": 10, "passed": 1 }, "deadlock_multi_approval_v01": { "mean_score": 0.7008, "n": 10, "passed": 3 }, "deadlock_multi_approval_v02": { "mean_score": 0.8642, "n": 10, "passed": 4 }, "deadlock_multi_approval_v03": { "mean_score": 0.76, "n": 10, "passed": 4 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.6321, "n": 10, "passed": 1 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.3625, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.4161, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v01": { "mean_score": 0.5831, "n": 10, "passed": 3 }, "employee_onboarding_workflow_v02": { "mean_score": 0.5532, "n": 10, "passed": 2 }, "employee_onboarding_workflow_v03": { "mean_score": 0.5142, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.7767, "n": 10, "passed": 3 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.75, "n": 10, "passed": 3 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.7357, "n": 10, "passed": 2 }, "expiring_resource_hold_v01": { "mean_score": 0.6462, "n": 10, "passed": 2 }, "expiring_resource_hold_v02": { "mean_score": 0.7012, "n": 10, "passed": 2 }, "expiring_resource_hold_v03": { "mean_score": 0.7055, "n": 10, "passed": 1 }, "incident_commander_selection_v01": { "mean_score": 0.4083, "n": 10, "passed": 1 }, "incident_commander_selection_v02": { "mean_score": 0.7217, "n": 10, "passed": 3 }, "incident_commander_selection_v03": { "mean_score": 0.735, "n": 10, "passed": 3 }, "it_deployment_gate_v01": { "mean_score": 0.7159, "n": 10, "passed": 4 }, "it_deployment_gate_v02": { "mean_score": 0.6124, "n": 10, "passed": 3 }, "it_deployment_gate_v03": { "mean_score": 0.7206, "n": 10, "passed": 3 }, "multi_device_pairing_v01": { "mean_score": 0.5322, "n": 10, "passed": 1 }, "multi_device_pairing_v02": { "mean_score": 0.6859, "n": 10, "passed": 3 }, "multi_device_pairing_v03": { "mean_score": 0.6818, "n": 10, "passed": 3 }, "multi_party_access_permit_v01": { "mean_score": 0.5875, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.7869, "n": 10, "passed": 5 }, "multi_party_access_permit_v03": { "mean_score": 0.6829, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.6116, "n": 10, "passed": 1 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.5076, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.5746, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.3833, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.4767, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.5633, "n": 10, "passed": 2 }, "payment_chargeback_dispute_v01": { "mean_score": 0.875, "n": 10, "passed": 7 }, "payment_chargeback_dispute_v02": { "mean_score": 0.4833, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v03": { "mean_score": 0.6333, "n": 10, "passed": 4 }, "satellite_ground_station_handover_v01": { "mean_score": 0.5583, "n": 10, "passed": 4 }, "satellite_ground_station_handover_v02": { "mean_score": 0.8, "n": 10, "passed": 5 }, "satellite_ground_station_handover_v03": { "mean_score": 0.6917, "n": 10, "passed": 5 }, "sla_support_response_v01": { "mean_score": 0.6148, "n": 10, "passed": 1 }, "sla_support_response_v02": { "mean_score": 0.6538, "n": 10, "passed": 1 }, "sla_support_response_v03": { "mean_score": 0.4635, "n": 10, "passed": 2 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.6333, "n": 10, "passed": 4 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.65, "n": 10, "passed": 4 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.2867, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 44, "fail_closed_pct": 2.1, "n": 2137 }, "side_effect_fields": { "approval_escalated": 7, "approval_granted": 4, "approval_recorded": 17, "approval_requested": 21, "assets_accessed": 33, "booking_extended": 4, "booking_held": 26, "booking_released": 11, "containment_action": 20, "deployment_approved": 7, "deployment_gate_set": 11, "deployment_requested": 27, "lock_acquired": 14, "messages_sent": 6, "pairing_code_entered": 1, "pairing_code_shown": 1, "pairing_completed": 1, "pairing_started": 9, "review_assigned": 5, "review_submitted": 8 }, "step_mean": 12.17, "step_p50": 12, "stops": { "incomplete_required_submission": 193, "missing_required_side_effect": 233, "none": 137, "reached max_steps=20": 37 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 120": 8, "wait 180": 2, "wait 20": 1, "wait 30": 77, "wait 300": 4, "wait 60": 25, "wait 90": 4 }, "max": 300.0, "mean": 55.5, "median": 30.0, "n": 121, "sub_30": 1, "total": 6710.0 }, "zero_open_tasks": 1, "zero_score": 20, "zero_write_tasks": 14 }, "opus5": { "avg_score": 0.6419, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 3, "checks_total": 3760, "display": "claude-opus-5", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'ERR-DATA-BACKFILL-409': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r03" }, { "message": "files_created.content contains 'AUTH-GATE-9942-SYNC': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v02_r03" }, { "message": "files_created.content contains '138.8 U/L': False", "role": "required", "task_id": "conflict_vlm_clinical_trial_dosage_conflict_v01_r02" } ], "content_file": [ { "message": "LLM semantic match: The report finds that reconciliation checks failed and denies access, contradicting the required finding that all reconciliation checks passed. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "LLM semantic match: The report denies the access request and explicitly states not to proceed with the access grant. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "LLM semantic match: The report explicitly states that handoff prerequisites are not satisfied and holds the runbook handoff, rather than deciding to verify the permit and initiate handoff. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r08" }, { "message": "LLM semantic match: The report identifies TXN-APP-WRITE-9912 as the decisive finding and explicitly disregards TXN-APP-WRITE-9908 as an inapplicable staging transaction. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r04" } ], "content_msg": [ { "message": "LLM semantic match: The message to Ravi in #iam-approvals-sec requested re-approval for WRITE scope on billing-prod, but cited AUDIT-APP-2026-ARCH instead of AUDIT-SEC-2026-READ. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: No message was sent to Ravi_Application_Owner confirming that passcode SEC-TELEMETRY-9914-BP was verified and security scope reconciliation is underway. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "LLM semantic match: No message was sent to Quinn_Engineering_Manager in #proj-billing-migration reporting that all reconciliation checks passed. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "LLM semantic match: No message was sent to Sana_Security_Reviewer in #iam-approvals-sec confirming that all required approver scopes reconcile to WRITE on billing-prod. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r01': False; normalized fallback disabled ('iam_console_access_scope_matrix_r01' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "assets_accessed.id contains 'ops_console_canary_cutover_telemetry_20260330_141500': False; normalized fallback disabled ('ops_console_canary_cutover_telemetry_20260330_141500' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r06" }, { "message": "assets_accessed.id contains 'telemetry_dashboard_worker_batch_rate_20231104_081115': False; normalized fallback disabled ('telemetry_dashboard_worker_batch_rate_20231104_081115' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v03_r04" }, { "message": "assets_accessed.id contains 'deployment_portal_gate_board_synchronized_20231104_081700': False; normalized fallback disabled ('deployment_portal_gate_board_synchronized_20231104_081700' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_barrier_sync_deployment_v03_r08" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r05" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 45 }, "content_file": { "diagnostic": 0, "required": 223 }, "content_msg": { "diagnostic": 0, "required": 584 }, "grounding": { "diagnostic": 59, "required": 26 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 312 }, "window": { "diagnostic": 0, "required": 25 } }, "failed_total": 1274, "gating": { "content_involved": 377, "grounding_only": 2, "grounding_only_ids": [ "conflict_vlm_multi_device_pairing_v01_r04", "conflict_vlm_multi_tab_slot_monitoring_v01_r07" ], "n_fail": 449, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 7, "score": 0.3333333333333333, "task_id": "conflict_vlm_warehouse_robotics_safety_halt_v03_r01", "verb": "asset open" } ], "loop_verbs": { "asset open": 1 }, "loopy_capped": 1, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 276, "ok_false": 0, "pass_pct": 25.2, "passed": 151, "rank": { "avg_score": 1, "capped": 1, "failed_total": 1, "loopy_capped": 1, "near_miss": 11, "pass_pct": 1, "step_mean": 2, "step_p50": 2, "zero_open_tasks": 5, "zero_score": 1, "zero_write_tasks": 3 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.7211, "n": 10, "passed": 2 }, "access_request_approval_v02": { "mean_score": 0.8042, "n": 10, "passed": 3 }, "access_request_approval_v03": { "mean_score": 0.6195, "n": 10, "passed": 2 }, "barrier_sync_deployment_v01": { "mean_score": 0.6074, "n": 10, "passed": 2 }, "barrier_sync_deployment_v02": { "mean_score": 0.7038, "n": 10, "passed": 3 }, "barrier_sync_deployment_v03": { "mean_score": 0.5767, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.4633, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.45, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.3906, "n": 10, "passed": 1 }, "collaborative_contract_review_v01": { "mean_score": 0.6242, "n": 10, "passed": 4 }, "collaborative_contract_review_v02": { "mean_score": 0.7683, "n": 10, "passed": 5 }, "collaborative_contract_review_v03": { "mean_score": 0.8112, "n": 10, "passed": 5 }, "coordinated_containment_v01": { "mean_score": 0.5288, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.6603, "n": 10, "passed": 3 }, "coordinated_containment_v03": { "mean_score": 0.6405, "n": 10, "passed": 1 }, "deadlock_multi_approval_v01": { "mean_score": 0.7175, "n": 10, "passed": 3 }, "deadlock_multi_approval_v02": { "mean_score": 0.8558, "n": 10, "passed": 5 }, "deadlock_multi_approval_v03": { "mean_score": 0.7029, "n": 10, "passed": 3 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.7874, "n": 10, "passed": 5 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.4069, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.5327, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v01": { "mean_score": 0.5748, "n": 10, "passed": 2 }, "employee_onboarding_workflow_v02": { "mean_score": 0.6099, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v03": { "mean_score": 0.4943, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.7217, "n": 10, "passed": 3 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.69, "n": 10, "passed": 2 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.7071, "n": 10, "passed": 2 }, "expiring_resource_hold_v01": { "mean_score": 0.6605, "n": 10, "passed": 3 }, "expiring_resource_hold_v02": { "mean_score": 0.7107, "n": 10, "passed": 2 }, "expiring_resource_hold_v03": { "mean_score": 0.7867, "n": 10, "passed": 3 }, "incident_commander_selection_v01": { "mean_score": 0.5833, "n": 10, "passed": 3 }, "incident_commander_selection_v02": { "mean_score": 0.69, "n": 10, "passed": 3 }, "incident_commander_selection_v03": { "mean_score": 0.7017, "n": 10, "passed": 3 }, "it_deployment_gate_v01": { "mean_score": 0.7159, "n": 10, "passed": 4 }, "it_deployment_gate_v02": { "mean_score": 0.4763, "n": 10, "passed": 2 }, "it_deployment_gate_v03": { "mean_score": 0.5974, "n": 10, "passed": 2 }, "multi_device_pairing_v01": { "mean_score": 0.555, "n": 10, "passed": 1 }, "multi_device_pairing_v02": { "mean_score": 0.6792, "n": 10, "passed": 2 }, "multi_device_pairing_v03": { "mean_score": 0.742, "n": 10, "passed": 4 }, "multi_party_access_permit_v01": { "mean_score": 0.6942, "n": 10, "passed": 2 }, "multi_party_access_permit_v02": { "mean_score": 0.8262, "n": 10, "passed": 6 }, "multi_party_access_permit_v03": { "mean_score": 0.7229, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.7005, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.5076, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.6614, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.4, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.5017, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.5217, "n": 10, "passed": 2 }, "payment_chargeback_dispute_v01": { "mean_score": 0.7167, "n": 10, "passed": 6 }, "payment_chargeback_dispute_v02": { "mean_score": 0.5833, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v03": { "mean_score": 0.6, "n": 10, "passed": 4 }, "satellite_ground_station_handover_v01": { "mean_score": 0.7833, "n": 10, "passed": 6 }, "satellite_ground_station_handover_v02": { "mean_score": 0.6833, "n": 10, "passed": 3 }, "satellite_ground_station_handover_v03": { "mean_score": 0.7417, "n": 10, "passed": 6 }, "sla_support_response_v01": { "mean_score": 0.6457, "n": 10, "passed": 1 }, "sla_support_response_v02": { "mean_score": 0.7125, "n": 10, "passed": 1 }, "sla_support_response_v03": { "mean_score": 0.4277, "n": 10, "passed": 2 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.75, "n": 10, "passed": 4 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.8, "n": 10, "passed": 5 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.465, "n": 10, "passed": 1 } }, "semantic": { "fail_closed": 15, "fail_closed_pct": 0.7, "n": 2137 }, "side_effect_fields": { "approval_escalated": 6, "approval_granted": 3, "approval_recorded": 20, "approval_requested": 22, "assets_accessed": 21, "booking_extended": 4, "booking_held": 22, "booking_released": 12, "containment_action": 22, "deployment_approved": 9, "deployment_cutover": 1, "deployment_gate_set": 13, "deployment_requested": 34, "document_signed": 1, "files_created": 1, "lock_acquired": 15, "messages_sent": 5, "pairing_code_entered": 3, "pairing_code_shown": 1, "pairing_completed": 1, "pairing_started": 11, "review_assigned": 4, "review_submitted": 8 }, "step_mean": 11.12, "step_p50": 11, "stops": { "generation_stall_empty_reply": 3, "incomplete_required_submission": 204, "missing_required_side_effect": 239, "none": 151, "reached max_steps=20": 3 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 120": 9, "wait 180": 3, "wait 30": 11, "wait 300": 6, "wait 45": 3, "wait 60": 54, "wait 90": 4 }, "max": 300.0, "mean": 83.2, "median": 60.0, "n": 90, "sub_30": 0, "total": 7485.0 }, "zero_open_tasks": 1, "zero_score": 18, "zero_write_tasks": 2 }, "pro": { "avg_score": 0.4871, "bucket_names": { "anchor": { "effect:files_created field_contains": 254 }, "content_file": { "effect:files_created field_semantic_match": 935 }, "content_msg": { "effect:messages_sent field_semantic_match": 1184 }, "grounding": { "effect:assets_accessed field_contains": 814 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 22, "effect:lock_released any_field_equals": 22, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 12 }, "window": { "effect:messages_sent within_window": 29 } }, "capped": 152, "checks_total": 3728, "display": "gemini-3.1-pro", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'SEC-TELEMETRY-9914-BP': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "files_created.content contains 'POL-SEC-RECON-3.4': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r03" }, { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'ERR-DATA-BACKFILL-409': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r03" } ], "content_file": [ { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r01" } ], "content_msg": [ { "message": "LLM semantic match: The message to Quinn_Engineering_Manager rejects the premature grant demand and explains the discrepancies, but omits mentioning the dashboard's UNRECONCILED overall status and 2/3 scope mismatch count. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The required message citing permit PERMIT-2026-BP-9982 and report path /reports/access_grant_decision.md to Sana_Security_Reviewer in #iam-approvals-sec was not sent. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The message sent to Quinn_Engineering_Manager in #proj-billing-migration explained the portal status and scope mismatches, but omitted the required reference code AUDIT-SEC-2026-READ. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: A message was sent to Sana_Security_Reviewer in #iam-approvals-sec citing AUDIT-SEC-2026-READ, but it failed to include the passcode SEC-PASS-8842-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r07': False; normalized fallback disabled ('iam_console_access_scope_matrix_r07' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20241024_085500': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20241024_085500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v02_r07" }, { "message": "assets_accessed.id contains 'audit_dashboard_passcode_telemetry_20250218_103000': False; normalized fallback disabled ('audit_dashboard_passcode_telemetry_20250218_103000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r06" }, { "message": "assets_accessed.id contains 'ops_console_gate_controller_metrics_20260330_143000': False; normalized fallback disabled ('ops_console_gate_controller_metrics_20260330_143000' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r04" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r04" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 88 }, "content_file": { "diagnostic": 0, "required": 364 }, "content_msg": { "diagnostic": 0, "required": 831 }, "grounding": { "diagnostic": 89, "required": 38 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 320 }, "window": { "diagnostic": 0, "required": 25 } }, "failed_total": 1755, "gating": { "content_involved": 495, "grounding_only": 0, "grounding_only_ids": [], "n_fail": 528, "no_required_check_failed": 5, "no_required_check_failed_ids": [ "conflict_vlm_collaborative_contract_review_v03_r05", "conflict_vlm_distributed_lock_shared_resource_v02_r02", "conflict_vlm_expense_reimbursement_conflict_v02_r05", "conflict_vlm_satellite_ground_station_handover_v02_r03", "conflict_vlm_sla_support_response_v03_r03" ] }, "loop_examples": [ { "count": 5, "score": 0.42857142857142855, "task_id": "conflict_vlm_access_request_approval_v01_r02", "verb": "message send" }, { "count": 8, "score": 0.2857142857142857, "task_id": "conflict_vlm_access_request_approval_v01_r05", "verb": "message search" }, { "count": 9, "score": 0.25, "task_id": "conflict_vlm_access_request_approval_v02_r01", "verb": "message search" }, { "count": 13, "score": 0.2, "task_id": "conflict_vlm_access_request_approval_v02_r03", "verb": "message search" } ], "loop_verbs": { "agent status": 4, "asset open": 40, "file read": 6, "message search": 48, "message send": 8, "wait 10": 3 }, "loopy_capped": 119, "missing_check_result_ids": [ "conflict_vlm_collaborative_contract_review_v03_r05", "conflict_vlm_distributed_lock_shared_resource_v02_r02", "conflict_vlm_expense_reimbursement_conflict_v02_r05", "conflict_vlm_satellite_ground_station_handover_v02_r03", "conflict_vlm_sla_support_response_v03_r03" ], "missing_check_results": 5, "n": 600, "near_miss": 233, "ok_false": 0, "pass_pct": 12.0, "passed": 72, "rank": { "avg_score": 11, "capped": 9, "failed_total": 12, "loopy_capped": 8, "near_miss": 4, "pass_pct": 12, "step_mean": 5, "step_p50": 4, "zero_open_tasks": 10, "zero_score": 11, "zero_write_tasks": 8 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.5969, "n": 10, "passed": 1 }, "access_request_approval_v02": { "mean_score": 0.475, "n": 10, "passed": 0 }, "access_request_approval_v03": { "mean_score": 0.5384, "n": 10, "passed": 1 }, "barrier_sync_deployment_v01": { "mean_score": 0.6353, "n": 10, "passed": 3 }, "barrier_sync_deployment_v02": { "mean_score": 0.5088, "n": 10, "passed": 2 }, "barrier_sync_deployment_v03": { "mean_score": 0.6383, "n": 10, "passed": 2 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.2633, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.344, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.2094, "n": 10, "passed": 0 }, "collaborative_contract_review_v01": { "mean_score": 0.4902, "n": 10, "passed": 1 }, "collaborative_contract_review_v02": { "mean_score": 0.645, "n": 10, "passed": 3 }, "collaborative_contract_review_v03": { "mean_score": 0.6492, "n": 10, "passed": 1 }, "coordinated_containment_v01": { "mean_score": 0.3788, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.5495, "n": 10, "passed": 0 }, "coordinated_containment_v03": { "mean_score": 0.6095, "n": 10, "passed": 1 }, "deadlock_multi_approval_v01": { "mean_score": 0.6058, "n": 10, "passed": 2 }, "deadlock_multi_approval_v02": { "mean_score": 0.7608, "n": 10, "passed": 3 }, "deadlock_multi_approval_v03": { "mean_score": 0.6757, "n": 10, "passed": 1 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.5031, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.2706, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.2458, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v01": { "mean_score": 0.5156, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v02": { "mean_score": 0.5139, "n": 10, "passed": 2 }, "employee_onboarding_workflow_v03": { "mean_score": 0.379, "n": 10, "passed": 0 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.5533, "n": 10, "passed": 2 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.59, "n": 10, "passed": 2 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.4831, "n": 10, "passed": 0 }, "expiring_resource_hold_v01": { "mean_score": 0.6064, "n": 10, "passed": 0 }, "expiring_resource_hold_v02": { "mean_score": 0.6488, "n": 10, "passed": 2 }, "expiring_resource_hold_v03": { "mean_score": 0.6071, "n": 10, "passed": 0 }, "incident_commander_selection_v01": { "mean_score": 0.325, "n": 10, "passed": 1 }, "incident_commander_selection_v02": { "mean_score": 0.4517, "n": 10, "passed": 1 }, "incident_commander_selection_v03": { "mean_score": 0.3983, "n": 10, "passed": 1 }, "it_deployment_gate_v01": { "mean_score": 0.4578, "n": 10, "passed": 0 }, "it_deployment_gate_v02": { "mean_score": 0.5489, "n": 10, "passed": 3 }, "it_deployment_gate_v03": { "mean_score": 0.6191, "n": 10, "passed": 1 }, "multi_device_pairing_v01": { "mean_score": 0.3311, "n": 10, "passed": 0 }, "multi_device_pairing_v02": { "mean_score": 0.6347, "n": 10, "passed": 2 }, "multi_device_pairing_v03": { "mean_score": 0.5591, "n": 10, "passed": 3 }, "multi_party_access_permit_v01": { "mean_score": 0.4817, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.6226, "n": 10, "passed": 0 }, "multi_party_access_permit_v03": { "mean_score": 0.6429, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.3746, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.3033, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.3736, "n": 10, "passed": 0 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.32, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.37, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.3, "n": 10, "passed": 0 }, "payment_chargeback_dispute_v01": { "mean_score": 0.7917, "n": 10, "passed": 6 }, "payment_chargeback_dispute_v02": { "mean_score": 0.4083, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v03": { "mean_score": 0.2667, "n": 10, "passed": 0 }, "satellite_ground_station_handover_v01": { "mean_score": 0.5, "n": 10, "passed": 3 }, "satellite_ground_station_handover_v02": { "mean_score": 0.7667, "n": 10, "passed": 5 }, "satellite_ground_station_handover_v03": { "mean_score": 0.5833, "n": 10, "passed": 4 }, "sla_support_response_v01": { "mean_score": 0.49, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.4851, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.3873, "n": 10, "passed": 1 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.345, "n": 10, "passed": 2 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.3917, "n": 10, "passed": 2 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.2033, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 127, "fail_closed_pct": 6.0, "n": 2119 }, "side_effect_fields": { "approval_escalated": 6, "approval_granted": 3, "approval_recorded": 17, "approval_requested": 24, "assets_accessed": 29, "booking_extended": 4, "booking_held": 20, "booking_released": 6, "containment_action": 13, "deployment_approved": 6, "deployment_cutover": 1, "deployment_gate_set": 6, "deployment_requested": 26, "files_created": 4, "lock_acquired": 9, "lock_released": 2, "messages_sent": 2, "pairing_code_entered": 3, "pairing_code_shown": 1, "pairing_started": 9, "review_assigned": 4, "review_submitted": 8 }, "step_mean": 13.07, "step_p50": 12, "stops": { "incomplete_required_submission": 168, "missing_required_side_effect": 203, "none": 77, "reached max_steps=20": 152 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 254 }, "content_file": { "diagnostic": 0, "required": 935 }, "content_msg": { "diagnostic": 0, "required": 1184 }, "grounding": { "diagnostic": 471, "required": 343 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 512 }, "window": { "diagnostic": 0, "required": 29 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 10": 290, "wait 120": 6, "wait 15": 2, "wait 20": 21, "wait 30": 29, "wait 300": 5, "wait 5": 43, "wait 60": 6 }, "max": 300.0, "mean": 17.5, "median": 10.0, "n": 402, "sub_30": 356, "total": 7015.0 }, "zero_open_tasks": 14, "zero_score": 69, "zero_write_tasks": 59 }, "sol": { "avg_score": 0.6326, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 7, "checks_total": 3760, "display": "gpt-5.6-sol", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'SEC-TELEMETRY-9914-BP': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'FULLY_RECONCILED_AND_GRANTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r10" }, { "message": "files_created.content contains 'ERR-DATA-BACKFILL-409': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r03" } ], "content_file": [ { "message": "LLM semantic match: The report's decisive finding is GRANTED rather than UNRECONCILED. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The report references transaction ID TXN-APP-WRITE-9912 instead of the required TXN-APP-WRITE-9908. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r04" }, { "message": "LLM semantic match: The report explicitly states that dataset provisioning is not authorized rather than authorizing dataset access. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r10" }, { "message": "LLM semantic match: The report contains a SHA-256 digest ending in 'ab' rather than the exact digest specified in the criterion ending in 'de'. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r02" } ], "content_msg": [ { "message": "LLM semantic match: The message sent in #iam-approvals-sec cited AUDIT-SEC-2026-READ but omitted the required passcode SEC-PASS-8842-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: The agent requested re-approval for WRITE scope on billing-prod in #iam-approvals-sec, but associated AUDIT-SEC-2026-READ with Security rather than using it to cite Ravi's request. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: The message was sent to Sana_Security_Reviewer in #iam-approvals-sec to request the scope upgrade, but provided SEC-PASS-8842-BP instead of the requested SEC-TELEMETRY-9914-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "LLM semantic match: No message was sent to Quinn_Engineering_Manager confirming that passcode SEC-TELEMETRY-9914-BP was submitted to Sana. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" } ], "grounding": [ { "message": "assets_accessed.id contains 'security_dashboard_telemetry_20260330_100000': False; normalized fallback disabled ('security_dashboard_telemetry_20260330_100000' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r08" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20250218_103000': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20250218_103000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r07" }, { "message": "assets_accessed.id contains 'ops_console_canary_cutover_telemetry_20260330_141500': False; normalized fallback disabled ('ops_console_canary_cutover_telemetry_20260330_141500' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r06" }, { "message": "assets_accessed.id contains 'telemetry_dashboard_staging_cutover_execution_20231104_081800': False; normalized fallback disabled ('telemetry_dashboard_staging_cutover_execution_20231104_081800' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v03_r08" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r04" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r05" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 62 }, "content_file": { "diagnostic": 0, "required": 202 }, "content_msg": { "diagnostic": 0, "required": 616 }, "grounding": { "diagnostic": 49, "required": 23 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 312 }, "window": { "diagnostic": 0, "required": 26 } }, "failed_total": 1290, "gating": { "content_involved": 391, "grounding_only": 0, "grounding_only_ids": [], "n_fail": 453, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 5, "score": 0.5, "task_id": "conflict_vlm_collaborative_contract_review_v01_r03", "verb": "asset open" }, { "count": 6, "score": 0.42857142857142855, "task_id": "conflict_vlm_collaborative_contract_review_v03_r05", "verb": "message search" }, { "count": 6, "score": 0.5, "task_id": "conflict_vlm_incident_commander_selection_v01_r07", "verb": "file read" }, { "count": 5, "score": 0.4, "task_id": "conflict_vlm_sla_support_response_v01_r01", "verb": "agent status" } ], "loop_verbs": { "agent status": 1, "asset open": 3, "file read": 1, "message search": 1 }, "loopy_capped": 6, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 270, "ok_false": 0, "pass_pct": 24.5, "passed": 147, "rank": { "avg_score": 2, "capped": 2, "failed_total": 2, "loopy_capped": 3, "near_miss": 9, "pass_pct": 2, "step_mean": 8, "step_p50": 8, "zero_open_tasks": 1, "zero_score": 3, "zero_write_tasks": 1 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.7339, "n": 10, "passed": 2 }, "access_request_approval_v02": { "mean_score": 0.7667, "n": 10, "passed": 2 }, "access_request_approval_v03": { "mean_score": 0.5996, "n": 10, "passed": 2 }, "barrier_sync_deployment_v01": { "mean_score": 0.6553, "n": 10, "passed": 2 }, "barrier_sync_deployment_v02": { "mean_score": 0.7638, "n": 10, "passed": 4 }, "barrier_sync_deployment_v03": { "mean_score": 0.6667, "n": 10, "passed": 2 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.26, "n": 10, "passed": 0 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.5524, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.4011, "n": 10, "passed": 1 }, "collaborative_contract_review_v01": { "mean_score": 0.6265, "n": 10, "passed": 2 }, "collaborative_contract_review_v02": { "mean_score": 0.7883, "n": 10, "passed": 5 }, "collaborative_contract_review_v03": { "mean_score": 0.7979, "n": 10, "passed": 4 }, "coordinated_containment_v01": { "mean_score": 0.5145, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.7389, "n": 10, "passed": 3 }, "coordinated_containment_v03": { "mean_score": 0.7305, "n": 10, "passed": 2 }, "deadlock_multi_approval_v01": { "mean_score": 0.6475, "n": 10, "passed": 3 }, "deadlock_multi_approval_v02": { "mean_score": 0.8442, "n": 10, "passed": 3 }, "deadlock_multi_approval_v03": { "mean_score": 0.755, "n": 10, "passed": 3 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.725, "n": 10, "passed": 5 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.3617, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.4387, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v01": { "mean_score": 0.5564, "n": 10, "passed": 2 }, "employee_onboarding_workflow_v02": { "mean_score": 0.5998, "n": 10, "passed": 2 }, "employee_onboarding_workflow_v03": { "mean_score": 0.4576, "n": 10, "passed": 0 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.8167, "n": 10, "passed": 6 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.665, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.6814, "n": 10, "passed": 2 }, "expiring_resource_hold_v01": { "mean_score": 0.6629, "n": 10, "passed": 2 }, "expiring_resource_hold_v02": { "mean_score": 0.4679, "n": 10, "passed": 0 }, "expiring_resource_hold_v03": { "mean_score": 0.7724, "n": 10, "passed": 2 }, "incident_commander_selection_v01": { "mean_score": 0.4833, "n": 10, "passed": 1 }, "incident_commander_selection_v02": { "mean_score": 0.7483, "n": 10, "passed": 6 }, "incident_commander_selection_v03": { "mean_score": 0.7017, "n": 10, "passed": 3 }, "it_deployment_gate_v01": { "mean_score": 0.6377, "n": 10, "passed": 3 }, "it_deployment_gate_v02": { "mean_score": 0.6707, "n": 10, "passed": 4 }, "it_deployment_gate_v03": { "mean_score": 0.8032, "n": 10, "passed": 4 }, "multi_device_pairing_v01": { "mean_score": 0.4956, "n": 10, "passed": 0 }, "multi_device_pairing_v02": { "mean_score": 0.7447, "n": 10, "passed": 4 }, "multi_device_pairing_v03": { "mean_score": 0.7188, "n": 10, "passed": 3 }, "multi_party_access_permit_v01": { "mean_score": 0.5583, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.7345, "n": 10, "passed": 3 }, "multi_party_access_permit_v03": { "mean_score": 0.7029, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.5413, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.4852, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.5946, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.4233, "n": 10, "passed": 2 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.5183, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.6667, "n": 10, "passed": 4 }, "payment_chargeback_dispute_v01": { "mean_score": 0.9, "n": 10, "passed": 8 }, "payment_chargeback_dispute_v02": { "mean_score": 0.425, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v03": { "mean_score": 0.7667, "n": 10, "passed": 6 }, "satellite_ground_station_handover_v01": { "mean_score": 0.6417, "n": 10, "passed": 5 }, "satellite_ground_station_handover_v02": { "mean_score": 0.8667, "n": 10, "passed": 7 }, "satellite_ground_station_handover_v03": { "mean_score": 0.725, "n": 10, "passed": 5 }, "sla_support_response_v01": { "mean_score": 0.6038, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.625, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.5087, "n": 10, "passed": 2 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.75, "n": 10, "passed": 5 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.5833, "n": 10, "passed": 3 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.285, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 13, "fail_closed_pct": 0.6, "n": 2137 }, "side_effect_fields": { "approval_escalated": 9, "approval_granted": 3, "approval_recorded": 20, "approval_requested": 21, "assets_accessed": 20, "booking_extended": 4, "booking_held": 28, "booking_released": 12, "containment_action": 19, "deployment_approved": 8, "deployment_gate_set": 12, "deployment_requested": 32, "files_created": 2, "lock_acquired": 16, "lock_released": 1, "messages_sent": 3, "pairing_code_entered": 4, "pairing_started": 13, "review_assigned": 5, "review_submitted": 6 }, "step_mean": 13.65, "step_p50": 14, "stops": { "incomplete_required_submission": 208, "missing_required_side_effect": 238, "none": 147, "reached max_steps=20": 7 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 1": 10, "wait 10": 35, "wait 15": 5, "wait 2": 31, "wait 20": 9, "wait 3": 27, "wait 30": 13, "wait 5": 94 }, "max": 300.0, "mean": 14.6, "median": 5.0, "n": 240, "sub_30": 216, "total": 3506.0 }, "zero_open_tasks": 0, "zero_score": 23, "zero_write_tasks": 1 }, "sonnet": { "avg_score": 0.5963, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 20, "checks_total": 3760, "display": "claude-sonnet-5", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'FULLY_RECONCILED_AND_GRANTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r10" }, { "message": "files_created.content contains 'ERR-DATA-BACKFILL-409': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r03" }, { "message": "files_created.content contains 'AUTH-GATE-9942-SYNC': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v02_r03" } ], "content_file": [ { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The report concludes that the request is NOT RECONCILED and denied, explicitly contradicting the required finding that all reconciliation checks passed. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "LLM semantic match: The report concludes that the request is not reconciled and explicitly advises against proceeding with the access grant. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" } ], "content_msg": [ { "message": "LLM semantic match: The message to Sana_Security_Reviewer in #iam-approvals-sec citing permit PERMIT-2026-BP-9982 and report path /reports/access_grant_decision.md was not sent. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The message sent to Quinn_Engineering_Manager reported that the status was UNRECONCILED / BLOCKED rather than ALL RECONCILIATION CHECKS PASSED. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "LLM semantic match: No message was sent in #iam-approvals-sec confirming that all required approver scopes now reconcile to WRITE on billing-prod. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "LLM semantic match: The message informed Quinn_Engineering_Manager in #iam-coordination about the required scope reconciliation on billing_prod, but specified the error code as ERR-SCOPE-MISMATCH instead of ERR-SCOPE-MISMATCH-8842-SEC. [self-consistency 0/3 pa", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r02" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r01': False; normalized fallback disabled ('iam_console_access_scope_matrix_r01' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r01': False; normalized fallback disabled ('iam_console_access_scope_matrix_r01' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20241024_085500': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20241024_085500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v02_r07" }, { "message": "assets_accessed.id contains 'security_dashboard_telemetry_security_chart_20241024_091500': False; normalized fallback disabled ('security_dashboard_telemetry_security_chart_20241024_091500' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r08" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r04" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 60 }, "content_file": { "diagnostic": 0, "required": 245 }, "content_msg": { "diagnostic": 0, "required": 643 }, "grounding": { "diagnostic": 149, "required": 76 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 319 }, "window": { "diagnostic": 0, "required": 29 } }, "failed_total": 1521, "gating": { "content_involved": 404, "grounding_only": 5, "grounding_only_ids": [ "conflict_vlm_deadlock_multi_approval_v02_r10", "conflict_vlm_distributed_lock_shared_resource_v03_r06", "conflict_vlm_multi_device_pairing_v01_r04", "conflict_vlm_multi_device_pairing_v02_r06", "conflict_vlm_multi_tab_slot_monitoring_v01_r07" ], "n_fail": 475, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 7, "score": 0.375, "task_id": "conflict_vlm_access_request_approval_v01_r01", "verb": "asset open" }, { "count": 7, "score": 0.5, "task_id": "conflict_vlm_barrier_sync_deployment_v02_r07", "verb": "agent status" }, { "count": 5, "score": 0.14285714285714285, "task_id": "conflict_vlm_coordinated_containment_v03_r02", "verb": "asset list" }, { "count": 6, "score": 0.6666666666666666, "task_id": "conflict_vlm_distributed_lock_shared_resource_v02_r01", "verb": "message search" } ], "loop_verbs": { "agent status": 1, "asset list": 1, "asset open": 4, "deploy status": 1, "file read": 3, "message search": 4 }, "loopy_capped": 15, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 264, "ok_false": 0, "pass_pct": 20.8, "passed": 125, "rank": { "avg_score": 6, "capped": 4, "failed_total": 10, "loopy_capped": 4, "near_miss": 7, "pass_pct": 5, "step_mean": 1, "step_p50": 1, "zero_open_tasks": 13, "zero_score": 7, "zero_write_tasks": 4 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.7061, "n": 10, "passed": 2 }, "access_request_approval_v02": { "mean_score": 0.7875, "n": 10, "passed": 3 }, "access_request_approval_v03": { "mean_score": 0.4731, "n": 10, "passed": 1 }, "barrier_sync_deployment_v01": { "mean_score": 0.632, "n": 10, "passed": 2 }, "barrier_sync_deployment_v02": { "mean_score": 0.6805, "n": 10, "passed": 4 }, "barrier_sync_deployment_v03": { "mean_score": 0.5667, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.35, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.3917, "n": 10, "passed": 2 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.537, "n": 10, "passed": 1 }, "collaborative_contract_review_v01": { "mean_score": 0.5908, "n": 10, "passed": 4 }, "collaborative_contract_review_v02": { "mean_score": 0.7883, "n": 10, "passed": 6 }, "collaborative_contract_review_v03": { "mean_score": 0.8036, "n": 10, "passed": 5 }, "coordinated_containment_v01": { "mean_score": 0.5034, "n": 10, "passed": 1 }, "coordinated_containment_v02": { "mean_score": 0.6279, "n": 10, "passed": 2 }, "coordinated_containment_v03": { "mean_score": 0.6476, "n": 10, "passed": 1 }, "deadlock_multi_approval_v01": { "mean_score": 0.7008, "n": 10, "passed": 3 }, "deadlock_multi_approval_v02": { "mean_score": 0.7724, "n": 10, "passed": 3 }, "deadlock_multi_approval_v03": { "mean_score": 0.715, "n": 10, "passed": 3 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.7148, "n": 10, "passed": 3 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.3179, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.4286, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v01": { "mean_score": 0.515, "n": 10, "passed": 2 }, "employee_onboarding_workflow_v02": { "mean_score": 0.6274, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v03": { "mean_score": 0.4326, "n": 10, "passed": 0 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.6667, "n": 10, "passed": 2 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.595, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.561, "n": 10, "passed": 1 }, "expiring_resource_hold_v01": { "mean_score": 0.6712, "n": 10, "passed": 1 }, "expiring_resource_hold_v02": { "mean_score": 0.6464, "n": 10, "passed": 1 }, "expiring_resource_hold_v03": { "mean_score": 0.6179, "n": 10, "passed": 0 }, "incident_commander_selection_v01": { "mean_score": 0.5583, "n": 10, "passed": 2 }, "incident_commander_selection_v02": { "mean_score": 0.6233, "n": 10, "passed": 3 }, "incident_commander_selection_v03": { "mean_score": 0.6617, "n": 10, "passed": 2 }, "it_deployment_gate_v01": { "mean_score": 0.7011, "n": 10, "passed": 3 }, "it_deployment_gate_v02": { "mean_score": 0.5513, "n": 10, "passed": 3 }, "it_deployment_gate_v03": { "mean_score": 0.6219, "n": 10, "passed": 3 }, "multi_device_pairing_v01": { "mean_score": 0.5383, "n": 10, "passed": 0 }, "multi_device_pairing_v02": { "mean_score": 0.7181, "n": 10, "passed": 2 }, "multi_device_pairing_v03": { "mean_score": 0.6989, "n": 10, "passed": 3 }, "multi_party_access_permit_v01": { "mean_score": 0.475, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.7619, "n": 10, "passed": 4 }, "multi_party_access_permit_v03": { "mean_score": 0.6295, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.4835, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.426, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.5764, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.35, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.535, "n": 10, "passed": 2 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.575, "n": 10, "passed": 3 }, "payment_chargeback_dispute_v01": { "mean_score": 0.8333, "n": 10, "passed": 7 }, "payment_chargeback_dispute_v02": { "mean_score": 0.5833, "n": 10, "passed": 2 }, "payment_chargeback_dispute_v03": { "mean_score": 0.4667, "n": 10, "passed": 3 }, "satellite_ground_station_handover_v01": { "mean_score": 0.6417, "n": 10, "passed": 4 }, "satellite_ground_station_handover_v02": { "mean_score": 0.8, "n": 10, "passed": 6 }, "satellite_ground_station_handover_v03": { "mean_score": 0.8083, "n": 10, "passed": 6 }, "sla_support_response_v01": { "mean_score": 0.5267, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.6048, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.4435, "n": 10, "passed": 1 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.575, "n": 10, "passed": 3 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.5833, "n": 10, "passed": 2 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.355, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 24, "fail_closed_pct": 1.1, "n": 2137 }, "side_effect_fields": { "approval_escalated": 7, "approval_granted": 3, "approval_recorded": 17, "approval_requested": 22, "assets_accessed": 63, "booking_extended": 4, "booking_held": 24, "booking_released": 6, "containment_action": 19, "deployment_approved": 8, "deployment_cutover": 1, "deployment_gate_set": 9, "deployment_requested": 30, "files_created": 4, "lock_acquired": 12, "lock_released": 1, "messages_sent": 1, "pairing_code_entered": 3, "pairing_code_shown": 1, "pairing_completed": 1, "pairing_started": 9, "review_assigned": 3, "review_submitted": 7 }, "step_mean": 10.64, "step_p50": 10, "stops": { "incomplete_required_submission": 200, "missing_required_side_effect": 255, "none": 125, "reached max_steps=20": 20 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 10": 10, "wait 120": 3, "wait 15": 4, "wait 30": 127, "wait 300": 4, "wait 45": 5, "wait 5": 17, "wait 60": 39 }, "max": 600.0, "mean": 42.4, "median": 30.0, "n": 215, "sub_30": 34, "total": 9110.0 }, "zero_open_tasks": 92, "zero_score": 36, "zero_write_tasks": 6 }, "terra": { "avg_score": 0.6024, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 8, "checks_total": 3760, "display": "gpt-5.6-terra", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'FULLY_RECONCILED_AND_GRANTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r10" }, { "message": "files_created.content contains 'ERR-DATA-BACKFILL-409': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r03" }, { "message": "files_created.content contains 'AUTH-GATE-9942-SYNC': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v02_r03" } ], "content_file": [ { "message": "LLM semantic match: The report details an initial UNRECONCILED state with two approver mismatches, but its final decisive finding is GRANTED rather than UNRECONCILED. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The report references transaction ID TXN-APP-WRITE-9912 instead of TXN-APP-WRITE-9908. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r04" }, { "message": "LLM semantic match: The report states the formal verdict as APPROVED conditional on handoff controls, rather than PENDING RUNBOOK RELEASE (Passcode Verification Required). [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r05" }, { "message": "LLM semantic match: The report explicitly rejects the 0.15 RRR score as belonging to a different ticket, finding instead that REQ-8841 has a 0.42 RRR score and is denied. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r05" } ], "content_msg": [ { "message": "LLM semantic match: No message was sent in #iam-approvals-sec to Sana_Security_Reviewer citing permit PERMIT-2026-BP-9982 and report path /reports/access_grant_decision.md. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The message requested re-approval for WRITE scope on billing-prod from Ravi_Application_Owner in #iam-approvals-sec, but cited reference code AUDIT-APP-2026-ARCH instead of AUDIT-SEC-2026-READ. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: No message was sent to Quinn_Engineering_Manager in #proj-billing-migration confirming that passcode SEC-TELEMETRY-9914-BP was submitted to Sana. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "LLM semantic match: No message was sent to Ravi_Application_Owner confirming that passcode SEC-TELEMETRY-9914-BP was verified. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r07': False; normalized fallback disabled ('iam_console_access_scope_matrix_r07' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "assets_accessed.id contains 'security_dashboard_telemetry_20260330_100000': False; normalized fallback disabled ('security_dashboard_telemetry_20260330_100000' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r08" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20241024_085500': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20241024_085500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v02_r07" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20250218_103000': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20250218_103000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r07" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r04" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 63 }, "content_file": { "diagnostic": 0, "required": 216 }, "content_msg": { "diagnostic": 0, "required": 643 }, "grounding": { "diagnostic": 103, "required": 54 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 334 }, "window": { "diagnostic": 0, "required": 27 } }, "failed_total": 1440, "gating": { "content_involved": 412, "grounding_only": 8, "grounding_only_ids": [ "conflict_vlm_distributed_lock_shared_resource_v03_r06", "conflict_vlm_employee_onboarding_workflow_v02_r06", "conflict_vlm_employee_onboarding_workflow_v02_r08", "conflict_vlm_expense_reimbursement_conflict_v01_r04", "conflict_vlm_expense_reimbursement_conflict_v01_r06", "conflict_vlm_expense_reimbursement_conflict_v03_r07", "conflict_vlm_it_deployment_gate_v03_r05", "conflict_vlm_it_deployment_gate_v03_r10" ], "n_fail": 480, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 7, "score": 0.5, "task_id": "conflict_vlm_access_request_approval_v01_r01", "verb": "asset open" }, { "count": 6, "score": 0.3333333333333333, "task_id": "conflict_vlm_incident_commander_selection_v01_r06", "verb": "message search" }, { "count": 7, "score": 0.6666666666666666, "task_id": "conflict_vlm_payment_chargeback_dispute_v03_r03", "verb": "message search" }, { "count": 5, "score": 0.16666666666666666, "task_id": "conflict_vlm_sla_support_response_v03_r02", "verb": "agent status" } ], "loop_verbs": { "agent status": 1, "asset open": 1, "file read": 1, "message search": 2 }, "loopy_capped": 5, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 283, "ok_false": 0, "pass_pct": 20.0, "passed": 120, "rank": { "avg_score": 5, "capped": 3, "failed_total": 5, "loopy_capped": 2, "near_miss": 12, "pass_pct": 7, "step_mean": 3, "step_p50": 2, "zero_open_tasks": 5, "zero_score": 5, "zero_write_tasks": 1 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.7343, "n": 10, "passed": 2 }, "access_request_approval_v02": { "mean_score": 0.7267, "n": 10, "passed": 1 }, "access_request_approval_v03": { "mean_score": 0.5663, "n": 10, "passed": 2 }, "barrier_sync_deployment_v01": { "mean_score": 0.6186, "n": 10, "passed": 2 }, "barrier_sync_deployment_v02": { "mean_score": 0.7638, "n": 10, "passed": 4 }, "barrier_sync_deployment_v03": { "mean_score": 0.7267, "n": 10, "passed": 3 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.2633, "n": 10, "passed": 0 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.4381, "n": 10, "passed": 0 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.442, "n": 10, "passed": 1 }, "collaborative_contract_review_v01": { "mean_score": 0.6224, "n": 10, "passed": 2 }, "collaborative_contract_review_v02": { "mean_score": 0.8283, "n": 10, "passed": 6 }, "collaborative_contract_review_v03": { "mean_score": 0.8098, "n": 10, "passed": 4 }, "coordinated_containment_v01": { "mean_score": 0.3954, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.7084, "n": 10, "passed": 4 }, "coordinated_containment_v03": { "mean_score": 0.7305, "n": 10, "passed": 1 }, "deadlock_multi_approval_v01": { "mean_score": 0.6642, "n": 10, "passed": 3 }, "deadlock_multi_approval_v02": { "mean_score": 0.7167, "n": 10, "passed": 3 }, "deadlock_multi_approval_v03": { "mean_score": 0.73, "n": 10, "passed": 2 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.6405, "n": 10, "passed": 3 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.2998, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.3976, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v01": { "mean_score": 0.5287, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v02": { "mean_score": 0.5282, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v03": { "mean_score": 0.5318, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.7917, "n": 10, "passed": 4 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.575, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.66, "n": 10, "passed": 0 }, "expiring_resource_hold_v01": { "mean_score": 0.6295, "n": 10, "passed": 3 }, "expiring_resource_hold_v02": { "mean_score": 0.4679, "n": 10, "passed": 1 }, "expiring_resource_hold_v03": { "mean_score": 0.7764, "n": 10, "passed": 1 }, "incident_commander_selection_v01": { "mean_score": 0.4833, "n": 10, "passed": 1 }, "incident_commander_selection_v02": { "mean_score": 0.6783, "n": 10, "passed": 4 }, "incident_commander_selection_v03": { "mean_score": 0.6683, "n": 10, "passed": 2 }, "it_deployment_gate_v01": { "mean_score": 0.6686, "n": 10, "passed": 3 }, "it_deployment_gate_v02": { "mean_score": 0.6382, "n": 10, "passed": 4 }, "it_deployment_gate_v03": { "mean_score": 0.7451, "n": 10, "passed": 2 }, "multi_device_pairing_v01": { "mean_score": 0.4878, "n": 10, "passed": 0 }, "multi_device_pairing_v02": { "mean_score": 0.6892, "n": 10, "passed": 1 }, "multi_device_pairing_v03": { "mean_score": 0.6133, "n": 10, "passed": 3 }, "multi_party_access_permit_v01": { "mean_score": 0.5417, "n": 10, "passed": 1 }, "multi_party_access_permit_v02": { "mean_score": 0.7345, "n": 10, "passed": 2 }, "multi_party_access_permit_v03": { "mean_score": 0.6629, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.513, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.4692, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.5821, "n": 10, "passed": 2 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.4233, "n": 10, "passed": 2 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.4733, "n": 10, "passed": 0 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.48, "n": 10, "passed": 3 }, "payment_chargeback_dispute_v01": { "mean_score": 0.8667, "n": 10, "passed": 6 }, "payment_chargeback_dispute_v02": { "mean_score": 0.425, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v03": { "mean_score": 0.7333, "n": 10, "passed": 4 }, "satellite_ground_station_handover_v01": { "mean_score": 0.6083, "n": 10, "passed": 4 }, "satellite_ground_station_handover_v02": { "mean_score": 0.8, "n": 10, "passed": 6 }, "satellite_ground_station_handover_v03": { "mean_score": 0.7583, "n": 10, "passed": 6 }, "sla_support_response_v01": { "mean_score": 0.5529, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.6431, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.4268, "n": 10, "passed": 1 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.555, "n": 10, "passed": 2 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.6417, "n": 10, "passed": 4 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.27, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 21, "fail_closed_pct": 1.0, "n": 2137 }, "side_effect_fields": { "approval_escalated": 8, "approval_granted": 3, "approval_recorded": 19, "approval_requested": 21, "assets_accessed": 43, "booking_extended": 4, "booking_held": 24, "booking_released": 11, "containment_action": 20, "deployment_approved": 8, "deployment_gate_set": 10, "deployment_requested": 33, "files_created": 2, "lock_acquired": 15, "messages_sent": 8, "pairing_code_entered": 5, "pairing_code_shown": 1, "pairing_completed": 1, "pairing_started": 12, "review_assigned": 5, "review_submitted": 8 }, "step_mean": 11.35, "step_p50": 11, "stops": { "incomplete_required_submission": 210, "missing_required_side_effect": 261, "none": 120, "reached max_steps=20": 8, "stopped after 3 identical no-progress transitions": 1 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 10": 63, "wait 15": 6, "wait 20": 6, "wait 3": 23, "wait 30": 8, "wait 4": 6, "wait 5": 124, "wait 8": 17 }, "max": 300.0, "mean": 13.4, "median": 5.0, "n": 267, "sub_30": 252, "total": 3570.0 }, "zero_open_tasks": 1, "zero_score": 33, "zero_write_tasks": 1 } } }, "low": { "head_to_head": { "flash": { "flash36": { "loss": 52, "tie": 3, "win": 5 }, "flash37": { "loss": 52, "tie": 2, "win": 6 }, "luna": { "loss": 51, "tie": 1, "win": 8 }, "opus": { "loss": 55, "tie": 0, "win": 5 }, "opus5": { "loss": 50, "tie": 0, "win": 10 }, "pro": { "loss": 21, "tie": 1, "win": 38 }, "sol": { "loss": 50, "tie": 2, "win": 8 }, "sonnet": { "loss": 39, "tie": 0, "win": 21 }, "terra": { "loss": 47, "tie": 0, "win": 13 } }, "flash36": { "flash": { "loss": 5, "tie": 3, "win": 52 }, "flash37": { "loss": 29, "tie": 4, "win": 27 }, "luna": { "loss": 26, "tie": 4, "win": 30 }, "opus": { "loss": 37, "tie": 3, "win": 20 }, "opus5": { "loss": 36, "tie": 0, "win": 24 }, "pro": { "loss": 2, "tie": 0, "win": 58 }, "sol": { "loss": 36, "tie": 2, "win": 22 }, "sonnet": { "loss": 20, "tie": 1, "win": 39 }, "terra": { "loss": 20, "tie": 3, "win": 37 } }, "flash37": { "flash": { "loss": 6, "tie": 2, "win": 52 }, "flash36": { "loss": 27, "tie": 4, "win": 29 }, "luna": { "loss": 28, "tie": 3, "win": 29 }, "opus": { "loss": 37, "tie": 2, "win": 21 }, "opus5": { "loss": 33, "tie": 1, "win": 26 }, "pro": { "loss": 3, "tie": 0, "win": 57 }, "sol": { "loss": 34, "tie": 2, "win": 24 }, "sonnet": { "loss": 14, "tie": 1, "win": 45 }, "terra": { "loss": 20, "tie": 1, "win": 39 } }, "luna": { "flash": { "loss": 8, "tie": 1, "win": 51 }, "flash36": { "loss": 30, "tie": 4, "win": 26 }, "flash37": { "loss": 29, "tie": 3, "win": 28 }, "opus": { "loss": 38, "tie": 3, "win": 19 }, "opus5": { "loss": 36, "tie": 2, "win": 22 }, "pro": { "loss": 5, "tie": 0, "win": 55 }, "sol": { "loss": 39, "tie": 2, "win": 19 }, "sonnet": { "loss": 18, "tie": 1, "win": 41 }, "terra": { "loss": 21, "tie": 4, "win": 35 } }, "opus": { "flash": { "loss": 5, "tie": 0, "win": 55 }, "flash36": { "loss": 20, "tie": 3, "win": 37 }, "flash37": { "loss": 21, "tie": 2, "win": 37 }, "luna": { "loss": 19, "tie": 3, "win": 38 }, "opus5": { "loss": 32, "tie": 2, "win": 26 }, "pro": { "loss": 4, "tie": 1, "win": 55 }, "sol": { "loss": 29, "tie": 0, "win": 31 }, "sonnet": { "loss": 15, "tie": 0, "win": 45 }, "terra": { "loss": 17, "tie": 2, "win": 41 } }, "opus5": { "flash": { "loss": 10, "tie": 0, "win": 50 }, "flash36": { "loss": 24, "tie": 0, "win": 36 }, "flash37": { "loss": 26, "tie": 1, "win": 33 }, "luna": { "loss": 22, "tie": 2, "win": 36 }, "opus": { "loss": 26, "tie": 2, "win": 32 }, "pro": { "loss": 6, "tie": 0, "win": 54 }, "sol": { "loss": 30, "tie": 4, "win": 26 }, "sonnet": { "loss": 13, "tie": 0, "win": 47 }, "terra": { "loss": 17, "tie": 1, "win": 42 } }, "pro": { "flash": { "loss": 38, "tie": 1, "win": 21 }, "flash36": { "loss": 58, "tie": 0, "win": 2 }, "flash37": { "loss": 57, "tie": 0, "win": 3 }, "luna": { "loss": 55, "tie": 0, "win": 5 }, "opus": { "loss": 55, "tie": 1, "win": 4 }, "opus5": { "loss": 54, "tie": 0, "win": 6 }, "sol": { "loss": 58, "tie": 0, "win": 2 }, "sonnet": { "loss": 46, "tie": 0, "win": 14 }, "terra": { "loss": 54, "tie": 0, "win": 6 } }, "sol": { "flash": { "loss": 8, "tie": 2, "win": 50 }, "flash36": { "loss": 22, "tie": 2, "win": 36 }, "flash37": { "loss": 24, "tie": 2, "win": 34 }, "luna": { "loss": 19, "tie": 2, "win": 39 }, "opus": { "loss": 31, "tie": 0, "win": 29 }, "opus5": { "loss": 26, "tie": 4, "win": 30 }, "pro": { "loss": 2, "tie": 0, "win": 58 }, "sonnet": { "loss": 11, "tie": 2, "win": 47 }, "terra": { "loss": 10, "tie": 6, "win": 44 } }, "sonnet": { "flash": { "loss": 21, "tie": 0, "win": 39 }, "flash36": { "loss": 39, "tie": 1, "win": 20 }, "flash37": { "loss": 45, "tie": 1, "win": 14 }, "luna": { "loss": 41, "tie": 1, "win": 18 }, "opus": { "loss": 45, "tie": 0, "win": 15 }, "opus5": { "loss": 47, "tie": 0, "win": 13 }, "pro": { "loss": 14, "tie": 0, "win": 46 }, "sol": { "loss": 47, "tie": 2, "win": 11 }, "terra": { "loss": 38, "tie": 1, "win": 21 } }, "terra": { "flash": { "loss": 13, "tie": 0, "win": 47 }, "flash36": { "loss": 37, "tie": 3, "win": 20 }, "flash37": { "loss": 39, "tie": 1, "win": 20 }, "luna": { "loss": 35, "tie": 4, "win": 21 }, "opus": { "loss": 41, "tie": 2, "win": 17 }, "opus5": { "loss": 42, "tie": 1, "win": 17 }, "pro": { "loss": 6, "tie": 0, "win": 54 }, "sol": { "loss": 44, "tie": 6, "win": 10 }, "sonnet": { "loss": 21, "tie": 1, "win": 38 } } }, "models": { "flash": { "avg_score": 0.4726, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 232, "checks_total": 3760, "display": "gemini-3.5-flash", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'SEC-TELEMETRY-9914-BP': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "files_created.content contains 'PERMIT-2026-BP-9982': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r08" }, { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'ERR-DATA-BACKFILL-409': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r03" } ], "content_file": [ { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r08" } ], "content_msg": [ { "message": "LLM semantic match: No message was sent to Sana_Security_Reviewer in #iam-approvals-sec citing permit PERMIT-2026-BP-9982 and report path /reports/access_grant_decision.md. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The message was sent to Sana in #iam-approvals-sec citing AUDIT-SEC-2026-READ for WRITE authorization on billing-prod, but omitted the required passcode SEC-PASS-8842-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: The message sent to Ravi_Application_Owner in #iam-approvals-sec requested re-approval for WRITE scope on billing-prod, but cited reference code AUDIT-APP-2026-ARCH instead of AUDIT-SEC-2026-READ. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: The message in #iam-approvals-sec informs Ravi of the dataset mismatch, but does not explicitly request a corrected sign-off for billing-prod with WRITE scope. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r01': False; normalized fallback disabled ('iam_console_access_scope_matrix_r01' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r07': False; normalized fallback disabled ('iam_console_access_scope_matrix_r07' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20250218_101000': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20250218_101000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r04" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20250218_101000': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20250218_101000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r05" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r04" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r05" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 123 }, "content_file": { "diagnostic": 0, "required": 422 }, "content_msg": { "diagnostic": 0, "required": 734 }, "grounding": { "diagnostic": 134, "required": 61 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 333 }, "window": { "diagnostic": 0, "required": 28 } }, "failed_total": 1835, "gating": { "content_involved": 469, "grounding_only": 2, "grounding_only_ids": [ "conflict_vlm_distributed_lock_shared_resource_v01_r09", "conflict_vlm_it_deployment_gate_v03_r10" ], "n_fail": 519, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 6, "score": 0.5714285714285714, "task_id": "conflict_vlm_access_request_approval_v01_r03", "verb": "message search" }, { "count": 6, "score": 0.14285714285714285, "task_id": "conflict_vlm_access_request_approval_v01_r05", "verb": "file read" }, { "count": 9, "score": 0.8, "task_id": "conflict_vlm_access_request_approval_v01_r06", "verb": "file read" }, { "count": 8, "score": 0.1, "task_id": "conflict_vlm_access_request_approval_v01_r08", "verb": "file read" } ], "loop_verbs": { "agent status": 2, "asset open": 21, "booking status": 4, "file read": 75, "message search": 98, "message send": 5 }, "loopy_capped": 210, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 203, "ok_false": 0, "pass_pct": 13.5, "passed": 81, "rank": { "avg_score": 9, "capped": 10, "failed_total": 8, "loopy_capped": 10, "near_miss": 1, "pass_pct": 9, "step_mean": 10, "step_p50": 10, "zero_open_tasks": 8, "zero_score": 10, "zero_write_tasks": 10 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.5836, "n": 10, "passed": 2 }, "access_request_approval_v02": { "mean_score": 0.6792, "n": 10, "passed": 1 }, "access_request_approval_v03": { "mean_score": 0.7862, "n": 10, "passed": 4 }, "barrier_sync_deployment_v01": { "mean_score": 0.5612, "n": 10, "passed": 2 }, "barrier_sync_deployment_v02": { "mean_score": 0.4936, "n": 10, "passed": 2 }, "barrier_sync_deployment_v03": { "mean_score": 0.6467, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.28, "n": 10, "passed": 0 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.2345, "n": 10, "passed": 0 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.1765, "n": 10, "passed": 0 }, "collaborative_contract_review_v01": { "mean_score": 0.5051, "n": 10, "passed": 2 }, "collaborative_contract_review_v02": { "mean_score": 0.7117, "n": 10, "passed": 5 }, "collaborative_contract_review_v03": { "mean_score": 0.4685, "n": 10, "passed": 1 }, "coordinated_containment_v01": { "mean_score": 0.3123, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.5171, "n": 10, "passed": 1 }, "coordinated_containment_v03": { "mean_score": 0.7043, "n": 10, "passed": 2 }, "deadlock_multi_approval_v01": { "mean_score": 0.6758, "n": 10, "passed": 2 }, "deadlock_multi_approval_v02": { "mean_score": 0.7117, "n": 10, "passed": 4 }, "deadlock_multi_approval_v03": { "mean_score": 0.6214, "n": 10, "passed": 1 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.4498, "n": 10, "passed": 1 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.2262, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.3518, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v01": { "mean_score": 0.4429, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v02": { "mean_score": 0.4174, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v03": { "mean_score": 0.4826, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.435, "n": 10, "passed": 2 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.48, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.5074, "n": 10, "passed": 0 }, "expiring_resource_hold_v01": { "mean_score": 0.406, "n": 10, "passed": 0 }, "expiring_resource_hold_v02": { "mean_score": 0.5679, "n": 10, "passed": 2 }, "expiring_resource_hold_v03": { "mean_score": 0.5964, "n": 10, "passed": 0 }, "incident_commander_selection_v01": { "mean_score": 0.175, "n": 10, "passed": 0 }, "incident_commander_selection_v02": { "mean_score": 0.49, "n": 10, "passed": 1 }, "incident_commander_selection_v03": { "mean_score": 0.4683, "n": 10, "passed": 1 }, "it_deployment_gate_v01": { "mean_score": 0.6286, "n": 10, "passed": 3 }, "it_deployment_gate_v02": { "mean_score": 0.6715, "n": 10, "passed": 5 }, "it_deployment_gate_v03": { "mean_score": 0.4703, "n": 10, "passed": 0 }, "multi_device_pairing_v01": { "mean_score": 0.4872, "n": 10, "passed": 0 }, "multi_device_pairing_v02": { "mean_score": 0.5081, "n": 10, "passed": 0 }, "multi_device_pairing_v03": { "mean_score": 0.541, "n": 10, "passed": 3 }, "multi_party_access_permit_v01": { "mean_score": 0.4383, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.6964, "n": 10, "passed": 4 }, "multi_party_access_permit_v03": { "mean_score": 0.5795, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.3958, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.451, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.4811, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.42, "n": 10, "passed": 2 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.2783, "n": 10, "passed": 0 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.3733, "n": 10, "passed": 2 }, "payment_chargeback_dispute_v01": { "mean_score": 0.7333, "n": 10, "passed": 6 }, "payment_chargeback_dispute_v02": { "mean_score": 0.4583, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v03": { "mean_score": 0.0667, "n": 10, "passed": 0 }, "satellite_ground_station_handover_v01": { "mean_score": 0.3, "n": 10, "passed": 2 }, "satellite_ground_station_handover_v02": { "mean_score": 0.4, "n": 10, "passed": 3 }, "satellite_ground_station_handover_v03": { "mean_score": 0.4333, "n": 10, "passed": 1 }, "sla_support_response_v01": { "mean_score": 0.5395, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.5762, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.3815, "n": 10, "passed": 0 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.4083, "n": 10, "passed": 3 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.2667, "n": 10, "passed": 2 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.2033, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 335, "fail_closed_pct": 15.7, "n": 2137 }, "side_effect_fields": { "approval_escalated": 4, "approval_granted": 2, "approval_recorded": 11, "approval_requested": 16, "assets_accessed": 36, "booking_extended": 3, "booking_held": 14, "booking_released": 6, "containment_action": 6, "deployment_approved": 8, "deployment_gate_set": 5, "deployment_requested": 21, "lock_acquired": 5, "messages_sent": 9, "pairing_code_entered": 2, "pairing_code_shown": 1, "pairing_started": 10, "review_assigned": 1, "review_submitted": 5 }, "step_mean": 16.22, "step_p50": 18, "stops": { "incomplete_required_submission": 122, "missing_required_side_effect": 165, "none": 81, "reached max_steps=20": 232 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 10": 203, "wait 120": 3, "wait 15": 13, "wait 180": 1, "wait 20": 4, "wait 30": 52, "wait 5": 142, "wait 60": 4 }, "max": 300.0, "mean": 13.4, "median": 10.0, "n": 423, "sub_30": 362, "total": 5655.0 }, "zero_open_tasks": 59, "zero_score": 89, "zero_write_tasks": 164 }, "flash36": { "avg_score": 0.5953, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 11, "checks_total": 3760, "display": "gemini-3.6-flash", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'SEC-TELEMETRY-9914-BP': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'FULLY_RECONCILED_AND_GRANTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r10" }, { "message": "files_created.content contains 'ERR-DATA-BACKFILL-409': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r03" } ], "content_file": [ { "message": "LLM semantic match: The report states the final decision as GRANTED & RECONCILED rather than UNRECONCILED (2/3 approver scopes mismatched). [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The report references transaction ID TXN-APP-WRITE-9912 instead of TXN-APP-WRITE-9908. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r04" }, { "message": "LLM semantic match: The report states the decisive finding as 'RECONCILED & APPROVED (WRITE Scope)' rather than 'PENDING RUNBOOK RELEASE (Passcode Verification Required)'. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r05" }, { "message": "LLM semantic match: The report states that dataset provisioning authorization is blocked/rejected rather than authorized. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r10" } ], "content_msg": [ { "message": "LLM semantic match: The message sent to Sana_Security_Reviewer in #iam-approvals-sec requested updated WRITE authorization and cited AUDIT-SEC-2026-READ, but failed to include the passcode SEC-PASS-8842-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: No message was sent to Quinn_Engineering_Manager in #proj-billing-migration mentioning Ravi's tracking code REAPP-BP-2026-0842-AO. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r04" }, { "message": "LLM semantic match: The message confirming the tracking code registration was sent to Ravi_Application_Owner instead of Sana_Security_Reviewer. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r04" }, { "message": "LLM semantic match: The message was sent to Sana_Security_Reviewer in #iam-approvals-sec to request the scope upgrade, but it provided an incorrect security telemetry passcode. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r01': False; normalized fallback disabled ('iam_console_access_scope_matrix_r01' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r07': False; normalized fallback disabled ('iam_console_access_scope_matrix_r07' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "assets_accessed.id contains 'security_dashboard_telemetry_20260330_100000': False; normalized fallback disabled ('security_dashboard_telemetry_20260330_100000' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r08" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20241024_085500': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20241024_085500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v02_r07" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r04" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 63 }, "content_file": { "diagnostic": 0, "required": 204 }, "content_msg": { "diagnostic": 0, "required": 662 }, "grounding": { "diagnostic": 149, "required": 63 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 341 }, "window": { "diagnostic": 0, "required": 30 } }, "failed_total": 1512, "gating": { "content_involved": 412, "grounding_only": 6, "grounding_only_ids": [ "conflict_vlm_barrier_sync_deployment_v01_r07", "conflict_vlm_barrier_sync_deployment_v02_r04", "conflict_vlm_distributed_lock_shared_resource_v01_r03", "conflict_vlm_distributed_lock_shared_resource_v03_r06", "conflict_vlm_expense_reimbursement_conflict_v01_r04", "conflict_vlm_multi_device_pairing_v01_r04" ], "n_fail": 489, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 6, "score": 0.5, "task_id": "conflict_vlm_access_request_approval_v01_r01", "verb": "approval approve" }, { "count": 6, "score": 0.5, "task_id": "conflict_vlm_clinical_trial_dosage_conflict_v02_r02", "verb": "message search" }, { "count": 5, "score": 0.42857142857142855, "task_id": "conflict_vlm_coordinated_containment_v03_r02", "verb": "file read" }, { "count": 7, "score": 0.5, "task_id": "conflict_vlm_distributed_lock_shared_resource_v01_r02", "verb": "file read" } ], "loop_verbs": { "approval approve": 1, "asset open": 1, "file read": 4, "message search": 2 }, "loopy_capped": 8, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 287, "ok_false": 0, "pass_pct": 18.5, "passed": 111, "rank": { "avg_score": 5, "capped": 7, "failed_total": 4, "loopy_capped": 7, "near_miss": 10, "pass_pct": 6, "step_mean": 7, "step_p50": 4, "zero_open_tasks": 7, "zero_score": 4, "zero_write_tasks": 8 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.6724, "n": 10, "passed": 1 }, "access_request_approval_v02": { "mean_score": 0.7192, "n": 10, "passed": 1 }, "access_request_approval_v03": { "mean_score": 0.6348, "n": 10, "passed": 2 }, "barrier_sync_deployment_v01": { "mean_score": 0.6694, "n": 10, "passed": 1 }, "barrier_sync_deployment_v02": { "mean_score": 0.7638, "n": 10, "passed": 4 }, "barrier_sync_deployment_v03": { "mean_score": 0.6567, "n": 10, "passed": 2 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.2967, "n": 10, "passed": 0 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.5774, "n": 10, "passed": 3 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.3465, "n": 10, "passed": 1 }, "collaborative_contract_review_v01": { "mean_score": 0.5444, "n": 10, "passed": 1 }, "collaborative_contract_review_v02": { "mean_score": 0.7117, "n": 10, "passed": 4 }, "collaborative_contract_review_v03": { "mean_score": 0.7931, "n": 10, "passed": 4 }, "coordinated_containment_v01": { "mean_score": 0.4149, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.5297, "n": 10, "passed": 0 }, "coordinated_containment_v03": { "mean_score": 0.7662, "n": 10, "passed": 3 }, "deadlock_multi_approval_v01": { "mean_score": 0.6342, "n": 10, "passed": 2 }, "deadlock_multi_approval_v02": { "mean_score": 0.8075, "n": 10, "passed": 3 }, "deadlock_multi_approval_v03": { "mean_score": 0.74, "n": 10, "passed": 4 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.6848, "n": 10, "passed": 2 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.3321, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.3976, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v01": { "mean_score": 0.5993, "n": 10, "passed": 2 }, "employee_onboarding_workflow_v02": { "mean_score": 0.5556, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v03": { "mean_score": 0.5076, "n": 10, "passed": 0 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.8017, "n": 10, "passed": 4 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.71, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.5781, "n": 10, "passed": 1 }, "expiring_resource_hold_v01": { "mean_score": 0.6712, "n": 10, "passed": 3 }, "expiring_resource_hold_v02": { "mean_score": 0.5988, "n": 10, "passed": 1 }, "expiring_resource_hold_v03": { "mean_score": 0.6379, "n": 10, "passed": 0 }, "incident_commander_selection_v01": { "mean_score": 0.425, "n": 10, "passed": 0 }, "incident_commander_selection_v02": { "mean_score": 0.6717, "n": 10, "passed": 4 }, "incident_commander_selection_v03": { "mean_score": 0.6067, "n": 10, "passed": 2 }, "it_deployment_gate_v01": { "mean_score": 0.6159, "n": 10, "passed": 3 }, "it_deployment_gate_v02": { "mean_score": 0.679, "n": 10, "passed": 4 }, "it_deployment_gate_v03": { "mean_score": 0.6795, "n": 10, "passed": 1 }, "multi_device_pairing_v01": { "mean_score": 0.5578, "n": 10, "passed": 0 }, "multi_device_pairing_v02": { "mean_score": 0.7592, "n": 10, "passed": 4 }, "multi_device_pairing_v03": { "mean_score": 0.6366, "n": 10, "passed": 3 }, "multi_party_access_permit_v01": { "mean_score": 0.515, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.7679, "n": 10, "passed": 3 }, "multi_party_access_permit_v03": { "mean_score": 0.6495, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.414, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.4033, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.5261, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.42, "n": 10, "passed": 2 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.4567, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.465, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v01": { "mean_score": 0.8417, "n": 10, "passed": 6 }, "payment_chargeback_dispute_v02": { "mean_score": 0.4833, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v03": { "mean_score": 0.6667, "n": 10, "passed": 3 }, "satellite_ground_station_handover_v01": { "mean_score": 0.5833, "n": 10, "passed": 3 }, "satellite_ground_station_handover_v02": { "mean_score": 0.8333, "n": 10, "passed": 6 }, "satellite_ground_station_handover_v03": { "mean_score": 0.6333, "n": 10, "passed": 3 }, "sla_support_response_v01": { "mean_score": 0.5329, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.5905, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.4435, "n": 10, "passed": 0 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.655, "n": 10, "passed": 3 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.65, "n": 10, "passed": 4 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.2033, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 42, "fail_closed_pct": 2.0, "n": 2137 }, "side_effect_fields": { "approval_escalated": 7, "approval_granted": 3, "approval_recorded": 18, "approval_requested": 19, "assets_accessed": 56, "booking_extended": 4, "booking_held": 24, "booking_released": 11, "containment_action": 17, "deployment_approved": 9, "deployment_gate_set": 11, "deployment_requested": 30, "files_created": 1, "lock_acquired": 12, "messages_sent": 20, "pairing_code_entered": 3, "pairing_code_shown": 1, "pairing_completed": 1, "pairing_started": 11, "review_assigned": 5, "review_submitted": 8 }, "step_mean": 9.99, "step_p50": 9, "stops": { "incomplete_required_submission": 207, "missing_required_side_effect": 271, "none": 111, "reached max_steps=20": 11 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 10": 6, "wait 300": 1, "wait 5": 7 }, "max": 300.0, "mean": 28.2, "median": 7.5, "n": 14, "sub_30": 13, "total": 395.0 }, "zero_open_tasks": 25, "zero_score": 31, "zero_write_tasks": 4 }, "flash37": { "avg_score": 0.5998, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 18, "checks_total": 3760, "display": "gemini-3.7-flash", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'FULLY_RECONCILED_AND_GRANTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r10" }, { "message": "files_created.content contains 'ERR-DATA-BACKFILL-409': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r03" }, { "message": "files_created.content contains 'AUTH-GATE-9942-SYNC': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v02_r03" } ], "content_file": [ { "message": "LLM semantic match: The report states that the request was GRANTED & RECONCILED rather than having a decisive finding of UNRECONCILED (2/3 approver scopes mismatched). [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The report states Transaction ID TXN-APP-WRITE-9912 instead of the required TXN-APP-WRITE-9908. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r04" }, { "message": "LLM semantic match: The report captures the required intent regarding pending runbook release and passcode verification, but does not state the exact specified decisive finding phrase. [self-consistency 1/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r05" }, { "message": "LLM semantic match: The report states the decision verdict as 'APPROVED - RUNBOOK HANDOFF CLEARED' rather than 'verify visual passcode'. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r08" } ], "content_msg": [ { "message": "LLM semantic match: No message was sent to Sana_Security_Reviewer citing permit PERMIT-2026-BP-9982 and report path /reports/access_grant_decision.md to authorize releasing the migration runbook. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The message was sent to Sana_Security_Reviewer in #iam-approvals-sec citing AUDIT-SEC-2026-READ and requesting WRITE authorization on billing-prod, but failed to include the passcode SEC-PASS-8842-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: A message was sent to Ravi_Application_Owner in #iam-approvals-sec requesting re-approval for billing-prod, but it cited AUDIT-APP-2026-ARCH instead of AUDIT-SEC-2026-READ. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: The message was sent to Quinn_Engineering_Manager in #proj-billing-migration reporting WRITE scope on billing-prod, but stated the status as 'VERIFIED' rather than 'ALL RECONCILIATION CHECKS PASSED'. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r01': False; normalized fallback disabled ('iam_console_access_scope_matrix_r01' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r07': False; normalized fallback disabled ('iam_console_access_scope_matrix_r07' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "assets_accessed.id contains 'iam_portal_approval_audit_log_20241024_083500': False; normalized fallback disabled ('iam_portal_approval_audit_log_20241024_083500' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r02" }, { "message": "assets_accessed.id contains 'policy_center_access_policy_doc_20241024_084000': False; normalized fallback disabled ('policy_center_access_policy_doc_20241024_084000' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r03" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r04" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 61 }, "content_file": { "diagnostic": 0, "required": 194 }, "content_msg": { "diagnostic": 0, "required": 647 }, "grounding": { "diagnostic": 222, "required": 100 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 325 }, "window": { "diagnostic": 0, "required": 29 } }, "failed_total": 1578, "gating": { "content_involved": 399, "grounding_only": 8, "grounding_only_ids": [ "conflict_vlm_collaborative_contract_review_v02_r08", "conflict_vlm_deadlock_multi_approval_v01_r04", "conflict_vlm_distributed_lock_shared_resource_v01_r04", "conflict_vlm_distributed_lock_shared_resource_v03_r06", "conflict_vlm_employee_onboarding_workflow_v03_r03", "conflict_vlm_expense_reimbursement_conflict_v01_r05", "conflict_vlm_expense_reimbursement_conflict_v03_r05", "conflict_vlm_expense_reimbursement_conflict_v03_r07" ], "n_fail": 477, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 5, "score": 0.8, "task_id": "conflict_vlm_barrier_sync_deployment_v03_r02", "verb": "file read" }, { "count": 5, "score": 0.2857142857142857, "task_id": "conflict_vlm_collaborative_contract_review_v03_r05", "verb": "file read" }, { "count": 8, "score": 0.4, "task_id": "conflict_vlm_deadlock_multi_approval_v03_r03", "verb": "asset open" }, { "count": 7, "score": 0.3333333333333333, "task_id": "conflict_vlm_distributed_lock_shared_resource_v02_r02", "verb": "file read" } ], "loop_verbs": { "asset open": 2, "file read": 8, "message search": 1, "message send": 1 }, "loopy_capped": 12, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 272, "ok_false": 0, "pass_pct": 20.5, "passed": 123, "rank": { "avg_score": 4, "capped": 8, "failed_total": 6, "loopy_capped": 8, "near_miss": 4, "pass_pct": 4, "step_mean": 9, "step_p50": 9, "zero_open_tasks": 9, "zero_score": 4, "zero_write_tasks": 5 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.7461, "n": 10, "passed": 2 }, "access_request_approval_v02": { "mean_score": 0.67, "n": 10, "passed": 0 }, "access_request_approval_v03": { "mean_score": 0.6148, "n": 10, "passed": 2 }, "barrier_sync_deployment_v01": { "mean_score": 0.7353, "n": 10, "passed": 4 }, "barrier_sync_deployment_v02": { "mean_score": 0.7221, "n": 10, "passed": 3 }, "barrier_sync_deployment_v03": { "mean_score": 0.7267, "n": 10, "passed": 3 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.3833, "n": 10, "passed": 2 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.5369, "n": 10, "passed": 4 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.3577, "n": 10, "passed": 0 }, "collaborative_contract_review_v01": { "mean_score": 0.5944, "n": 10, "passed": 1 }, "collaborative_contract_review_v02": { "mean_score": 0.7383, "n": 10, "passed": 5 }, "collaborative_contract_review_v03": { "mean_score": 0.7752, "n": 10, "passed": 4 }, "coordinated_containment_v01": { "mean_score": 0.3627, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.6146, "n": 10, "passed": 1 }, "coordinated_containment_v03": { "mean_score": 0.7305, "n": 10, "passed": 2 }, "deadlock_multi_approval_v01": { "mean_score": 0.6642, "n": 10, "passed": 2 }, "deadlock_multi_approval_v02": { "mean_score": 0.793, "n": 10, "passed": 4 }, "deadlock_multi_approval_v03": { "mean_score": 0.6957, "n": 10, "passed": 1 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.7464, "n": 10, "passed": 4 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.3045, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.3952, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v01": { "mean_score": 0.6004, "n": 10, "passed": 2 }, "employee_onboarding_workflow_v02": { "mean_score": 0.5246, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v03": { "mean_score": 0.5076, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.7333, "n": 10, "passed": 3 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.64, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.6707, "n": 10, "passed": 1 }, "expiring_resource_hold_v01": { "mean_score": 0.7271, "n": 10, "passed": 1 }, "expiring_resource_hold_v02": { "mean_score": 0.6012, "n": 10, "passed": 2 }, "expiring_resource_hold_v03": { "mean_score": 0.6555, "n": 10, "passed": 0 }, "incident_commander_selection_v01": { "mean_score": 0.475, "n": 10, "passed": 1 }, "incident_commander_selection_v02": { "mean_score": 0.655, "n": 10, "passed": 3 }, "incident_commander_selection_v03": { "mean_score": 0.6517, "n": 10, "passed": 3 }, "it_deployment_gate_v01": { "mean_score": 0.6759, "n": 10, "passed": 3 }, "it_deployment_gate_v02": { "mean_score": 0.6915, "n": 10, "passed": 5 }, "it_deployment_gate_v03": { "mean_score": 0.6882, "n": 10, "passed": 3 }, "multi_device_pairing_v01": { "mean_score": 0.5517, "n": 10, "passed": 0 }, "multi_device_pairing_v02": { "mean_score": 0.5859, "n": 10, "passed": 1 }, "multi_device_pairing_v03": { "mean_score": 0.6212, "n": 10, "passed": 3 }, "multi_party_access_permit_v01": { "mean_score": 0.5183, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.7512, "n": 10, "passed": 3 }, "multi_party_access_permit_v03": { "mean_score": 0.6612, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.439, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.406, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.6021, "n": 10, "passed": 2 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.42, "n": 10, "passed": 2 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.535, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.455, "n": 10, "passed": 2 }, "payment_chargeback_dispute_v01": { "mean_score": 0.8417, "n": 10, "passed": 7 }, "payment_chargeback_dispute_v02": { "mean_score": 0.4583, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v03": { "mean_score": 0.7, "n": 10, "passed": 4 }, "satellite_ground_station_handover_v01": { "mean_score": 0.5583, "n": 10, "passed": 3 }, "satellite_ground_station_handover_v02": { "mean_score": 0.8, "n": 10, "passed": 4 }, "satellite_ground_station_handover_v03": { "mean_score": 0.75, "n": 10, "passed": 5 }, "sla_support_response_v01": { "mean_score": 0.4886, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.6048, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.4182, "n": 10, "passed": 1 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.6083, "n": 10, "passed": 4 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.65, "n": 10, "passed": 4 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.1583, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 22, "fail_closed_pct": 1.0, "n": 2137 }, "side_effect_fields": { "approval_escalated": 5, "approval_granted": 3, "approval_recorded": 17, "approval_requested": 14, "assets_accessed": 90, "booking_extended": 2, "booking_held": 22, "booking_released": 5, "containment_action": 17, "deployment_approved": 6, "deployment_gate_set": 6, "deployment_requested": 23, "document_signed": 1, "files_created": 1, "lock_acquired": 9, "messages_sent": 63, "pairing_code_entered": 3, "pairing_code_shown": 1, "pairing_completed": 1, "pairing_started": 9, "review_assigned": 4, "review_submitted": 6 }, "step_mean": 12.18, "step_p50": 12, "stops": { "incomplete_required_submission": 151, "missing_required_side_effect": 308, "none": 123, "reached max_steps=20": 18 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 10": 8, "wait 15": 4, "wait 20": 1, "wait 30": 2, "wait 300": 2, "wait 5": 15 }, "max": 300.0, "mean": 28.0, "median": 10.0, "n": 32, "sub_30": 28, "total": 895.0 }, "zero_open_tasks": 226, "zero_score": 31, "zero_write_tasks": 2 }, "luna": { "avg_score": 0.5898, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 3, "checks_total": 3760, "display": "gpt-5.6-luna", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'SEC-TELEMETRY-9914-BP': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'SEC-VERIFY-4109': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r06" }, { "message": "files_created.content contains 'FULLY_RECONCILED_AND_GRANTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r10" } ], "content_file": [ { "message": "LLM semantic match: The report's final decisive finding is GRANTED rather than UNRECONCILED. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The report details verifying the passcode and requesting a security scope upgrade, but the final stated verdict is DENY / DO NOT ISSUE ACCESS PERMIT rather than approving the upgrade. [self-consistency 1/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "LLM semantic match: The report states that the access decision is unreconciled and not ready for permit issuance, failing to find that all reconciliation checks passed. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "LLM semantic match: The report explicitly states not to proceed with the access grant due to unreconciled approvals. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" } ], "content_msg": [ { "message": "LLM semantic match: The message sent to Sana_Security_Reviewer in #iam-approvals-sec requested updated WRITE authorization on billing-prod and cited AUDIT-SEC-2026-READ, but omitted the required passcode SEC-PASS-8842-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: The message was sent to Ravi_Application_Owner in #iam-approvals-sec requesting re-approval for WRITE scope on billing-prod, but cited AUDIT-APP-2026-ARCH instead of AUDIT-SEC-2026-READ. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: The request was sent to Sana_Security_Reviewer in #iam-approvals-sec for a WRITE upgrade on billing-prod, but it provided an incorrect passcode (SEC-PASS-8842-BP instead of SEC-TELEMETRY-9914-BP). [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "LLM semantic match: No message was sent to Quinn_Engineering_Manager in #proj-billing-migration confirming that passcode SEC-TELEMETRY-9914-BP was submitted to Sana. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r01': False; normalized fallback disabled ('iam_console_access_scope_matrix_r01' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r07': False; normalized fallback disabled ('iam_console_access_scope_matrix_r07' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "assets_accessed.id contains 'security_dashboard_telemetry_20260330_100000': False; normalized fallback disabled ('security_dashboard_telemetry_20260330_100000' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r08" }, { "message": "assets_accessed.id contains 'security_dashboard_telemetry_20260330_103000': False; normalized fallback disabled ('security_dashboard_telemetry_20260330_103000' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r10" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r04" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 60 }, "content_file": { "diagnostic": 0, "required": 200 }, "content_msg": { "diagnostic": 0, "required": 674 }, "grounding": { "diagnostic": 148, "required": 62 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 360 }, "window": { "diagnostic": 0, "required": 29 } }, "failed_total": 1533, "gating": { "content_involved": 422, "grounding_only": 3, "grounding_only_ids": [ "conflict_vlm_access_request_approval_v01_r10", "conflict_vlm_barrier_sync_deployment_v01_r06", "conflict_vlm_collaborative_contract_review_v02_r08" ], "n_fail": 483, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 5, "score": 0.5, "task_id": "conflict_vlm_sla_support_response_v02_r02", "verb": "asset open" } ], "loop_verbs": { "asset open": 1 }, "loopy_capped": 1, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 277, "ok_false": 0, "pass_pct": 19.5, "passed": 117, "rank": { "avg_score": 6, "capped": 5, "failed_total": 5, "loopy_capped": 2, "near_miss": 8, "pass_pct": 5, "step_mean": 4, "step_p50": 4, "zero_open_tasks": 4, "zero_score": 6, "zero_write_tasks": 3 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.5971, "n": 10, "passed": 1 }, "access_request_approval_v02": { "mean_score": 0.6892, "n": 10, "passed": 1 }, "access_request_approval_v03": { "mean_score": 0.5544, "n": 10, "passed": 2 }, "barrier_sync_deployment_v01": { "mean_score": 0.6826, "n": 10, "passed": 3 }, "barrier_sync_deployment_v02": { "mean_score": 0.6938, "n": 10, "passed": 3 }, "barrier_sync_deployment_v03": { "mean_score": 0.6367, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.3133, "n": 10, "passed": 0 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.4238, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.4271, "n": 10, "passed": 1 }, "collaborative_contract_review_v01": { "mean_score": 0.5789, "n": 10, "passed": 1 }, "collaborative_contract_review_v02": { "mean_score": 0.735, "n": 10, "passed": 5 }, "collaborative_contract_review_v03": { "mean_score": 0.7921, "n": 10, "passed": 4 }, "coordinated_containment_v01": { "mean_score": 0.3704, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.697, "n": 10, "passed": 3 }, "coordinated_containment_v03": { "mean_score": 0.7805, "n": 10, "passed": 3 }, "deadlock_multi_approval_v01": { "mean_score": 0.6342, "n": 10, "passed": 3 }, "deadlock_multi_approval_v02": { "mean_score": 0.82, "n": 10, "passed": 3 }, "deadlock_multi_approval_v03": { "mean_score": 0.715, "n": 10, "passed": 3 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.5762, "n": 10, "passed": 2 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.3006, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.3911, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v01": { "mean_score": 0.5331, "n": 10, "passed": 2 }, "employee_onboarding_workflow_v02": { "mean_score": 0.5631, "n": 10, "passed": 2 }, "employee_onboarding_workflow_v03": { "mean_score": 0.5293, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.7917, "n": 10, "passed": 4 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.68, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.6788, "n": 10, "passed": 3 }, "expiring_resource_hold_v01": { "mean_score": 0.6295, "n": 10, "passed": 2 }, "expiring_resource_hold_v02": { "mean_score": 0.4893, "n": 10, "passed": 1 }, "expiring_resource_hold_v03": { "mean_score": 0.7121, "n": 10, "passed": 0 }, "incident_commander_selection_v01": { "mean_score": 0.4833, "n": 10, "passed": 1 }, "incident_commander_selection_v02": { "mean_score": 0.6817, "n": 10, "passed": 3 }, "incident_commander_selection_v03": { "mean_score": 0.61, "n": 10, "passed": 1 }, "it_deployment_gate_v01": { "mean_score": 0.6141, "n": 10, "passed": 3 }, "it_deployment_gate_v02": { "mean_score": 0.596, "n": 10, "passed": 2 }, "it_deployment_gate_v03": { "mean_score": 0.716, "n": 10, "passed": 3 }, "multi_device_pairing_v01": { "mean_score": 0.5511, "n": 10, "passed": 1 }, "multi_device_pairing_v02": { "mean_score": 0.6347, "n": 10, "passed": 2 }, "multi_device_pairing_v03": { "mean_score": 0.5935, "n": 10, "passed": 3 }, "multi_party_access_permit_v01": { "mean_score": 0.5417, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.7762, "n": 10, "passed": 5 }, "multi_party_access_permit_v03": { "mean_score": 0.6662, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.5173, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.436, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.5046, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.3833, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.435, "n": 10, "passed": 0 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.56, "n": 10, "passed": 4 }, "payment_chargeback_dispute_v01": { "mean_score": 0.8, "n": 10, "passed": 6 }, "payment_chargeback_dispute_v02": { "mean_score": 0.4583, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v03": { "mean_score": 0.6667, "n": 10, "passed": 3 }, "satellite_ground_station_handover_v01": { "mean_score": 0.5833, "n": 10, "passed": 3 }, "satellite_ground_station_handover_v02": { "mean_score": 0.8, "n": 10, "passed": 5 }, "satellite_ground_station_handover_v03": { "mean_score": 0.6667, "n": 10, "passed": 3 }, "sla_support_response_v01": { "mean_score": 0.5552, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.6413, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.4015, "n": 10, "passed": 1 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.58, "n": 10, "passed": 3 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.65, "n": 10, "passed": 3 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.27, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 37, "fail_closed_pct": 1.7, "n": 2137 }, "side_effect_fields": { "approval_escalated": 7, "approval_granted": 3, "approval_recorded": 19, "approval_requested": 20, "assets_accessed": 53, "booking_extended": 4, "booking_held": 27, "booking_released": 10, "containment_action": 17, "deployment_approved": 8, "deployment_cutover": 1, "deployment_gate_set": 10, "deployment_requested": 30, "document_signed": 1, "lock_acquired": 15, "messages_sent": 14, "pairing_code_entered": 2, "pairing_code_shown": 1, "pairing_completed": 1, "pairing_started": 12, "review_assigned": 4, "review_submitted": 8 }, "step_mean": 9.12, "step_p50": 9, "stops": { "incomplete_required_submission": 211, "missing_required_side_effect": 267, "none": 117, "reached max_steps=20": 3, "stopped after 3 identical no-progress transitions": 2 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 1": 21, "wait 2": 33, "wait 3": 14, "wait 300": 3, "wait 5": 13 }, "max": 300.0, "mean": 13.0, "median": 2.0, "n": 84, "sub_30": 81, "total": 1094.0 }, "zero_open_tasks": 5, "zero_score": 33, "zero_write_tasks": 1 }, "opus": { "avg_score": 0.6078, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 5, "checks_total": 3760, "display": "claude-opus-4.8", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'FULLY_RECONCILED_AND_GRANTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r10" }, { "message": "files_created.content contains 'ERR-DATA-BACKFILL-409': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r03" }, { "message": "files_created.content contains 'AUTH-GATE-9942-SYNC': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v02_r03" } ], "content_file": [ { "message": "LLM semantic match: The report states the initial status was UNRECONCILED, but identifies one scope mismatch and one resource mismatch rather than two scope mismatches. [self-consistency 1/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The report finds that scope reconciliation is UNRECONCILED and denied, rather than stating that all reconciliation checks passed. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "LLM semantic match: The report's verdict explicitly denies the access grant and states not to issue the access permit. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "LLM semantic match: The report mentions transaction ID TXN-APP-WRITE-9912 instead of TXN-APP-WRITE-9908. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r04" } ], "content_msg": [ { "message": "LLM semantic match: The message to Sana_Security_Reviewer requested updated WRITE authorization citing AUDIT-SEC-2026-READ, but omitted the required passcode SEC-PASS-8842-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: The message to Ravi_Application_Owner in #iam-approvals-sec requested re-approval for WRITE scope on billing-prod, but cited reference code AUDIT-APP-2026-ARCH instead of AUDIT-SEC-2026-READ. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: The message was sent to Ravi in #iam-approvals-sec requesting a corrected sign-off for billing-prod WRITE, but stated 'billing-analytics' instead of 'billing-archive'. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" }, { "message": "LLM semantic match: A message was sent to Sana_Security_Reviewer in #iam-approvals-sec regarding REQ-2026-0842, but specified billing-analytics instead of billing-archive. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r07': False; normalized fallback disabled ('iam_console_access_scope_matrix_r07' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20241024_085500': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20241024_085500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v02_r07" }, { "message": "assets_accessed.id contains 'audit_dashboard_passcode_telemetry_20250218_103000': False; normalized fallback disabled ('audit_dashboard_passcode_telemetry_20250218_103000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r06" }, { "message": "assets_accessed.id contains 'audit_dashboard_sync_telemetry_20250218_110000': False; normalized fallback disabled ('audit_dashboard_sync_telemetry_20250218_110000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r09" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r05" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 54 }, "content_file": { "diagnostic": 0, "required": 204 }, "content_msg": { "diagnostic": 0, "required": 655 }, "grounding": { "diagnostic": 135, "required": 66 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 317 }, "window": { "diagnostic": 0, "required": 25 } }, "failed_total": 1456, "gating": { "content_involved": 408, "grounding_only": 5, "grounding_only_ids": [ "conflict_vlm_distributed_lock_shared_resource_v03_r06", "conflict_vlm_expense_reimbursement_conflict_v03_r07", "conflict_vlm_multi_device_pairing_v01_r04", "conflict_vlm_multi_device_pairing_v03_r06", "conflict_vlm_multi_tab_slot_monitoring_v01_r07" ], "n_fail": 472, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 11, "score": 0.8, "task_id": "conflict_vlm_expiring_resource_hold_v01_r10", "verb": "message search" }, { "count": 7, "score": 0.75, "task_id": "conflict_vlm_incident_commander_selection_v01_r07", "verb": "message search" }, { "count": 7, "score": 0.6666666666666666, "task_id": "conflict_vlm_payment_chargeback_dispute_v03_r01", "verb": "message search" } ], "loop_verbs": { "message search": 3 }, "loopy_capped": 3, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 275, "ok_false": 0, "pass_pct": 21.3, "passed": 128, "rank": { "avg_score": 3, "capped": 6, "failed_total": 2, "loopy_capped": 6, "near_miss": 6, "pass_pct": 2, "step_mean": 5, "step_p50": 4, "zero_open_tasks": 3, "zero_score": 2, "zero_write_tasks": 3 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.6615, "n": 10, "passed": 1 }, "access_request_approval_v02": { "mean_score": 0.7592, "n": 10, "passed": 2 }, "access_request_approval_v03": { "mean_score": 0.5278, "n": 10, "passed": 2 }, "barrier_sync_deployment_v01": { "mean_score": 0.6686, "n": 10, "passed": 3 }, "barrier_sync_deployment_v02": { "mean_score": 0.7138, "n": 10, "passed": 3 }, "barrier_sync_deployment_v03": { "mean_score": 0.5267, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.4167, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.356, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.3685, "n": 10, "passed": 0 }, "collaborative_contract_review_v01": { "mean_score": 0.5837, "n": 10, "passed": 2 }, "collaborative_contract_review_v02": { "mean_score": 0.7583, "n": 10, "passed": 5 }, "collaborative_contract_review_v03": { "mean_score": 0.8036, "n": 10, "passed": 5 }, "coordinated_containment_v01": { "mean_score": 0.3909, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.6837, "n": 10, "passed": 2 }, "coordinated_containment_v03": { "mean_score": 0.7138, "n": 10, "passed": 1 }, "deadlock_multi_approval_v01": { "mean_score": 0.6842, "n": 10, "passed": 3 }, "deadlock_multi_approval_v02": { "mean_score": 0.8192, "n": 10, "passed": 4 }, "deadlock_multi_approval_v03": { "mean_score": 0.755, "n": 10, "passed": 3 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.7031, "n": 10, "passed": 2 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.3583, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.4143, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v01": { "mean_score": 0.5539, "n": 10, "passed": 3 }, "employee_onboarding_workflow_v02": { "mean_score": 0.5556, "n": 10, "passed": 2 }, "employee_onboarding_workflow_v03": { "mean_score": 0.4543, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.7717, "n": 10, "passed": 2 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.73, "n": 10, "passed": 3 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.6317, "n": 10, "passed": 0 }, "expiring_resource_hold_v01": { "mean_score": 0.7281, "n": 10, "passed": 2 }, "expiring_resource_hold_v02": { "mean_score": 0.6107, "n": 10, "passed": 2 }, "expiring_resource_hold_v03": { "mean_score": 0.7605, "n": 10, "passed": 2 }, "incident_commander_selection_v01": { "mean_score": 0.4833, "n": 10, "passed": 1 }, "incident_commander_selection_v02": { "mean_score": 0.71, "n": 10, "passed": 3 }, "incident_commander_selection_v03": { "mean_score": 0.6317, "n": 10, "passed": 3 }, "it_deployment_gate_v01": { "mean_score": 0.6959, "n": 10, "passed": 3 }, "it_deployment_gate_v02": { "mean_score": 0.5915, "n": 10, "passed": 3 }, "it_deployment_gate_v03": { "mean_score": 0.8099, "n": 10, "passed": 4 }, "multi_device_pairing_v01": { "mean_score": 0.5522, "n": 10, "passed": 2 }, "multi_device_pairing_v02": { "mean_score": 0.6459, "n": 10, "passed": 2 }, "multi_device_pairing_v03": { "mean_score": 0.6436, "n": 10, "passed": 2 }, "multi_party_access_permit_v01": { "mean_score": 0.555, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.8286, "n": 10, "passed": 6 }, "multi_party_access_permit_v03": { "mean_score": 0.6429, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.5677, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.4894, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.5621, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.35, "n": 10, "passed": 0 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.4433, "n": 10, "passed": 0 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.5883, "n": 10, "passed": 3 }, "payment_chargeback_dispute_v01": { "mean_score": 0.8667, "n": 10, "passed": 7 }, "payment_chargeback_dispute_v02": { "mean_score": 0.4833, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v03": { "mean_score": 0.7, "n": 10, "passed": 4 }, "satellite_ground_station_handover_v01": { "mean_score": 0.5833, "n": 10, "passed": 4 }, "satellite_ground_station_handover_v02": { "mean_score": 0.7667, "n": 10, "passed": 4 }, "satellite_ground_station_handover_v03": { "mean_score": 0.65, "n": 10, "passed": 4 }, "sla_support_response_v01": { "mean_score": 0.6171, "n": 10, "passed": 2 }, "sla_support_response_v02": { "mean_score": 0.6413, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.4492, "n": 10, "passed": 2 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.6083, "n": 10, "passed": 4 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.5833, "n": 10, "passed": 3 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.2617, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 16, "fail_closed_pct": 0.7, "n": 2137 }, "side_effect_fields": { "approval_escalated": 7, "approval_granted": 4, "approval_recorded": 16, "approval_requested": 20, "assets_accessed": 55, "booking_extended": 4, "booking_held": 26, "booking_released": 11, "containment_action": 21, "deployment_approved": 8, "deployment_cutover": 1, "deployment_gate_set": 9, "deployment_requested": 28, "files_created": 3, "lock_acquired": 14, "messages_sent": 5, "pairing_code_entered": 1, "pairing_code_shown": 1, "pairing_completed": 1, "pairing_started": 8, "review_assigned": 5, "review_submitted": 8 }, "step_mean": 9.67, "step_p50": 9, "stops": { "incomplete_required_submission": 211, "missing_required_side_effect": 256, "none": 128, "reached max_steps=20": 5 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 120": 6, "wait 180": 4, "wait 30": 54, "wait 300": 4, "wait 60": 18, "wait 90": 2 }, "max": 300.0, "mean": 62.7, "median": 30.0, "n": 88, "sub_30": 0, "total": 5520.0 }, "zero_open_tasks": 4, "zero_score": 28, "zero_write_tasks": 1 }, "opus5": { "avg_score": 0.609, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 0, "checks_total": 3760, "display": "claude-opus-5", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'FULLY_RECONCILED_AND_GRANTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r10" }, { "message": "files_created.content contains 'ERR-DATA-BACKFILL-409': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r03" }, { "message": "files_created.content contains 'AUTH-GATE-9942-SYNC': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v02_r03" } ], "content_file": [ { "message": "LLM semantic match: The report concludes that the status is not reconciled and permit issuance is denied, contradicting the criterion requirement that all reconciliation checks passed. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "LLM semantic match: The report explicitly denies and blocks the access grant rather than deciding to proceed with it. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "LLM semantic match: The report concludes the opposite verdict, stating that prerequisites are not satisfied and the runbook handoff must not be initiated. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r08" }, { "message": "LLM semantic match: The report's decisive finding is 'NOT CONFIRMED COMPLETE', explicitly rejecting the 'PROVISIONING COMPLETE' banner readout. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r10" } ], "content_msg": [ { "message": "LLM semantic match: The message sent to Sana_Security_Reviewer in #iam-approvals-sec requested updated WRITE authorization and cited AUDIT-SEC-2026-READ, but omitted the required passcode SEC-PASS-8842-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: The message to Ravi requested re-approval for WRITE scope on billing-prod in #iam-approvals-sec, but cited AUDIT-APP-2026-ARCH instead of AUDIT-SEC-2026-READ. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: No message was sent to Quinn_Engineering_Manager in #proj-billing-migration confirming that passcode SEC-TELEMETRY-9914-BP was submitted to Sana and the security scope upgrade is processing. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "LLM semantic match: No message was sent to Ravi_Application_Owner in #iam-approvals-sec confirming that passcode SEC-TELEMETRY-9914-BP was verified and security scope reconciliation is underway. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r07': False; normalized fallback disabled ('iam_console_access_scope_matrix_r07' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "assets_accessed.id contains 'security_dashboard_telemetry_20260330_100000': False; normalized fallback disabled ('security_dashboard_telemetry_20260330_100000' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r08" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20241024_085500': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20241024_085500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v02_r07" }, { "message": "assets_accessed.id contains 'audit_dashboard_passcode_telemetry_20250218_103000': False; normalized fallback disabled ('audit_dashboard_passcode_telemetry_20250218_103000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r06" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r05" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 50 }, "content_file": { "diagnostic": 0, "required": 225 }, "content_msg": { "diagnostic": 0, "required": 642 }, "grounding": { "diagnostic": 144, "required": 68 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 333 }, "window": { "diagnostic": 0, "required": 24 } }, "failed_total": 1486, "gating": { "content_involved": 396, "grounding_only": 3, "grounding_only_ids": [ "conflict_vlm_distributed_lock_shared_resource_v03_r06", "conflict_vlm_expense_reimbursement_conflict_v03_r07", "conflict_vlm_multi_tab_slot_monitoring_v01_r07" ], "n_fail": 459, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [], "loop_verbs": {}, "loopy_capped": 0, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 258, "ok_false": 0, "pass_pct": 23.5, "passed": 141, "rank": { "avg_score": 2, "capped": 1, "failed_total": 3, "loopy_capped": 1, "near_miss": 3, "pass_pct": 1, "step_mean": 2, "step_p50": 2, "zero_open_tasks": 1, "zero_score": 1, "zero_write_tasks": 5 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.575, "n": 10, "passed": 1 }, "access_request_approval_v02": { "mean_score": 0.7992, "n": 10, "passed": 2 }, "access_request_approval_v03": { "mean_score": 0.5584, "n": 10, "passed": 2 }, "barrier_sync_deployment_v01": { "mean_score": 0.5765, "n": 10, "passed": 2 }, "barrier_sync_deployment_v02": { "mean_score": 0.7888, "n": 10, "passed": 5 }, "barrier_sync_deployment_v03": { "mean_score": 0.42, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.4233, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.3226, "n": 10, "passed": 0 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.4168, "n": 10, "passed": 1 }, "collaborative_contract_review_v01": { "mean_score": 0.6182, "n": 10, "passed": 4 }, "collaborative_contract_review_v02": { "mean_score": 0.735, "n": 10, "passed": 5 }, "collaborative_contract_review_v03": { "mean_score": 0.7802, "n": 10, "passed": 4 }, "coordinated_containment_v01": { "mean_score": 0.5377, "n": 10, "passed": 1 }, "coordinated_containment_v02": { "mean_score": 0.5835, "n": 10, "passed": 2 }, "coordinated_containment_v03": { "mean_score": 0.6805, "n": 10, "passed": 1 }, "deadlock_multi_approval_v01": { "mean_score": 0.7175, "n": 10, "passed": 3 }, "deadlock_multi_approval_v02": { "mean_score": 0.7433, "n": 10, "passed": 4 }, "deadlock_multi_approval_v03": { "mean_score": 0.7379, "n": 10, "passed": 3 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.7448, "n": 10, "passed": 3 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.3583, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.4238, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v01": { "mean_score": 0.5498, "n": 10, "passed": 3 }, "employee_onboarding_workflow_v02": { "mean_score": 0.5831, "n": 10, "passed": 2 }, "employee_onboarding_workflow_v03": { "mean_score": 0.4707, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.7167, "n": 10, "passed": 4 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.665, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.5795, "n": 10, "passed": 0 }, "expiring_resource_hold_v01": { "mean_score": 0.6471, "n": 10, "passed": 3 }, "expiring_resource_hold_v02": { "mean_score": 0.5845, "n": 10, "passed": 2 }, "expiring_resource_hold_v03": { "mean_score": 0.825, "n": 10, "passed": 3 }, "incident_commander_selection_v01": { "mean_score": 0.4833, "n": 10, "passed": 1 }, "incident_commander_selection_v02": { "mean_score": 0.7433, "n": 10, "passed": 5 }, "incident_commander_selection_v03": { "mean_score": 0.6517, "n": 10, "passed": 2 }, "it_deployment_gate_v01": { "mean_score": 0.5832, "n": 10, "passed": 3 }, "it_deployment_gate_v02": { "mean_score": 0.4265, "n": 10, "passed": 0 }, "it_deployment_gate_v03": { "mean_score": 0.6391, "n": 10, "passed": 3 }, "multi_device_pairing_v01": { "mean_score": 0.485, "n": 10, "passed": 0 }, "multi_device_pairing_v02": { "mean_score": 0.6659, "n": 10, "passed": 3 }, "multi_device_pairing_v03": { "mean_score": 0.6485, "n": 10, "passed": 3 }, "multi_party_access_permit_v01": { "mean_score": 0.6283, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.8595, "n": 10, "passed": 7 }, "multi_party_access_permit_v03": { "mean_score": 0.7229, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.625, "n": 10, "passed": 1 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.4075, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.5746, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.3633, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.56, "n": 10, "passed": 2 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.555, "n": 10, "passed": 3 }, "payment_chargeback_dispute_v01": { "mean_score": 0.8333, "n": 10, "passed": 7 }, "payment_chargeback_dispute_v02": { "mean_score": 0.5833, "n": 10, "passed": 2 }, "payment_chargeback_dispute_v03": { "mean_score": 0.6333, "n": 10, "passed": 3 }, "satellite_ground_station_handover_v01": { "mean_score": 0.7, "n": 10, "passed": 5 }, "satellite_ground_station_handover_v02": { "mean_score": 0.7, "n": 10, "passed": 3 }, "satellite_ground_station_handover_v03": { "mean_score": 0.7417, "n": 10, "passed": 6 }, "sla_support_response_v01": { "mean_score": 0.5743, "n": 10, "passed": 1 }, "sla_support_response_v02": { "mean_score": 0.6847, "n": 10, "passed": 1 }, "sla_support_response_v03": { "mean_score": 0.4968, "n": 10, "passed": 2 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.75, "n": 10, "passed": 5 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.7667, "n": 10, "passed": 5 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.29, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 20, "fail_closed_pct": 0.9, "n": 2137 }, "side_effect_fields": { "approval_escalated": 4, "approval_granted": 3, "approval_recorded": 17, "approval_requested": 21, "assets_accessed": 59, "booking_extended": 4, "booking_held": 22, "booking_released": 11, "containment_action": 21, "deployment_approved": 8, "deployment_gate_set": 8, "deployment_requested": 29, "document_signed": 1, "files_created": 2, "lock_acquired": 14, "messages_sent": 5, "pairing_code_entered": 1, "pairing_code_shown": 1, "pairing_completed": 1, "pairing_started": 10, "review_assigned": 5, "review_submitted": 7 }, "step_mean": 7.99, "step_p50": 8, "stops": { "generation_stall_empty_reply": 2, "incomplete_required_submission": 203, "missing_required_side_effect": 254, "none": 141 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 120": 9, "wait 180": 3, "wait 30": 9, "wait 300": 5, "wait 60": 19 }, "max": 300.0, "mean": 100.7, "median": 60.0, "n": 45, "sub_30": 0, "total": 4530.0 }, "zero_open_tasks": 1, "zero_score": 26, "zero_write_tasks": 2 }, "pro": { "avg_score": 0.4626, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 28, "checks_total": 3760, "display": "gemini-3.1-pro", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'SEC-TELEMETRY-9914-BP': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'ERR-DATA-BACKFILL-409': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r03" }, { "message": "files_created.content contains 'AUTH-GATE-9942-SYNC': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v02_r03" } ], "content_file": [ { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The report states the overall finding as PERMIT BLOCKED rather than RECONCILIATION IN PROGRESS (2/3 roles verified WRITE). [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r06" }, { "message": "LLM semantic match: The report references transaction ID TXN-APP-WRITE-9912 rather than the required decisive finding TXN-APP-WRITE-9908. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r04" } ], "content_msg": [ { "message": "LLM semantic match: A message was sent to Quinn in #proj-billing-migration noting scope mismatches, but it failed to state the overall status as UNRECONCILED (2/3 approver scopes mismatched) and did not explicitly reject the premature grant demand. [self-consi", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The required message to Sana_Security_Reviewer in #iam-approvals-sec citing permit PERMIT-2026-BP-9982 and report path /reports/access_grant_decision.md was not sent. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The message to Quinn_Engineering_Manager in #proj-billing-migration explains the scope mismatches and portal discrepancies, but omits the required reference code AUDIT-SEC-2026-READ. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: The request for updated WRITE authorization on billing-prod was sent to Sana_Security_Reviewer in #iam-approvals-sec, but omitted the required reference code AUDIT-SEC-2026-READ and passcode SEC-PASS-8842-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r07': False; normalized fallback disabled ('iam_console_access_scope_matrix_r07' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "assets_accessed.id contains 'iam_portal_approval_audit_log_20241024_083500': False; normalized fallback disabled ('iam_portal_approval_audit_log_20241024_083500' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r02" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20241024_085500': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20241024_085500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v02_r07" }, { "message": "assets_accessed.id contains 'audit_dashboard_passcode_telemetry_20250218_103000': False; normalized fallback disabled ('audit_dashboard_passcode_telemetry_20250218_103000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r06" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r04" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 86 }, "content_file": { "diagnostic": 0, "required": 394 }, "content_msg": { "diagnostic": 0, "required": 870 }, "grounding": { "diagnostic": 127, "required": 51 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 344 }, "window": { "diagnostic": 0, "required": 26 } }, "failed_total": 1898, "gating": { "content_involved": 526, "grounding_only": 1, "grounding_only_ids": [ "conflict_vlm_access_request_approval_v01_r07" ], "n_fail": 550, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 7, "score": 0.25, "task_id": "conflict_vlm_access_request_approval_v01_r01", "verb": "asset open" }, { "count": 6, "score": 0.16666666666666666, "task_id": "conflict_vlm_barrier_sync_deployment_v02_r07", "verb": "agent status" }, { "count": 5, "score": 0.4, "task_id": "conflict_vlm_clinical_trial_dosage_conflict_v03_r01", "verb": "message send" }, { "count": 5, "score": 0.0, "task_id": "conflict_vlm_clinical_trial_dosage_conflict_v03_r09", "verb": "file read" } ], "loop_verbs": { "agent status": 3, "asset open": 2, "file read": 2, "message search": 4, "message send": 2, "wait 10": 2 }, "loopy_capped": 17, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 248, "ok_false": 0, "pass_pct": 8.3, "passed": 50, "rank": { "avg_score": 10, "capped": 9, "failed_total": 10, "loopy_capped": 9, "near_miss": 2, "pass_pct": 10, "step_mean": 6, "step_p50": 4, "zero_open_tasks": 5, "zero_score": 9, "zero_write_tasks": 9 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.5836, "n": 10, "passed": 1 }, "access_request_approval_v02": { "mean_score": 0.605, "n": 10, "passed": 0 }, "access_request_approval_v03": { "mean_score": 0.6194, "n": 10, "passed": 2 }, "barrier_sync_deployment_v01": { "mean_score": 0.5911, "n": 10, "passed": 2 }, "barrier_sync_deployment_v02": { "mean_score": 0.5221, "n": 10, "passed": 2 }, "barrier_sync_deployment_v03": { "mean_score": 0.5833, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.23, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.306, "n": 10, "passed": 0 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.1535, "n": 10, "passed": 0 }, "collaborative_contract_review_v01": { "mean_score": 0.451, "n": 10, "passed": 0 }, "collaborative_contract_review_v02": { "mean_score": 0.6017, "n": 10, "passed": 2 }, "collaborative_contract_review_v03": { "mean_score": 0.7105, "n": 10, "passed": 2 }, "coordinated_containment_v01": { "mean_score": 0.2825, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.4819, "n": 10, "passed": 0 }, "coordinated_containment_v03": { "mean_score": 0.5229, "n": 10, "passed": 1 }, "deadlock_multi_approval_v01": { "mean_score": 0.5767, "n": 10, "passed": 1 }, "deadlock_multi_approval_v02": { "mean_score": 0.6925, "n": 10, "passed": 2 }, "deadlock_multi_approval_v03": { "mean_score": 0.5821, "n": 10, "passed": 1 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.4643, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.242, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.2435, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v01": { "mean_score": 0.4884, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v02": { "mean_score": 0.3757, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v03": { "mean_score": 0.3567, "n": 10, "passed": 0 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.5683, "n": 10, "passed": 0 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.56, "n": 10, "passed": 0 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.4345, "n": 10, "passed": 0 }, "expiring_resource_hold_v01": { "mean_score": 0.6145, "n": 10, "passed": 3 }, "expiring_resource_hold_v02": { "mean_score": 0.5536, "n": 10, "passed": 1 }, "expiring_resource_hold_v03": { "mean_score": 0.6271, "n": 10, "passed": 0 }, "incident_commander_selection_v01": { "mean_score": 0.25, "n": 10, "passed": 0 }, "incident_commander_selection_v02": { "mean_score": 0.3733, "n": 10, "passed": 1 }, "incident_commander_selection_v03": { "mean_score": 0.4317, "n": 10, "passed": 1 }, "it_deployment_gate_v01": { "mean_score": 0.5176, "n": 10, "passed": 0 }, "it_deployment_gate_v02": { "mean_score": 0.5989, "n": 10, "passed": 3 }, "it_deployment_gate_v03": { "mean_score": 0.6157, "n": 10, "passed": 1 }, "multi_device_pairing_v01": { "mean_score": 0.3122, "n": 10, "passed": 0 }, "multi_device_pairing_v02": { "mean_score": 0.4547, "n": 10, "passed": 0 }, "multi_device_pairing_v03": { "mean_score": 0.5246, "n": 10, "passed": 2 }, "multi_party_access_permit_v01": { "mean_score": 0.445, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.55, "n": 10, "passed": 0 }, "multi_party_access_permit_v03": { "mean_score": 0.5762, "n": 10, "passed": 1 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.3636, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.245, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.2721, "n": 10, "passed": 0 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.3167, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.27, "n": 10, "passed": 0 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.23, "n": 10, "passed": 0 }, "payment_chargeback_dispute_v01": { "mean_score": 0.8667, "n": 10, "passed": 7 }, "payment_chargeback_dispute_v02": { "mean_score": 0.325, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v03": { "mean_score": 0.4667, "n": 10, "passed": 0 }, "satellite_ground_station_handover_v01": { "mean_score": 0.6417, "n": 10, "passed": 4 }, "satellite_ground_station_handover_v02": { "mean_score": 0.6667, "n": 10, "passed": 2 }, "satellite_ground_station_handover_v03": { "mean_score": 0.5, "n": 10, "passed": 1 }, "sla_support_response_v01": { "mean_score": 0.47, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.5208, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.3706, "n": 10, "passed": 0 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.4883, "n": 10, "passed": 1 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.3583, "n": 10, "passed": 1 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.1117, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 54, "fail_closed_pct": 2.5, "n": 2137 }, "side_effect_fields": { "approval_escalated": 8, "approval_granted": 4, "approval_recorded": 19, "approval_requested": 23, "assets_accessed": 43, "booking_extended": 4, "booking_held": 25, "booking_released": 10, "containment_action": 22, "deployment_approved": 10, "deployment_cutover": 1, "deployment_gate_set": 12, "deployment_requested": 28, "document_signed": 1, "files_created": 7, "lock_acquired": 16, "lock_released": 1, "messages_sent": 9, "pairing_code_entered": 3, "pairing_code_shown": 2, "pairing_completed": 1, "pairing_started": 11, "review_assigned": 6, "review_submitted": 9 }, "step_mean": 9.84, "step_p50": 9, "stops": { "incomplete_required_submission": 246, "missing_required_side_effect": 275, "none": 50, "reached max_steps=20": 28, "stopped after 3 identical no-progress transitions": 1 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 10": 141, "wait 120": 5, "wait 180": 1, "wait 20": 4, "wait 30": 14, "wait 300": 9, "wait 5": 20, "wait 60": 6 }, "max": 1200.0, "mean": 37.9, "median": 10.0, "n": 202, "sub_30": 165, "total": 7650.0 }, "zero_open_tasks": 10, "zero_score": 67, "zero_write_tasks": 11 }, "sol": { "avg_score": 0.6102, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 1, "checks_total": 3760, "display": "gpt-5.6-sol", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'SEC-TELEMETRY-9914-BP': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'FULLY_RECONCILED_AND_GRANTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r10" }, { "message": "files_created.content contains 'ERR-DATA-BACKFILL-409': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r03" } ], "content_file": [ { "message": "LLM semantic match: The report's decisive finding is GRANTED / permit issued, rather than UNRECONCILED. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The report states the decision as 'GRANTED / permit issued' rather than 'reconcile scopes before grant'. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The report states that reconciliation is in progress, but only 1 out of 3 roles is verified rather than 2 out of 3. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r06" }, { "message": "LLM semantic match: The report references transaction TXN-APP-WRITE-9912 rather than the required TXN-APP-WRITE-9908. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r04" } ], "content_msg": [ { "message": "LLM semantic match: No message was sent in #iam-approvals-sec to Sana_Security_Reviewer requesting an updated WRITE approval on billing-prod with the passcode. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The message to Sana_Security_Reviewer in #iam-approvals-sec requested updated WRITE authorization on billing-prod and cited AUDIT-SEC-2026-READ, but omitted the required passcode SEC-PASS-8842-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: The user requested Sana's scope upgrade to WRITE on billing-prod in #iam-approvals-sec, but provided the wrong telemetry passcode instead of SEC-TELEMETRY-9914-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "LLM semantic match: No message was sent to Quinn_Engineering_Manager in #proj-billing-migration confirming that passcode SEC-TELEMETRY-9914-BP was submitted to Sana. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20241024_085500': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20241024_085500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v02_r07" }, { "message": "assets_accessed.id contains 'audit_dashboard_passcode_telemetry_20250218_103000': False; normalized fallback disabled ('audit_dashboard_passcode_telemetry_20250218_103000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r06" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20250218_103000': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20250218_103000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r07" }, { "message": "assets_accessed.id contains 'audit_dashboard_sync_telemetry_20250218_110000': False; normalized fallback disabled ('audit_dashboard_sync_telemetry_20250218_110000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r09" } ], "other": [], "side_effect": [ { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r04" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 62 }, "content_file": { "diagnostic": 0, "required": 210 }, "content_msg": { "diagnostic": 0, "required": 633 }, "grounding": { "diagnostic": 109, "required": 50 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 335 }, "window": { "diagnostic": 0, "required": 27 } }, "failed_total": 1426, "gating": { "content_involved": 414, "grounding_only": 4, "grounding_only_ids": [ "conflict_vlm_distributed_lock_shared_resource_v03_r06", "conflict_vlm_expense_reimbursement_conflict_v01_r04", "conflict_vlm_it_deployment_gate_v01_r06", "conflict_vlm_multi_tab_slot_monitoring_v02_r06" ], "n_fail": 475, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 5, "score": 0.5714285714285714, "task_id": "conflict_vlm_coordinated_containment_v03_r02", "verb": "file read" } ], "loop_verbs": { "file read": 1 }, "loopy_capped": 1, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 278, "ok_false": 0, "pass_pct": 20.8, "passed": 125, "rank": { "avg_score": 1, "capped": 2, "failed_total": 1, "loopy_capped": 2, "near_miss": 9, "pass_pct": 3, "step_mean": 8, "step_p50": 4, "zero_open_tasks": 2, "zero_score": 3, "zero_write_tasks": 1 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.7157, "n": 10, "passed": 2 }, "access_request_approval_v02": { "mean_score": 0.7267, "n": 10, "passed": 1 }, "access_request_approval_v03": { "mean_score": 0.6306, "n": 10, "passed": 2 }, "barrier_sync_deployment_v01": { "mean_score": 0.622, "n": 10, "passed": 2 }, "barrier_sync_deployment_v02": { "mean_score": 0.7921, "n": 10, "passed": 4 }, "barrier_sync_deployment_v03": { "mean_score": 0.6467, "n": 10, "passed": 2 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.26, "n": 10, "passed": 0 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.369, "n": 10, "passed": 0 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.5249, "n": 10, "passed": 1 }, "collaborative_contract_review_v01": { "mean_score": 0.6289, "n": 10, "passed": 2 }, "collaborative_contract_review_v02": { "mean_score": 0.735, "n": 10, "passed": 5 }, "collaborative_contract_review_v03": { "mean_score": 0.7681, "n": 10, "passed": 3 }, "coordinated_containment_v01": { "mean_score": 0.4427, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.7279, "n": 10, "passed": 2 }, "coordinated_containment_v03": { "mean_score": 0.7038, "n": 10, "passed": 2 }, "deadlock_multi_approval_v01": { "mean_score": 0.6633, "n": 10, "passed": 2 }, "deadlock_multi_approval_v02": { "mean_score": 0.9125, "n": 10, "passed": 6 }, "deadlock_multi_approval_v03": { "mean_score": 0.735, "n": 10, "passed": 3 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.6714, "n": 10, "passed": 4 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.3607, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.3976, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v01": { "mean_score": 0.5498, "n": 10, "passed": 2 }, "employee_onboarding_workflow_v02": { "mean_score": 0.5974, "n": 10, "passed": 2 }, "employee_onboarding_workflow_v03": { "mean_score": 0.4876, "n": 10, "passed": 0 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.7917, "n": 10, "passed": 5 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.62, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.6288, "n": 10, "passed": 1 }, "expiring_resource_hold_v01": { "mean_score": 0.6521, "n": 10, "passed": 1 }, "expiring_resource_hold_v02": { "mean_score": 0.606, "n": 10, "passed": 1 }, "expiring_resource_hold_v03": { "mean_score": 0.706, "n": 10, "passed": 1 }, "incident_commander_selection_v01": { "mean_score": 0.5083, "n": 10, "passed": 1 }, "incident_commander_selection_v02": { "mean_score": 0.6983, "n": 10, "passed": 4 }, "incident_commander_selection_v03": { "mean_score": 0.6517, "n": 10, "passed": 2 }, "it_deployment_gate_v01": { "mean_score": 0.6252, "n": 10, "passed": 2 }, "it_deployment_gate_v02": { "mean_score": 0.636, "n": 10, "passed": 3 }, "it_deployment_gate_v03": { "mean_score": 0.7064, "n": 10, "passed": 3 }, "multi_device_pairing_v01": { "mean_score": 0.4872, "n": 10, "passed": 0 }, "multi_device_pairing_v02": { "mean_score": 0.6147, "n": 10, "passed": 2 }, "multi_device_pairing_v03": { "mean_score": 0.6532, "n": 10, "passed": 3 }, "multi_party_access_permit_v01": { "mean_score": 0.525, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.7345, "n": 10, "passed": 3 }, "multi_party_access_permit_v03": { "mean_score": 0.7029, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.4995, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.4476, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.5346, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.4233, "n": 10, "passed": 2 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.5117, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.6417, "n": 10, "passed": 4 }, "payment_chargeback_dispute_v01": { "mean_score": 0.8333, "n": 10, "passed": 7 }, "payment_chargeback_dispute_v02": { "mean_score": 0.45, "n": 10, "passed": 2 }, "payment_chargeback_dispute_v03": { "mean_score": 0.7333, "n": 10, "passed": 4 }, "satellite_ground_station_handover_v01": { "mean_score": 0.6083, "n": 10, "passed": 3 }, "satellite_ground_station_handover_v02": { "mean_score": 0.8333, "n": 10, "passed": 6 }, "satellite_ground_station_handover_v03": { "mean_score": 0.75, "n": 10, "passed": 5 }, "sla_support_response_v01": { "mean_score": 0.5671, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.6274, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.4611, "n": 10, "passed": 2 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.6467, "n": 10, "passed": 3 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.6167, "n": 10, "passed": 3 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.2117, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 15, "fail_closed_pct": 0.7, "n": 2137 }, "side_effect_fields": { "approval_escalated": 6, "approval_granted": 3, "approval_recorded": 20, "approval_requested": 23, "assets_accessed": 40, "booking_extended": 4, "booking_held": 27, "booking_released": 11, "containment_action": 21, "deployment_approved": 8, "deployment_gate_set": 9, "deployment_requested": 31, "files_created": 1, "lock_acquired": 15, "messages_sent": 5, "pairing_code_entered": 4, "pairing_completed": 1, "pairing_started": 11, "review_assigned": 6, "review_submitted": 7 }, "step_mean": 10.04, "step_p50": 9, "stops": { "incomplete_required_submission": 221, "missing_required_side_effect": 253, "none": 125, "reached max_steps=20": 1 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 1": 4, "wait 10": 16, "wait 2": 25, "wait 20": 7, "wait 3": 10, "wait 30": 8, "wait 300": 4, "wait 5": 54 }, "max": 300.0, "mean": 17.9, "median": 5.0, "n": 131, "sub_30": 117, "total": 2349.0 }, "zero_open_tasks": 2, "zero_score": 30, "zero_write_tasks": 0 }, "sonnet": { "avg_score": 0.5371, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 2, "checks_total": 3760, "display": "claude-sonnet-5", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'SEC-VERIFY-4109': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r06" }, { "message": "files_created.content contains 'FULLY_RECONCILED_AND_GRANTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r10" }, { "message": "files_created.content contains 'ERR-DATA-BACKFILL-409': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r03" } ], "content_file": [ { "message": "LLM semantic match: The report's final decisive finding is GRANTED rather than UNRECONCILED. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The report states that the overall status is UNRECONCILED with only 1/3 full match, rather than stating RECONCILIATION IN PROGRESS (2/3 roles verified WRITE). [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r06" }, { "message": "LLM semantic match: The report concludes that the request is NOT RECONCILED and the permit must not be issued, directly contradicting the requirement that all reconciliation checks passed. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "LLM semantic match: The report explicitly decides that the access permit must not be issued, rather than stating to proceed with the access grant. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" } ], "content_msg": [ { "message": "LLM semantic match: The message to Sana_Security_Reviewer in #iam-approvals-sec cited AUDIT-SEC-2026-READ and requested updated WRITE authorization on billing-prod, but omitted the required passcode SEC-PASS-8842-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: No message was sent to Ravi_Application_Owner in #iam-approvals-sec. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: The message with the confirmation details was sent in #iam-approvals-sec to Ravi_Application_Owner rather than Sana_Security_Reviewer. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r04" }, { "message": "LLM semantic match: The message was sent to Sana_Security_Reviewer in #iam-approvals-sec requesting a WRITE scope upgrade on billing-prod, but used passcode SEC-PASS-8842-BP instead of the requested SEC-TELEMETRY-9914-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r01': False; normalized fallback disabled ('iam_console_access_scope_matrix_r01' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r01': False; normalized fallback disabled ('iam_console_access_scope_matrix_r01' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" }, { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r07': False; normalized fallback disabled ('iam_console_access_scope_matrix_r07' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "assets_accessed.id contains 'policy_center_access_policy_doc_20241024_084000': False; normalized fallback disabled ('policy_center_access_policy_doc_20241024_084000' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r03" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r04" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 70 }, "content_file": { "diagnostic": 0, "required": 234 }, "content_msg": { "diagnostic": 0, "required": 682 }, "grounding": { "diagnostic": 332, "required": 188 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 357 }, "window": { "diagnostic": 0, "required": 29 } }, "failed_total": 1892, "gating": { "content_involved": 423, "grounding_only": 23, "grounding_only_ids": [ "conflict_vlm_barrier_sync_deployment_v02_r04", "conflict_vlm_collaborative_contract_review_v02_r06", "conflict_vlm_deadlock_multi_approval_v01_r04", "conflict_vlm_deadlock_multi_approval_v01_r08", "conflict_vlm_deadlock_multi_approval_v01_r10", "conflict_vlm_deadlock_multi_approval_v02_r07", "conflict_vlm_deadlock_multi_approval_v03_r06", "conflict_vlm_distributed_lock_shared_resource_v01_r09", "conflict_vlm_distributed_lock_shared_resource_v01_r10", "conflict_vlm_distributed_lock_shared_resource_v02_r10", "conflict_vlm_distributed_lock_shared_resource_v03_r06", "conflict_vlm_expense_reimbursement_conflict_v01_r06" ], "n_fail": 517, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 5, "score": 0.7142857142857143, "task_id": "conflict_vlm_employee_onboarding_workflow_v02_r08", "verb": "agent status" }, { "count": 7, "score": 0.75, "task_id": "conflict_vlm_incident_commander_selection_v01_r07", "verb": "file read" } ], "loop_verbs": { "agent status": 1, "file read": 1 }, "loopy_capped": 2, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 274, "ok_false": 0, "pass_pct": 13.8, "passed": 83, "rank": { "avg_score": 8, "capped": 3, "failed_total": 9, "loopy_capped": 5, "near_miss": 5, "pass_pct": 8, "step_mean": 1, "step_p50": 1, "zero_open_tasks": 10, "zero_score": 8, "zero_write_tasks": 1 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.6048, "n": 10, "passed": 1 }, "access_request_approval_v02": { "mean_score": 0.7267, "n": 10, "passed": 1 }, "access_request_approval_v03": { "mean_score": 0.3623, "n": 10, "passed": 0 }, "barrier_sync_deployment_v01": { "mean_score": 0.6886, "n": 10, "passed": 3 }, "barrier_sync_deployment_v02": { "mean_score": 0.6688, "n": 10, "passed": 3 }, "barrier_sync_deployment_v03": { "mean_score": 0.5667, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.3633, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.381, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.423, "n": 10, "passed": 1 }, "collaborative_contract_review_v01": { "mean_score": 0.5504, "n": 10, "passed": 3 }, "collaborative_contract_review_v02": { "mean_score": 0.6883, "n": 10, "passed": 4 }, "collaborative_contract_review_v03": { "mean_score": 0.7036, "n": 10, "passed": 4 }, "coordinated_containment_v01": { "mean_score": 0.3544, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.426, "n": 10, "passed": 0 }, "coordinated_containment_v03": { "mean_score": 0.7305, "n": 10, "passed": 1 }, "deadlock_multi_approval_v01": { "mean_score": 0.6358, "n": 10, "passed": 0 }, "deadlock_multi_approval_v02": { "mean_score": 0.6283, "n": 10, "passed": 1 }, "deadlock_multi_approval_v03": { "mean_score": 0.675, "n": 10, "passed": 2 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.6331, "n": 10, "passed": 1 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.4425, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.3827, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v01": { "mean_score": 0.4762, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v02": { "mean_score": 0.5264, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v03": { "mean_score": 0.384, "n": 10, "passed": 0 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.4967, "n": 10, "passed": 0 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.49, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.5014, "n": 10, "passed": 0 }, "expiring_resource_hold_v01": { "mean_score": 0.6921, "n": 10, "passed": 2 }, "expiring_resource_hold_v02": { "mean_score": 0.4464, "n": 10, "passed": 0 }, "expiring_resource_hold_v03": { "mean_score": 0.5012, "n": 10, "passed": 0 }, "incident_commander_selection_v01": { "mean_score": 0.5333, "n": 10, "passed": 1 }, "incident_commander_selection_v02": { "mean_score": 0.6317, "n": 10, "passed": 2 }, "incident_commander_selection_v03": { "mean_score": 0.6083, "n": 10, "passed": 2 }, "it_deployment_gate_v01": { "mean_score": 0.5551, "n": 10, "passed": 0 }, "it_deployment_gate_v02": { "mean_score": 0.3979, "n": 10, "passed": 1 }, "it_deployment_gate_v03": { "mean_score": 0.5583, "n": 10, "passed": 2 }, "multi_device_pairing_v01": { "mean_score": 0.4067, "n": 10, "passed": 0 }, "multi_device_pairing_v02": { "mean_score": 0.6001, "n": 10, "passed": 1 }, "multi_device_pairing_v03": { "mean_score": 0.5404, "n": 10, "passed": 1 }, "multi_party_access_permit_v01": { "mean_score": 0.3933, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.6619, "n": 10, "passed": 2 }, "multi_party_access_permit_v03": { "mean_score": 0.5229, "n": 10, "passed": 1 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.4901, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.3667, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.5314, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.3867, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.4767, "n": 10, "passed": 2 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.6067, "n": 10, "passed": 4 }, "payment_chargeback_dispute_v01": { "mean_score": 0.875, "n": 10, "passed": 7 }, "payment_chargeback_dispute_v02": { "mean_score": 0.4667, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v03": { "mean_score": 0.5333, "n": 10, "passed": 3 }, "satellite_ground_station_handover_v01": { "mean_score": 0.6667, "n": 10, "passed": 4 }, "satellite_ground_station_handover_v02": { "mean_score": 0.8333, "n": 10, "passed": 6 }, "satellite_ground_station_handover_v03": { "mean_score": 0.6833, "n": 10, "passed": 4 }, "sla_support_response_v01": { "mean_score": 0.5338, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.5909, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.3714, "n": 10, "passed": 0 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.5333, "n": 10, "passed": 2 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.45, "n": 10, "passed": 2 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.27, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 14, "fail_closed_pct": 0.7, "n": 2137 }, "side_effect_fields": { "approval_escalated": 6, "approval_granted": 3, "approval_recorded": 13, "approval_requested": 16, "assets_accessed": 177, "booking_extended": 3, "booking_held": 17, "booking_released": 6, "containment_action": 14, "deployment_approved": 6, "deployment_gate_set": 7, "deployment_requested": 26, "files_created": 1, "lock_acquired": 8, "messages_sent": 4, "pairing_code_entered": 1, "pairing_code_shown": 1, "pairing_completed": 1, "pairing_started": 8, "review_assigned": 1, "review_submitted": 7 }, "step_mean": 7.31, "step_p50": 7, "stops": { "incomplete_required_submission": 189, "missing_required_side_effect": 326, "none": 83, "reached max_steps=20": 2 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 10": 3, "wait 120": 3, "wait 15": 2, "wait 180": 1, "wait 20": 5, "wait 30": 100, "wait 5": 13, "wait 60": 5 }, "max": 300.0, "mean": 32.9, "median": 30.0, "n": 134, "sub_30": 23, "total": 4410.0 }, "zero_open_tasks": 336, "zero_score": 39, "zero_write_tasks": 0 }, "terra": { "avg_score": 0.5716, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 2, "checks_total": 3760, "display": "gpt-5.6-terra", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'SEC-TELEMETRY-9914-BP': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'FULLY_RECONCILED_AND_GRANTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r10" }, { "message": "files_created.content contains 'ERR-DATA-BACKFILL-409': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r03" } ], "content_file": [ { "message": "LLM semantic match: The report's decisive finding is GRANTED rather than UNRECONCILED. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The report references transaction ID TXN-APP-WRITE-9912 instead of TXN-APP-WRITE-9908. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r04" }, { "message": "LLM semantic match: The report conveys that release is held pending passcode confirmation, but does not state the exact decisive finding 'PENDING RUNBOOK RELEASE (Passcode Verification Required)'. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r05" }, { "message": "LLM semantic match: The report concludes that access is incomplete (2/3 WRITE scopes active) rather than stating dataset access is provisioned with 3/3 WRITE scopes active. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r10" } ], "content_msg": [ { "message": "LLM semantic match: The message sent to Quinn in #proj-billing-migration explains that portal completion does not equal alignment and that access cannot be granted immediately, but fails to include the reference code AUDIT-SEC-2026-READ. [self-consistency 0/3 ", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: A message was sent to Ravi_Application_Owner in #iam-approvals-sec requesting re-approval for WRITE scope on billing-prod, but it cited reference code AUDIT-APP-2026-ARCH instead of AUDIT-SEC-2026-READ. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: A message was sent to Sana_Security_Reviewer in #iam-approvals-sec requesting a WRITE upgrade for billing-prod, but it contained the wrong telemetry passcode (SEC-PASS-8842-BP instead of SEC-TELEMETRY-9914-BP). [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "LLM semantic match: No message was sent to Quinn_Engineering_Manager in #proj-billing-migration confirming that passcode SEC-TELEMETRY-9914-BP was submitted to Sana. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r07': False; normalized fallback disabled ('iam_console_access_scope_matrix_r07' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "assets_accessed.id contains 'security_dashboard_telemetry_20260330_100000': False; normalized fallback disabled ('security_dashboard_telemetry_20260330_100000' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r08" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20241024_085500': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20241024_085500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v02_r07" }, { "message": "assets_accessed.id contains 'audit_dashboard_passcode_telemetry_20250218_103000': False; normalized fallback disabled ('audit_dashboard_passcode_telemetry_20250218_103000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r06" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r04" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 61 }, "content_file": { "diagnostic": 0, "required": 238 }, "content_msg": { "diagnostic": 0, "required": 688 }, "grounding": { "diagnostic": 168, "required": 84 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 349 }, "window": { "diagnostic": 0, "required": 27 } }, "failed_total": 1615, "gating": { "content_involved": 439, "grounding_only": 6, "grounding_only_ids": [ "conflict_vlm_distributed_lock_shared_resource_v03_r06", "conflict_vlm_expense_reimbursement_conflict_v01_r04", "conflict_vlm_expense_reimbursement_conflict_v02_r09", "conflict_vlm_expense_reimbursement_conflict_v03_r07", "conflict_vlm_it_deployment_gate_v03_r10", "conflict_vlm_multi_device_pairing_v02_r05" ], "n_fail": 494, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 5, "score": 0.5714285714285714, "task_id": "conflict_vlm_coordinated_containment_v03_r02", "verb": "asset open" } ], "loop_verbs": { "asset open": 1 }, "loopy_capped": 1, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 276, "ok_false": 0, "pass_pct": 17.7, "passed": 106, "rank": { "avg_score": 7, "capped": 3, "failed_total": 7, "loopy_capped": 2, "near_miss": 7, "pass_pct": 7, "step_mean": 3, "step_p50": 2, "zero_open_tasks": 6, "zero_score": 7, "zero_write_tasks": 5 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.7039, "n": 10, "passed": 2 }, "access_request_approval_v02": { "mean_score": 0.6125, "n": 10, "passed": 0 }, "access_request_approval_v03": { "mean_score": 0.6014, "n": 10, "passed": 2 }, "barrier_sync_deployment_v01": { "mean_score": 0.6353, "n": 10, "passed": 3 }, "barrier_sync_deployment_v02": { "mean_score": 0.7038, "n": 10, "passed": 3 }, "barrier_sync_deployment_v03": { "mean_score": 0.67, "n": 10, "passed": 3 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.2867, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.4321, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.4004, "n": 10, "passed": 0 }, "collaborative_contract_review_v01": { "mean_score": 0.6146, "n": 10, "passed": 2 }, "collaborative_contract_review_v02": { "mean_score": 0.755, "n": 10, "passed": 6 }, "collaborative_contract_review_v03": { "mean_score": 0.7573, "n": 10, "passed": 2 }, "coordinated_containment_v01": { "mean_score": 0.3937, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.6576, "n": 10, "passed": 4 }, "coordinated_containment_v03": { "mean_score": 0.6538, "n": 10, "passed": 0 }, "deadlock_multi_approval_v01": { "mean_score": 0.6842, "n": 10, "passed": 3 }, "deadlock_multi_approval_v02": { "mean_score": 0.745, "n": 10, "passed": 3 }, "deadlock_multi_approval_v03": { "mean_score": 0.735, "n": 10, "passed": 3 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.5681, "n": 10, "passed": 2 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.2845, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.3619, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v01": { "mean_score": 0.5328, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v02": { "mean_score": 0.5306, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v03": { "mean_score": 0.4317, "n": 10, "passed": 0 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.7917, "n": 10, "passed": 4 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.59, "n": 10, "passed": 0 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.5471, "n": 10, "passed": 0 }, "expiring_resource_hold_v01": { "mean_score": 0.5843, "n": 10, "passed": 2 }, "expiring_resource_hold_v02": { "mean_score": 0.406, "n": 10, "passed": 0 }, "expiring_resource_hold_v03": { "mean_score": 0.7455, "n": 10, "passed": 1 }, "incident_commander_selection_v01": { "mean_score": 0.4583, "n": 10, "passed": 1 }, "incident_commander_selection_v02": { "mean_score": 0.7233, "n": 10, "passed": 5 }, "incident_commander_selection_v03": { "mean_score": 0.6767, "n": 10, "passed": 3 }, "it_deployment_gate_v01": { "mean_score": 0.5959, "n": 10, "passed": 3 }, "it_deployment_gate_v02": { "mean_score": 0.5607, "n": 10, "passed": 1 }, "it_deployment_gate_v03": { "mean_score": 0.6934, "n": 10, "passed": 1 }, "multi_device_pairing_v01": { "mean_score": 0.4067, "n": 10, "passed": 0 }, "multi_device_pairing_v02": { "mean_score": 0.6447, "n": 10, "passed": 1 }, "multi_device_pairing_v03": { "mean_score": 0.5629, "n": 10, "passed": 2 }, "multi_party_access_permit_v01": { "mean_score": 0.525, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.6845, "n": 10, "passed": 3 }, "multi_party_access_permit_v03": { "mean_score": 0.6495, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.438, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.3589, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.5221, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.38, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.4867, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.6083, "n": 10, "passed": 3 }, "payment_chargeback_dispute_v01": { "mean_score": 0.8333, "n": 10, "passed": 6 }, "payment_chargeback_dispute_v02": { "mean_score": 0.425, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v03": { "mean_score": 0.6667, "n": 10, "passed": 3 }, "satellite_ground_station_handover_v01": { "mean_score": 0.5833, "n": 10, "passed": 4 }, "satellite_ground_station_handover_v02": { "mean_score": 0.8, "n": 10, "passed": 5 }, "satellite_ground_station_handover_v03": { "mean_score": 0.6917, "n": 10, "passed": 3 }, "sla_support_response_v01": { "mean_score": 0.5529, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.5752, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.4611, "n": 10, "passed": 2 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.5883, "n": 10, "passed": 3 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.5167, "n": 10, "passed": 2 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.2117, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 29, "fail_closed_pct": 1.4, "n": 2137 }, "side_effect_fields": { "approval_escalated": 7, "approval_granted": 3, "approval_recorded": 19, "approval_requested": 20, "assets_accessed": 68, "booking_extended": 4, "booking_held": 27, "booking_released": 9, "containment_action": 19, "deployment_approved": 8, "deployment_cutover": 1, "deployment_gate_set": 10, "deployment_requested": 31, "files_created": 1, "lock_acquired": 14, "messages_sent": 7, "pairing_code_entered": 1, "pairing_code_shown": 1, "pairing_completed": 1, "pairing_started": 11, "review_assigned": 5, "review_submitted": 9 }, "step_mean": 8.65, "step_p50": 8, "stops": { "generation_stall_empty_reply": 3, "incomplete_required_submission": 211, "missing_required_side_effect": 276, "none": 106, "reached max_steps=20": 2, "stopped after 3 identical no-progress transitions": 2 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 1": 5, "wait 10": 13, "wait 2": 16, "wait 3": 25, "wait 300": 4, "wait 4": 2, "wait 5": 83, "wait 8": 2 }, "max": 300.0, "mean": 13.7, "median": 5.0, "n": 152, "sub_30": 147, "total": 2081.0 }, "zero_open_tasks": 20, "zero_score": 38, "zero_write_tasks": 2 } } }, "medium": { "head_to_head": { "flash": { "flash36": { "loss": 59, "tie": 0, "win": 1 }, "flash37": { "loss": 60, "tie": 0, "win": 0 }, "luna": { "loss": 58, "tie": 0, "win": 2 }, "opus": { "loss": 58, "tie": 0, "win": 2 }, "opus5": { "loss": 58, "tie": 0, "win": 2 }, "pro": { "loss": 53, "tie": 0, "win": 7 }, "sol": { "loss": 59, "tie": 0, "win": 1 }, "sonnet": { "loss": 57, "tie": 0, "win": 3 }, "terra": { "loss": 59, "tie": 0, "win": 1 } }, "flash36": { "flash": { "loss": 1, "tie": 0, "win": 59 }, "flash37": { "loss": 36, "tie": 5, "win": 19 }, "luna": { "loss": 26, "tie": 3, "win": 31 }, "opus": { "loss": 37, "tie": 1, "win": 22 }, "opus5": { "loss": 40, "tie": 1, "win": 19 }, "pro": { "loss": 1, "tie": 0, "win": 59 }, "sol": { "loss": 35, "tie": 2, "win": 23 }, "sonnet": { "loss": 21, "tie": 2, "win": 37 }, "terra": { "loss": 23, "tie": 3, "win": 34 } }, "flash37": { "flash": { "loss": 0, "tie": 0, "win": 60 }, "flash36": { "loss": 19, "tie": 5, "win": 36 }, "luna": { "loss": 23, "tie": 1, "win": 36 }, "opus": { "loss": 29, "tie": 3, "win": 28 }, "opus5": { "loss": 35, "tie": 0, "win": 25 }, "pro": { "loss": 1, "tie": 0, "win": 59 }, "sol": { "loss": 35, "tie": 1, "win": 24 }, "sonnet": { "loss": 19, "tie": 1, "win": 40 }, "terra": { "loss": 22, "tie": 2, "win": 36 } }, "luna": { "flash": { "loss": 2, "tie": 0, "win": 58 }, "flash36": { "loss": 31, "tie": 3, "win": 26 }, "flash37": { "loss": 36, "tie": 1, "win": 23 }, "opus": { "loss": 43, "tie": 1, "win": 16 }, "opus5": { "loss": 40, "tie": 3, "win": 17 }, "pro": { "loss": 4, "tie": 0, "win": 56 }, "sol": { "loss": 40, "tie": 0, "win": 20 }, "sonnet": { "loss": 24, "tie": 0, "win": 36 }, "terra": { "loss": 23, "tie": 4, "win": 33 } }, "opus": { "flash": { "loss": 2, "tie": 0, "win": 58 }, "flash36": { "loss": 22, "tie": 1, "win": 37 }, "flash37": { "loss": 28, "tie": 3, "win": 29 }, "luna": { "loss": 16, "tie": 1, "win": 43 }, "opus5": { "loss": 35, "tie": 2, "win": 23 }, "pro": { "loss": 3, "tie": 0, "win": 57 }, "sol": { "loss": 28, "tie": 1, "win": 31 }, "sonnet": { "loss": 14, "tie": 2, "win": 44 }, "terra": { "loss": 17, "tie": 0, "win": 43 } }, "opus5": { "flash": { "loss": 2, "tie": 0, "win": 58 }, "flash36": { "loss": 19, "tie": 1, "win": 40 }, "flash37": { "loss": 25, "tie": 0, "win": 35 }, "luna": { "loss": 17, "tie": 3, "win": 40 }, "opus": { "loss": 23, "tie": 2, "win": 35 }, "pro": { "loss": 5, "tie": 0, "win": 55 }, "sol": { "loss": 30, "tie": 0, "win": 30 }, "sonnet": { "loss": 15, "tie": 1, "win": 44 }, "terra": { "loss": 21, "tie": 1, "win": 38 } }, "pro": { "flash": { "loss": 7, "tie": 0, "win": 53 }, "flash36": { "loss": 59, "tie": 0, "win": 1 }, "flash37": { "loss": 59, "tie": 0, "win": 1 }, "luna": { "loss": 56, "tie": 0, "win": 4 }, "opus": { "loss": 57, "tie": 0, "win": 3 }, "opus5": { "loss": 55, "tie": 0, "win": 5 }, "sol": { "loss": 58, "tie": 1, "win": 1 }, "sonnet": { "loss": 50, "tie": 0, "win": 10 }, "terra": { "loss": 54, "tie": 0, "win": 6 } }, "sol": { "flash": { "loss": 1, "tie": 0, "win": 59 }, "flash36": { "loss": 23, "tie": 2, "win": 35 }, "flash37": { "loss": 24, "tie": 1, "win": 35 }, "luna": { "loss": 20, "tie": 0, "win": 40 }, "opus": { "loss": 31, "tie": 1, "win": 28 }, "opus5": { "loss": 30, "tie": 0, "win": 30 }, "pro": { "loss": 1, "tie": 1, "win": 58 }, "sonnet": { "loss": 15, "tie": 0, "win": 45 }, "terra": { "loss": 14, "tie": 5, "win": 41 } }, "sonnet": { "flash": { "loss": 3, "tie": 0, "win": 57 }, "flash36": { "loss": 37, "tie": 2, "win": 21 }, "flash37": { "loss": 40, "tie": 1, "win": 19 }, "luna": { "loss": 36, "tie": 0, "win": 24 }, "opus": { "loss": 44, "tie": 2, "win": 14 }, "opus5": { "loss": 44, "tie": 1, "win": 15 }, "pro": { "loss": 10, "tie": 0, "win": 50 }, "sol": { "loss": 45, "tie": 0, "win": 15 }, "terra": { "loss": 32, "tie": 1, "win": 27 } }, "terra": { "flash": { "loss": 1, "tie": 0, "win": 59 }, "flash36": { "loss": 34, "tie": 3, "win": 23 }, "flash37": { "loss": 36, "tie": 2, "win": 22 }, "luna": { "loss": 33, "tie": 4, "win": 23 }, "opus": { "loss": 43, "tie": 0, "win": 17 }, "opus5": { "loss": 38, "tie": 1, "win": 21 }, "pro": { "loss": 6, "tie": 0, "win": 54 }, "sol": { "loss": 41, "tie": 5, "win": 14 }, "sonnet": { "loss": 27, "tie": 1, "win": 32 } } }, "models": { "flash": { "avg_score": 0.299, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 941 }, "content_msg": { "effect:messages_sent field_semantic_match": 1191 }, "grounding": { "effect:assets_accessed field_contains": 821 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 510, "checks_total": 3754, "display": "gemini-3.5-flash", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'AUDIT-SEC-2026-READ': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "files_created.content contains 'SEC-TELEMETRY-9914-BP': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "files_created.content contains 'PERMIT-2026-BP-9982': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r08" }, { "message": "files_created.content contains 'POL-SEC-RECON-3.4': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r03" } ], "content_file": [ { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "content_msg": [ { "message": "LLM semantic match: no 'message' content in messages_sent to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'message' content in messages_sent to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'message' content in messages_sent to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'message' content in messages_sent to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20241024_085500': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20241024_085500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v02_r07" }, { "message": "assets_accessed.id contains 'iam_portal_provisioning_status_matrix_20241024_093000': False; normalized fallback disabled ('iam_portal_provisioning_status_matrix_20241024_093000' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r10" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20250218_103000': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20250218_103000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r07" }, { "message": "assets_accessed.id contains 'ops_console_gate_board_20260330_141000': False; normalized fallback disabled ('ops_console_gate_board_20260330_141000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r06" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r04" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 193 }, "content_file": { "diagnostic": 0, "required": 714 }, "content_msg": { "diagnostic": 0, "required": 894 }, "grounding": { "diagnostic": 93, "required": 46 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 353 }, "window": { "diagnostic": 0, "required": 28 } }, "failed_total": 2321, "gating": { "content_involved": 560, "grounding_only": 0, "grounding_only_ids": [], "n_fail": 581, "no_required_check_failed": 1, "no_required_check_failed_ids": [ "conflict_vlm_expense_reimbursement_conflict_v03_r08" ] }, "loop_examples": [ { "count": 7, "score": 0.125, "task_id": "conflict_vlm_access_request_approval_v01_r01", "verb": "message search" }, { "count": 9, "score": 0.14285714285714285, "task_id": "conflict_vlm_access_request_approval_v01_r02", "verb": "asset open" }, { "count": 9, "score": 0.14285714285714285, "task_id": "conflict_vlm_access_request_approval_v01_r03", "verb": "message search" }, { "count": 8, "score": 0.4, "task_id": "conflict_vlm_access_request_approval_v01_r06", "verb": "file read" } ], "loop_verbs": { "asset info": 5, "asset open": 87, "booking status": 4, "file read": 212, "message search": 164, "message send": 3 }, "loopy_capped": 483, "missing_check_result_ids": [ "conflict_vlm_expense_reimbursement_conflict_v03_r08" ], "missing_check_results": 1, "n": 600, "near_miss": 128, "ok_false": 0, "pass_pct": 3.2, "passed": 19, "rank": { "avg_score": 10, "capped": 10, "failed_total": 10, "loopy_capped": 10, "near_miss": 1, "pass_pct": 10, "step_mean": 10, "step_p50": 10, "zero_open_tasks": 8, "zero_score": 10, "zero_write_tasks": 10 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.3654, "n": 10, "passed": 2 }, "access_request_approval_v02": { "mean_score": 0.32, "n": 10, "passed": 0 }, "access_request_approval_v03": { "mean_score": 0.556, "n": 10, "passed": 2 }, "barrier_sync_deployment_v01": { "mean_score": 0.4012, "n": 10, "passed": 1 }, "barrier_sync_deployment_v02": { "mean_score": 0.3936, "n": 10, "passed": 1 }, "barrier_sync_deployment_v03": { "mean_score": 0.375, "n": 10, "passed": 0 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.1733, "n": 10, "passed": 0 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.0833, "n": 10, "passed": 0 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.2004, "n": 10, "passed": 0 }, "collaborative_contract_review_v01": { "mean_score": 0.3515, "n": 10, "passed": 0 }, "collaborative_contract_review_v02": { "mean_score": 0.505, "n": 10, "passed": 2 }, "collaborative_contract_review_v03": { "mean_score": 0.3576, "n": 10, "passed": 1 }, "coordinated_containment_v01": { "mean_score": 0.1976, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.1471, "n": 10, "passed": 0 }, "coordinated_containment_v03": { "mean_score": 0.3667, "n": 10, "passed": 0 }, "deadlock_multi_approval_v01": { "mean_score": 0.3975, "n": 10, "passed": 0 }, "deadlock_multi_approval_v02": { "mean_score": 0.3506, "n": 10, "passed": 0 }, "deadlock_multi_approval_v03": { "mean_score": 0.4429, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.3121, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.1688, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.2893, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v01": { "mean_score": 0.3076, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v02": { "mean_score": 0.3382, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v03": { "mean_score": 0.4374, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.27, "n": 10, "passed": 0 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.295, "n": 10, "passed": 0 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.2779, "n": 10, "passed": 0 }, "expiring_resource_hold_v01": { "mean_score": 0.3026, "n": 10, "passed": 0 }, "expiring_resource_hold_v02": { "mean_score": 0.3315, "n": 10, "passed": 0 }, "expiring_resource_hold_v03": { "mean_score": 0.276, "n": 10, "passed": 0 }, "incident_commander_selection_v01": { "mean_score": 0.1083, "n": 10, "passed": 0 }, "incident_commander_selection_v02": { "mean_score": 0.515, "n": 10, "passed": 1 }, "incident_commander_selection_v03": { "mean_score": 0.18, "n": 10, "passed": 0 }, "it_deployment_gate_v01": { "mean_score": 0.4245, "n": 10, "passed": 1 }, "it_deployment_gate_v02": { "mean_score": 0.6424, "n": 10, "passed": 4 }, "it_deployment_gate_v03": { "mean_score": 0.3326, "n": 10, "passed": 0 }, "multi_device_pairing_v01": { "mean_score": 0.3867, "n": 10, "passed": 0 }, "multi_device_pairing_v02": { "mean_score": 0.2988, "n": 10, "passed": 0 }, "multi_device_pairing_v03": { "mean_score": 0.4487, "n": 10, "passed": 0 }, "multi_party_access_permit_v01": { "mean_score": 0.355, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.5012, "n": 10, "passed": 1 }, "multi_party_access_permit_v03": { "mean_score": 0.261, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.1985, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.3476, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.23, "n": 10, "passed": 0 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.2733, "n": 10, "passed": 0 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.145, "n": 10, "passed": 0 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.1817, "n": 10, "passed": 0 }, "payment_chargeback_dispute_v01": { "mean_score": 0.125, "n": 10, "passed": 0 }, "payment_chargeback_dispute_v02": { "mean_score": 0.2083, "n": 10, "passed": 0 }, "payment_chargeback_dispute_v03": { "mean_score": 0.0667, "n": 10, "passed": 0 }, "satellite_ground_station_handover_v01": { "mean_score": 0.05, "n": 10, "passed": 0 }, "satellite_ground_station_handover_v02": { "mean_score": 0.2333, "n": 10, "passed": 1 }, "satellite_ground_station_handover_v03": { "mean_score": 0.1, "n": 10, "passed": 0 }, "sla_support_response_v01": { "mean_score": 0.4424, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.4754, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.393, "n": 10, "passed": 0 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.17, "n": 10, "passed": 0 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.0667, "n": 10, "passed": 0 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.19, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 1070, "fail_closed_pct": 50.2, "n": 2132 }, "side_effect_fields": { "approval_granted": 1, "approval_recorded": 2, "approval_requested": 3, "assets_accessed": 8, "booking_extended": 1, "booking_held": 3, "booking_released": 2, "containment_action": 1, "deployment_approved": 1, "deployment_gate_set": 2, "deployment_requested": 5, "lock_acquired": 1, "messages_sent": 3, "pairing_code_entered": 1, "review_assigned": 1 }, "step_mean": 19.43, "step_p50": 20, "stops": { "generation_stall_empty_reply": 1, "incomplete_required_submission": 33, "missing_required_side_effect": 35, "none": 20, "reached max_steps=20": 510, "stopped after 3 identical no-progress transitions": 1 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 941 }, "content_msg": { "diagnostic": 0, "required": 1191 }, "grounding": { "diagnostic": 476, "required": 345 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 10": 145, "wait 15": 6, "wait 20": 1, "wait 30": 28, "wait 300": 3, "wait 5": 101, "wait 60": 4 }, "max": 300.0, "mean": 14.0, "median": 10.0, "n": 288, "sub_30": 253, "total": 4045.0 }, "zero_open_tasks": 30, "zero_score": 145, "zero_write_tasks": 390 }, "flash36": { "avg_score": 0.6023, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 87, "checks_total": 3760, "display": "gemini-3.6-flash", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'SEC-TELEMETRY-9914-BP': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'FULLY_RECONCILED_AND_GRANTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r10" }, { "message": "files_created.content contains 'ERR-DATA-BACKFILL-409': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r03" } ], "content_file": [ { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "LLM semantic match: The report references transaction ID TXN-APP-WRITE-9912 rather than TXN-APP-WRITE-9908. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r04" } ], "content_msg": [ { "message": "LLM semantic match: No message was sent to Sana_Security_Reviewer citing permit PERMIT-2026-BP-9982 or report path /reports/access_grant_decision.md to authorize releasing the migration runbook. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: A message was sent to Sana_Security_Reviewer citing reference code AUDIT-SEC-2026-READ for updated WRITE authorization, but it omitted the required passcode SEC-PASS-8842-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: A message was sent to Ravi_Application_Owner in #iam-approvals-sec requesting re-approval for WRITE scope on billing-prod, but it cited reference code AUDIT-APP-2026-ARCH instead of AUDIT-SEC-2026-READ. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: A message was sent to Sana_Security_Reviewer in #iam-approvals-sec requesting a scope upgrade, but it contained the incorrect passcode SEC-PASS-8842-BP instead of SEC-TELEMETRY-9914-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20250218_103000': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20250218_103000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r07" }, { "message": "assets_accessed.id contains 'audit_dashboard_reconciliation_verdict_20250218_110500': False; normalized fallback disabled ('audit_dashboard_reconciliation_verdict_20250218_110500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r10" }, { "message": "assets_accessed.id contains 'ops_console_gate_controller_metrics_20260330_143000': False; normalized fallback disabled ('ops_console_gate_controller_metrics_20260330_143000' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r04" }, { "message": "assets_accessed.id contains 'ops_console_gate_board_20260330_141000': False; normalized fallback disabled ('ops_console_gate_board_20260330_141000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r06" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r04" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 64 }, "content_file": { "diagnostic": 0, "required": 239 }, "content_msg": { "diagnostic": 0, "required": 656 }, "grounding": { "diagnostic": 83, "required": 35 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 316 }, "window": { "diagnostic": 0, "required": 27 } }, "failed_total": 1420, "gating": { "content_involved": 408, "grounding_only": 3, "grounding_only_ids": [ "conflict_vlm_expense_reimbursement_conflict_v01_r04", "conflict_vlm_multi_device_pairing_v01_r04", "conflict_vlm_multi_party_access_permit_v01_r07" ], "n_fail": 479, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 7, "score": 0.375, "task_id": "conflict_vlm_access_request_approval_v01_r01", "verb": "asset open" }, { "count": 6, "score": 0.14285714285714285, "task_id": "conflict_vlm_access_request_approval_v01_r05", "verb": "file read" }, { "count": 9, "score": 0.8, "task_id": "conflict_vlm_access_request_approval_v01_r09", "verb": "file read" }, { "count": 7, "score": 0.4, "task_id": "conflict_vlm_barrier_sync_deployment_v03_r02", "verb": "asset open" } ], "loop_verbs": { "asset open": 18, "booking status": 1, "deploy status": 1, "file read": 39, "message search": 4, "message send": 4 }, "loopy_capped": 67, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 273, "ok_false": 0, "pass_pct": 20.2, "passed": 121, "rank": { "avg_score": 5, "capped": 9, "failed_total": 4, "loopy_capped": 9, "near_miss": 7, "pass_pct": 5, "step_mean": 8, "step_p50": 8, "zero_open_tasks": 5, "zero_score": 5, "zero_write_tasks": 9 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.6946, "n": 10, "passed": 2 }, "access_request_approval_v02": { "mean_score": 0.7067, "n": 10, "passed": 1 }, "access_request_approval_v03": { "mean_score": 0.6608, "n": 10, "passed": 1 }, "barrier_sync_deployment_v01": { "mean_score": 0.6732, "n": 10, "passed": 3 }, "barrier_sync_deployment_v02": { "mean_score": 0.6788, "n": 10, "passed": 2 }, "barrier_sync_deployment_v03": { "mean_score": 0.7067, "n": 10, "passed": 3 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.38, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.569, "n": 10, "passed": 4 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.3408, "n": 10, "passed": 0 }, "collaborative_contract_review_v01": { "mean_score": 0.7039, "n": 10, "passed": 2 }, "collaborative_contract_review_v02": { "mean_score": 0.8083, "n": 10, "passed": 5 }, "collaborative_contract_review_v03": { "mean_score": 0.7326, "n": 10, "passed": 3 }, "coordinated_containment_v01": { "mean_score": 0.4159, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.5894, "n": 10, "passed": 1 }, "coordinated_containment_v03": { "mean_score": 0.7186, "n": 10, "passed": 2 }, "deadlock_multi_approval_v01": { "mean_score": 0.6925, "n": 10, "passed": 3 }, "deadlock_multi_approval_v02": { "mean_score": 0.8392, "n": 10, "passed": 4 }, "deadlock_multi_approval_v03": { "mean_score": 0.76, "n": 10, "passed": 4 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.7064, "n": 10, "passed": 3 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.414, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.3744, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v01": { "mean_score": 0.5544, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v02": { "mean_score": 0.4851, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v03": { "mean_score": 0.5711, "n": 10, "passed": 2 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.7117, "n": 10, "passed": 5 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.705, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.6824, "n": 10, "passed": 1 }, "expiring_resource_hold_v01": { "mean_score": 0.6445, "n": 10, "passed": 2 }, "expiring_resource_hold_v02": { "mean_score": 0.6679, "n": 10, "passed": 2 }, "expiring_resource_hold_v03": { "mean_score": 0.6998, "n": 10, "passed": 0 }, "incident_commander_selection_v01": { "mean_score": 0.4833, "n": 10, "passed": 1 }, "incident_commander_selection_v02": { "mean_score": 0.5033, "n": 10, "passed": 1 }, "incident_commander_selection_v03": { "mean_score": 0.5983, "n": 10, "passed": 1 }, "it_deployment_gate_v01": { "mean_score": 0.6559, "n": 10, "passed": 3 }, "it_deployment_gate_v02": { "mean_score": 0.6449, "n": 10, "passed": 3 }, "it_deployment_gate_v03": { "mean_score": 0.6732, "n": 10, "passed": 2 }, "multi_device_pairing_v01": { "mean_score": 0.6244, "n": 10, "passed": 0 }, "multi_device_pairing_v02": { "mean_score": 0.677, "n": 10, "passed": 2 }, "multi_device_pairing_v03": { "mean_score": 0.6576, "n": 10, "passed": 3 }, "multi_party_access_permit_v01": { "mean_score": 0.525, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.7143, "n": 10, "passed": 4 }, "multi_party_access_permit_v03": { "mean_score": 0.6695, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.4279, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.4185, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.5011, "n": 10, "passed": 0 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.4, "n": 10, "passed": 2 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.4567, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.505, "n": 10, "passed": 2 }, "payment_chargeback_dispute_v01": { "mean_score": 0.9, "n": 10, "passed": 8 }, "payment_chargeback_dispute_v02": { "mean_score": 0.4583, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v03": { "mean_score": 0.5, "n": 10, "passed": 2 }, "satellite_ground_station_handover_v01": { "mean_score": 0.6083, "n": 10, "passed": 4 }, "satellite_ground_station_handover_v02": { "mean_score": 0.9, "n": 10, "passed": 7 }, "satellite_ground_station_handover_v03": { "mean_score": 0.6667, "n": 10, "passed": 3 }, "sla_support_response_v01": { "mean_score": 0.5805, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.6246, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.4525, "n": 10, "passed": 1 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.575, "n": 10, "passed": 4 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.65, "n": 10, "passed": 5 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.1983, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 98, "fail_closed_pct": 4.6, "n": 2137 }, "side_effect_fields": { "approval_escalated": 6, "approval_granted": 3, "approval_recorded": 16, "approval_requested": 20, "assets_accessed": 26, "booking_extended": 4, "booking_held": 19, "booking_released": 7, "containment_action": 16, "deployment_approved": 7, "deployment_gate_set": 8, "deployment_requested": 28, "files_created": 1, "lock_acquired": 15, "lock_released": 2, "messages_sent": 7, "pairing_code_entered": 3, "pairing_code_shown": 2, "pairing_started": 8, "review_assigned": 2, "review_submitted": 7 }, "step_mean": 14.77, "step_p50": 15, "stops": { "incomplete_required_submission": 185, "missing_required_side_effect": 207, "none": 121, "reached max_steps=20": 87 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 10": 16, "wait 15": 2, "wait 30": 2, "wait 300": 4, "wait 5": 23 }, "max": 300.0, "mean": 33.3, "median": 10.0, "n": 47, "sub_30": 41, "total": 1565.0 }, "zero_open_tasks": 4, "zero_score": 29, "zero_write_tasks": 36 }, "flash37": { "avg_score": 0.6176, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 76, "checks_total": 3760, "display": "gemini-3.7-flash", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'SEC-TELEMETRY-9914-BP': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'ERR-DATA-BACKFILL-409': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r03" }, { "message": "files_created.content contains 'AUTH-GATE-9942-SYNC': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v02_r03" } ], "content_file": [ { "message": "LLM semantic match: The report concludes that the decision is 'GRANTED & RECONCILED' rather than stating the decisive finding as 'UNRECONCILED (2/3 approver scopes mismatched)'. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "LLM semantic match: The report specifies transaction ID TXN-APP-WRITE-9912 instead of TXN-APP-WRITE-9908. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r04" }, { "message": "LLM semantic match: The report captures that release is pending runbook passcode verification, but the formal verdict is stated as 'SECURITY SCOPE RECONCILED / APPROVED (WRITE)' rather than 'PENDING RUNBOOK RELEASE (Passcode Verification Required)'. [self-cons", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r05" } ], "content_msg": [ { "message": "LLM semantic match: No message was sent to Sana_Security_Reviewer in #iam-approvals-sec citing the permit PERMIT-2026-BP-9982 or report path /reports/access_grant_decision.md. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The message was sent to Sana_Security_Reviewer in #iam-approvals-sec citing AUDIT-SEC-2026-READ and requesting WRITE authorization on billing-prod, but omitted the required passcode SEC-PASS-8842-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: A message was sent to Ravi in #iam-approvals-sec requesting re-approval for WRITE scope on billing-prod, but it cited AUDIT-APP-2026-ARCH instead of AUDIT-SEC-2026-READ. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: No message was sent to Quinn_Engineering_Manager in #proj-billing-migration. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r01': False; normalized fallback disabled ('iam_console_access_scope_matrix_r01' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r07': False; normalized fallback disabled ('iam_console_access_scope_matrix_r07' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20241024_085500': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20241024_085500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v02_r07" }, { "message": "assets_accessed.id contains 'audit_dashboard_passcode_telemetry_20250218_103000': False; normalized fallback disabled ('audit_dashboard_passcode_telemetry_20250218_103000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r06" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r05" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 57 }, "content_file": { "diagnostic": 0, "required": 217 }, "content_msg": { "diagnostic": 0, "required": 608 }, "grounding": { "diagnostic": 141, "required": 58 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 319 }, "window": { "diagnostic": 0, "required": 28 } }, "failed_total": 1428, "gating": { "content_involved": 397, "grounding_only": 5, "grounding_only_ids": [ "conflict_vlm_distributed_lock_shared_resource_v03_r06", "conflict_vlm_expense_reimbursement_conflict_v01_r04", "conflict_vlm_expense_reimbursement_conflict_v03_r06", "conflict_vlm_multi_device_pairing_v01_r04", "conflict_vlm_sla_support_response_v03_r07" ], "n_fail": 471, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 5, "score": 0.2857142857142857, "task_id": "conflict_vlm_access_request_approval_v01_r05", "verb": "asset open" }, { "count": 5, "score": 0.75, "task_id": "conflict_vlm_access_request_approval_v02_r07", "verb": "file read" }, { "count": 6, "score": 0.3333333333333333, "task_id": "conflict_vlm_barrier_sync_deployment_v03_r01", "verb": "message send" }, { "count": 5, "score": 0.4, "task_id": "conflict_vlm_barrier_sync_deployment_v03_r02", "verb": "file read" } ], "loop_verbs": { "asset info": 1, "asset open": 5, "deploy status": 2, "file read": 35, "message send": 9 }, "loopy_capped": 52, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 277, "ok_false": 0, "pass_pct": 21.5, "passed": 129, "rank": { "avg_score": 4, "capped": 8, "failed_total": 5, "loopy_capped": 8, "near_miss": 8, "pass_pct": 4, "step_mean": 9, "step_p50": 8, "zero_open_tasks": 9, "zero_score": 3, "zero_write_tasks": 7 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.6889, "n": 10, "passed": 2 }, "access_request_approval_v02": { "mean_score": 0.7067, "n": 10, "passed": 0 }, "access_request_approval_v03": { "mean_score": 0.7306, "n": 10, "passed": 3 }, "barrier_sync_deployment_v01": { "mean_score": 0.7353, "n": 10, "passed": 4 }, "barrier_sync_deployment_v02": { "mean_score": 0.6821, "n": 10, "passed": 3 }, "barrier_sync_deployment_v03": { "mean_score": 0.62, "n": 10, "passed": 2 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.4333, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.569, "n": 10, "passed": 4 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.3794, "n": 10, "passed": 0 }, "collaborative_contract_review_v01": { "mean_score": 0.6444, "n": 10, "passed": 2 }, "collaborative_contract_review_v02": { "mean_score": 0.6867, "n": 10, "passed": 5 }, "collaborative_contract_review_v03": { "mean_score": 0.7544, "n": 10, "passed": 3 }, "coordinated_containment_v01": { "mean_score": 0.4548, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.6179, "n": 10, "passed": 1 }, "coordinated_containment_v03": { "mean_score": 0.7305, "n": 10, "passed": 1 }, "deadlock_multi_approval_v01": { "mean_score": 0.7008, "n": 10, "passed": 3 }, "deadlock_multi_approval_v02": { "mean_score": 0.8758, "n": 10, "passed": 6 }, "deadlock_multi_approval_v03": { "mean_score": 0.76, "n": 10, "passed": 4 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.6455, "n": 10, "passed": 3 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.3498, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.4119, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v01": { "mean_score": 0.6197, "n": 10, "passed": 2 }, "employee_onboarding_workflow_v02": { "mean_score": 0.6065, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v03": { "mean_score": 0.5919, "n": 10, "passed": 3 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.7833, "n": 10, "passed": 5 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.705, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.6967, "n": 10, "passed": 1 }, "expiring_resource_hold_v01": { "mean_score": 0.7162, "n": 10, "passed": 2 }, "expiring_resource_hold_v02": { "mean_score": 0.6345, "n": 10, "passed": 3 }, "expiring_resource_hold_v03": { "mean_score": 0.7712, "n": 10, "passed": 1 }, "incident_commander_selection_v01": { "mean_score": 0.425, "n": 10, "passed": 0 }, "incident_commander_selection_v02": { "mean_score": 0.7217, "n": 10, "passed": 3 }, "incident_commander_selection_v03": { "mean_score": 0.5733, "n": 10, "passed": 1 }, "it_deployment_gate_v01": { "mean_score": 0.5977, "n": 10, "passed": 3 }, "it_deployment_gate_v02": { "mean_score": 0.6768, "n": 10, "passed": 3 }, "it_deployment_gate_v03": { "mean_score": 0.6607, "n": 10, "passed": 2 }, "multi_device_pairing_v01": { "mean_score": 0.5161, "n": 10, "passed": 0 }, "multi_device_pairing_v02": { "mean_score": 0.6792, "n": 10, "passed": 1 }, "multi_device_pairing_v03": { "mean_score": 0.6253, "n": 10, "passed": 3 }, "multi_party_access_permit_v01": { "mean_score": 0.5383, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.7845, "n": 10, "passed": 4 }, "multi_party_access_permit_v03": { "mean_score": 0.6779, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.5019, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.4501, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.5946, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.42, "n": 10, "passed": 2 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.5483, "n": 10, "passed": 2 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.495, "n": 10, "passed": 2 }, "payment_chargeback_dispute_v01": { "mean_score": 0.9, "n": 10, "passed": 8 }, "payment_chargeback_dispute_v02": { "mean_score": 0.5167, "n": 10, "passed": 2 }, "payment_chargeback_dispute_v03": { "mean_score": 0.7333, "n": 10, "passed": 5 }, "satellite_ground_station_handover_v01": { "mean_score": 0.6667, "n": 10, "passed": 5 }, "satellite_ground_station_handover_v02": { "mean_score": 0.7, "n": 10, "passed": 3 }, "satellite_ground_station_handover_v03": { "mean_score": 0.75, "n": 10, "passed": 4 }, "sla_support_response_v01": { "mean_score": 0.5529, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.669, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.4492, "n": 10, "passed": 0 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.54, "n": 10, "passed": 3 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.5833, "n": 10, "passed": 4 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.2033, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 49, "fail_closed_pct": 2.3, "n": 2137 }, "side_effect_fields": { "approval_escalated": 5, "approval_granted": 3, "approval_recorded": 19, "approval_requested": 20, "assets_accessed": 41, "booking_extended": 4, "booking_held": 19, "booking_released": 10, "containment_action": 20, "deployment_approved": 6, "deployment_gate_set": 7, "deployment_requested": 24, "files_created": 1, "lock_acquired": 9, "messages_sent": 10, "pairing_code_entered": 3, "pairing_code_shown": 1, "pairing_completed": 1, "pairing_started": 9, "review_assigned": 2, "review_submitted": 7 }, "step_mean": 14.91, "step_p50": 15, "stops": { "incomplete_required_submission": 174, "missing_required_side_effect": 221, "none": 129, "reached max_steps=20": 76 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 10": 4, "wait 15": 6, "wait 180": 1, "wait 2": 1, "wait 30": 1, "wait 300": 2, "wait 5": 33 }, "max": 300.0, "mean": 23.1, "median": 5.0, "n": 48, "sub_30": 44, "total": 1107.0 }, "zero_open_tasks": 130, "zero_score": 26, "zero_write_tasks": 18 }, "luna": { "avg_score": 0.5909, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 16, "checks_total": 3760, "display": "gpt-5.6-luna", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'SEC-TELEMETRY-9914-BP': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'SEC-VERIFY-4109': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r06" }, { "message": "files_created.content contains 'FULLY_RECONCILED_AND_GRANTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r10" } ], "content_file": [ { "message": "LLM semantic match: The report states the final decisive finding as GRANTED after reconciliation rather than UNRECONCILED (2/3 approver scopes mismatched). [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The report identifies the target resource mismatch for 'billing-archive' (noted as 'mismatch: Wrong Resource'), but does not explicitly state the exact finding identifier 'TARGET-MISMATCH (billing-archive)'. [self-consistency 1/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" }, { "message": "LLM semantic match: The report finds that reconciliation checks failed and the status is UNRECONCILED, directly contradicting the criterion. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "LLM semantic match: The report explicitly concludes that access should not be granted because the request is not fully reconciled. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" } ], "content_msg": [ { "message": "LLM semantic match: No message was sent to Sana_Security_Reviewer in #iam-approvals-sec citing permit PERMIT-2026-BP-9982 and report path /reports/access_grant_decision.md. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The message sent to Ravi_Application_Owner requested re-approval for WRITE scope on billing-prod in #iam-approvals-sec, but cited AUDIT-APP-2026-ARCH instead of AUDIT-SEC-2026-READ. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: The message was sent to Sana_Security_Reviewer in #iam-approvals-sec to request the scope upgrade, but it provided the wrong passcode (SEC-PASS-8842-BP instead of SEC-TELEMETRY-9914-BP). [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "LLM semantic match: No message was sent to Quinn_Engineering_Manager in #proj-billing-migration confirming submission of passcode SEC-TELEMETRY-9914-BP to Sana. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r07': False; normalized fallback disabled ('iam_console_access_scope_matrix_r07' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "assets_accessed.id contains 'security_dashboard_telemetry_20260330_100000': False; normalized fallback disabled ('security_dashboard_telemetry_20260330_100000' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r08" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20241024_085500': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20241024_085500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v02_r07" }, { "message": "assets_accessed.id contains 'security_dashboard_soc_clearance_console_20241024_092500': False; normalized fallback disabled ('security_dashboard_soc_clearance_console_20241024_092500' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r09" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r04" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 68 }, "content_file": { "diagnostic": 0, "required": 217 }, "content_msg": { "diagnostic": 0, "required": 669 }, "grounding": { "diagnostic": 121, "required": 50 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 337 }, "window": { "diagnostic": 0, "required": 27 } }, "failed_total": 1489, "gating": { "content_involved": 429, "grounding_only": 5, "grounding_only_ids": [ "conflict_vlm_collaborative_contract_review_v02_r08", "conflict_vlm_distributed_lock_shared_resource_v03_r06", "conflict_vlm_employee_onboarding_workflow_v02_r08", "conflict_vlm_expense_reimbursement_conflict_v01_r04", "conflict_vlm_it_deployment_gate_v03_r10" ], "n_fail": 494, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 5, "score": 0.5, "task_id": "conflict_vlm_access_request_approval_v01_r01", "verb": "asset open" }, { "count": 6, "score": 0.75, "task_id": "conflict_vlm_access_request_approval_v02_r07", "verb": "approval approve" }, { "count": 5, "score": 0.3333333333333333, "task_id": "conflict_vlm_clinical_trial_dosage_conflict_v02_r01", "verb": "file read" }, { "count": 5, "score": 0.5714285714285714, "task_id": "conflict_vlm_coordinated_containment_v03_r02", "verb": "file read" } ], "loop_verbs": { "approval approve": 1, "asset open": 5, "booking confirm": 1, "file read": 2, "message search": 2, "message send": 1 }, "loopy_capped": 12, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 293, "ok_false": 0, "pass_pct": 17.7, "passed": 106, "rank": { "avg_score": 6, "capped": 5, "failed_total": 6, "loopy_capped": 5, "near_miss": 10, "pass_pct": 7, "step_mean": 4, "step_p50": 6, "zero_open_tasks": 2, "zero_score": 6, "zero_write_tasks": 3 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.6314, "n": 10, "passed": 2 }, "access_request_approval_v02": { "mean_score": 0.71, "n": 10, "passed": 1 }, "access_request_approval_v03": { "mean_score": 0.5433, "n": 10, "passed": 2 }, "barrier_sync_deployment_v01": { "mean_score": 0.672, "n": 10, "passed": 3 }, "barrier_sync_deployment_v02": { "mean_score": 0.7105, "n": 10, "passed": 2 }, "barrier_sync_deployment_v03": { "mean_score": 0.6667, "n": 10, "passed": 2 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.2933, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.3917, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.4465, "n": 10, "passed": 1 }, "collaborative_contract_review_v01": { "mean_score": 0.567, "n": 10, "passed": 1 }, "collaborative_contract_review_v02": { "mean_score": 0.7133, "n": 10, "passed": 3 }, "collaborative_contract_review_v03": { "mean_score": 0.8464, "n": 10, "passed": 6 }, "coordinated_containment_v01": { "mean_score": 0.4204, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.6594, "n": 10, "passed": 3 }, "coordinated_containment_v03": { "mean_score": 0.7805, "n": 10, "passed": 3 }, "deadlock_multi_approval_v01": { "mean_score": 0.6842, "n": 10, "passed": 3 }, "deadlock_multi_approval_v02": { "mean_score": 0.77, "n": 10, "passed": 3 }, "deadlock_multi_approval_v03": { "mean_score": 0.74, "n": 10, "passed": 4 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.6131, "n": 10, "passed": 2 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.2982, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.431, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v01": { "mean_score": 0.5664, "n": 10, "passed": 3 }, "employee_onboarding_workflow_v02": { "mean_score": 0.574, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v03": { "mean_score": 0.566, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.7667, "n": 10, "passed": 3 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.62, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.706, "n": 10, "passed": 1 }, "expiring_resource_hold_v01": { "mean_score": 0.6545, "n": 10, "passed": 2 }, "expiring_resource_hold_v02": { "mean_score": 0.4673, "n": 10, "passed": 1 }, "expiring_resource_hold_v03": { "mean_score": 0.7374, "n": 10, "passed": 1 }, "incident_commander_selection_v01": { "mean_score": 0.4833, "n": 10, "passed": 1 }, "incident_commander_selection_v02": { "mean_score": 0.6367, "n": 10, "passed": 3 }, "incident_commander_selection_v03": { "mean_score": 0.5767, "n": 10, "passed": 2 }, "it_deployment_gate_v01": { "mean_score": 0.6559, "n": 10, "passed": 2 }, "it_deployment_gate_v02": { "mean_score": 0.571, "n": 10, "passed": 1 }, "it_deployment_gate_v03": { "mean_score": 0.5716, "n": 10, "passed": 0 }, "multi_device_pairing_v01": { "mean_score": 0.4933, "n": 10, "passed": 1 }, "multi_device_pairing_v02": { "mean_score": 0.6336, "n": 10, "passed": 1 }, "multi_device_pairing_v03": { "mean_score": 0.6427, "n": 10, "passed": 3 }, "multi_party_access_permit_v01": { "mean_score": 0.5283, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.7512, "n": 10, "passed": 3 }, "multi_party_access_permit_v03": { "mean_score": 0.6662, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.5591, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.5102, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.6021, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.3833, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.4433, "n": 10, "passed": 0 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.51, "n": 10, "passed": 2 }, "payment_chargeback_dispute_v01": { "mean_score": 0.8667, "n": 10, "passed": 7 }, "payment_chargeback_dispute_v02": { "mean_score": 0.4583, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v03": { "mean_score": 0.6, "n": 10, "passed": 3 }, "satellite_ground_station_handover_v01": { "mean_score": 0.6833, "n": 10, "passed": 5 }, "satellite_ground_station_handover_v02": { "mean_score": 0.7333, "n": 10, "passed": 3 }, "satellite_ground_station_handover_v03": { "mean_score": 0.6, "n": 10, "passed": 2 }, "sla_support_response_v01": { "mean_score": 0.5924, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.6444, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.4492, "n": 10, "passed": 2 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.55, "n": 10, "passed": 1 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.5167, "n": 10, "passed": 2 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.295, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 22, "fail_closed_pct": 1.0, "n": 2137 }, "side_effect_fields": { "approval_escalated": 7, "approval_granted": 3, "approval_recorded": 18, "approval_requested": 19, "assets_accessed": 40, "booking_extended": 4, "booking_held": 25, "booking_released": 12, "containment_action": 19, "deployment_approved": 9, "deployment_cutover": 1, "deployment_gate_set": 12, "deployment_requested": 31, "document_signed": 1, "lock_acquired": 15, "messages_sent": 7, "pairing_code_entered": 4, "pairing_code_shown": 1, "pairing_completed": 1, "pairing_started": 12, "review_assigned": 6, "review_submitted": 7 }, "step_mean": 10.91, "step_p50": 11, "stops": { "incomplete_required_submission": 223, "missing_required_side_effect": 254, "none": 106, "reached max_steps=20": 16, "stopped after 3 identical no-progress transitions": 1 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 1": 33, "wait 10": 1, "wait 2": 45, "wait 3": 38, "wait 300": 5, "wait 4": 6, "wait 5": 28, "wait 8": 1 }, "max": 300.0, "mean": 12.2, "median": 3.0, "n": 158, "sub_30": 153, "total": 1925.0 }, "zero_open_tasks": 1, "zero_score": 34, "zero_write_tasks": 2 }, "opus": { "avg_score": 0.618, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 17, "checks_total": 3760, "display": "claude-opus-4.8", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'ERR-DATA-BACKFILL-409': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r03" }, { "message": "files_created.content contains 'AUTH-GATE-9942-SYNC': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v02_r03" }, { "message": "files_created.content contains '138.8 U/L': False", "role": "required", "task_id": "conflict_vlm_clinical_trial_dosage_conflict_v01_r02" } ], "content_file": [ { "message": "LLM semantic match: The report finds that reconciliation failed due to a scope mismatch rather than stating that all reconciliation checks passed. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "LLM semantic match: The report concludes with a verdict NOT to issue the access permit, directly contradicting the requirement to decide to proceed with the access grant. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "LLM semantic match: The report identifies TXN-APP-WRITE-9912 as the decisive transaction ID rather than TXN-APP-WRITE-9908. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r04" }, { "message": "LLM semantic match: The report's decision/verdict denies permit issuance rather than granting/issuing the access permit. [self-consistency 1/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r07" } ], "content_msg": [ { "message": "LLM semantic match: The message to Sana_Security_Reviewer in #iam-approvals-sec citing PERMIT-2026-BP-9982, /reports/access_grant_decision.md, and passcode SEC-PASS-8842-BP was not sent. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The message sent to Ravi_Application_Owner in #iam-approvals-sec requested re-approval for WRITE scope on billing-prod, but cited reference code AUDIT-APP-2026-ARCH instead of AUDIT-SEC-2026-READ. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: No message was sent to Quinn_Engineering_Manager reporting that all reconciliation checks passed. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "LLM semantic match: No message was sent to Sana_Security_Reviewer in #iam-approvals-sec confirming that all required approver scopes reconcile to WRITE on billing-prod; instead, a message was sent stating there is still a scope mismatch. [self-consistency 0/3 ", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20241024_085500': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20241024_085500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v02_r07" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20250218_103000': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20250218_103000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r07" }, { "message": "assets_accessed.id contains 'ops_console_gate_controller_metrics_20260330_143000': False; normalized fallback disabled ('ops_console_gate_controller_metrics_20260330_143000' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r04" }, { "message": "assets_accessed.id contains 'ops_console_canary_cutover_telemetry_20260330_141500': False; normalized fallback disabled ('ops_console_canary_cutover_telemetry_20260330_141500' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r06" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r05" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 55 }, "content_file": { "diagnostic": 0, "required": 202 }, "content_msg": { "diagnostic": 0, "required": 648 }, "grounding": { "diagnostic": 100, "required": 48 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 308 }, "window": { "diagnostic": 0, "required": 27 } }, "failed_total": 1388, "gating": { "content_involved": 396, "grounding_only": 1, "grounding_only_ids": [ "conflict_vlm_multi_party_access_permit_v01_r07" ], "n_fail": 457, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 5, "score": 0.625, "task_id": "conflict_vlm_access_request_approval_v01_r01", "verb": "asset open" }, { "count": 6, "score": 0.5714285714285714, "task_id": "conflict_vlm_coordinated_containment_v03_r02", "verb": "file read" }, { "count": 7, "score": 0.5, "task_id": "conflict_vlm_expense_reimbursement_conflict_v01_r08", "verb": "file read" }, { "count": 6, "score": 0.3333333333333333, "task_id": "conflict_vlm_expiring_resource_hold_v01_r08", "verb": "asset open" } ], "loop_verbs": { "asset open": 5, "file read": 3, "message search": 4, "message send": 1 }, "loopy_capped": 13, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 262, "ok_false": 0, "pass_pct": 23.8, "passed": 143, "rank": { "avg_score": 3, "capped": 6, "failed_total": 3, "loopy_capped": 6, "near_miss": 3, "pass_pct": 2, "step_mean": 6, "step_p50": 4, "zero_open_tasks": 1, "zero_score": 4, "zero_write_tasks": 6 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.7554, "n": 10, "passed": 2 }, "access_request_approval_v02": { "mean_score": 0.7592, "n": 10, "passed": 2 }, "access_request_approval_v03": { "mean_score": 0.4643, "n": 10, "passed": 1 }, "barrier_sync_deployment_v01": { "mean_score": 0.6886, "n": 10, "passed": 3 }, "barrier_sync_deployment_v02": { "mean_score": 0.7888, "n": 10, "passed": 5 }, "barrier_sync_deployment_v03": { "mean_score": 0.53, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.49, "n": 10, "passed": 2 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.306, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.4627, "n": 10, "passed": 1 }, "collaborative_contract_review_v01": { "mean_score": 0.4777, "n": 10, "passed": 1 }, "collaborative_contract_review_v02": { "mean_score": 0.7683, "n": 10, "passed": 5 }, "collaborative_contract_review_v03": { "mean_score": 0.8036, "n": 10, "passed": 5 }, "coordinated_containment_v01": { "mean_score": 0.5323, "n": 10, "passed": 1 }, "coordinated_containment_v02": { "mean_score": 0.6856, "n": 10, "passed": 3 }, "coordinated_containment_v03": { "mean_score": 0.7305, "n": 10, "passed": 1 }, "deadlock_multi_approval_v01": { "mean_score": 0.7175, "n": 10, "passed": 3 }, "deadlock_multi_approval_v02": { "mean_score": 0.8225, "n": 10, "passed": 4 }, "deadlock_multi_approval_v03": { "mean_score": 0.74, "n": 10, "passed": 4 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.7021, "n": 10, "passed": 4 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.3583, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.4911, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v01": { "mean_score": 0.5922, "n": 10, "passed": 3 }, "employee_onboarding_workflow_v02": { "mean_score": 0.527, "n": 10, "passed": 2 }, "employee_onboarding_workflow_v03": { "mean_score": 0.4918, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.8167, "n": 10, "passed": 5 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.79, "n": 10, "passed": 3 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.6824, "n": 10, "passed": 1 }, "expiring_resource_hold_v01": { "mean_score": 0.6405, "n": 10, "passed": 2 }, "expiring_resource_hold_v02": { "mean_score": 0.6345, "n": 10, "passed": 2 }, "expiring_resource_hold_v03": { "mean_score": 0.7571, "n": 10, "passed": 1 }, "incident_commander_selection_v01": { "mean_score": 0.5083, "n": 10, "passed": 1 }, "incident_commander_selection_v02": { "mean_score": 0.6733, "n": 10, "passed": 3 }, "incident_commander_selection_v03": { "mean_score": 0.6183, "n": 10, "passed": 2 }, "it_deployment_gate_v01": { "mean_score": 0.6959, "n": 10, "passed": 3 }, "it_deployment_gate_v02": { "mean_score": 0.6249, "n": 10, "passed": 4 }, "it_deployment_gate_v03": { "mean_score": 0.5787, "n": 10, "passed": 2 }, "multi_device_pairing_v01": { "mean_score": 0.4739, "n": 10, "passed": 1 }, "multi_device_pairing_v02": { "mean_score": 0.6659, "n": 10, "passed": 3 }, "multi_device_pairing_v03": { "mean_score": 0.6818, "n": 10, "passed": 3 }, "multi_party_access_permit_v01": { "mean_score": 0.5917, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.8762, "n": 10, "passed": 7 }, "multi_party_access_permit_v03": { "mean_score": 0.6429, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.5962, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.5019, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.5746, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.4033, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.5267, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.5883, "n": 10, "passed": 3 }, "payment_chargeback_dispute_v01": { "mean_score": 0.875, "n": 10, "passed": 6 }, "payment_chargeback_dispute_v02": { "mean_score": 0.475, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v03": { "mean_score": 0.6333, "n": 10, "passed": 4 }, "satellite_ground_station_handover_v01": { "mean_score": 0.5917, "n": 10, "passed": 4 }, "satellite_ground_station_handover_v02": { "mean_score": 0.8333, "n": 10, "passed": 5 }, "satellite_ground_station_handover_v03": { "mean_score": 0.625, "n": 10, "passed": 4 }, "sla_support_response_v01": { "mean_score": 0.6029, "n": 10, "passed": 1 }, "sla_support_response_v02": { "mean_score": 0.6565, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.4801, "n": 10, "passed": 2 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.6083, "n": 10, "passed": 4 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.5833, "n": 10, "passed": 3 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.2867, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 24, "fail_closed_pct": 1.1, "n": 2137 }, "side_effect_fields": { "approval_escalated": 7, "approval_granted": 3, "approval_recorded": 15, "approval_requested": 22, "assets_accessed": 39, "booking_extended": 4, "booking_held": 27, "booking_released": 11, "containment_action": 19, "deployment_approved": 8, "deployment_cutover": 1, "deployment_gate_set": 9, "deployment_requested": 27, "files_created": 2, "lock_acquired": 12, "messages_sent": 7, "pairing_code_entered": 1, "pairing_code_shown": 1, "pairing_completed": 1, "pairing_started": 8, "review_assigned": 5, "review_submitted": 8 }, "step_mean": 11.11, "step_p50": 10, "stops": { "incomplete_required_submission": 203, "missing_required_side_effect": 237, "none": 143, "reached max_steps=20": 17 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 120": 3, "wait 180": 4, "wait 30": 70, "wait 300": 2, "wait 45": 1, "wait 60": 23, "wait 90": 4 }, "max": 300.0, "mean": 52.0, "median": 30.0, "n": 107, "sub_30": 0, "total": 5565.0 }, "zero_open_tasks": 0, "zero_score": 28, "zero_write_tasks": 6 }, "opus5": { "avg_score": 0.6328, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 0, "checks_total": 3760, "display": "claude-opus-5", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'ERR-DATA-BACKFILL-409': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r03" }, { "message": "files_created.content contains 'AUTH-GATE-9942-SYNC': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v02_r03" }, { "message": "files_created.content contains 'BARRIER-SATISFIED-V03': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v03_r07" } ], "content_file": [ { "message": "LLM semantic match: The report finds that reconciliation checks failed and access was denied, rather than stating that all reconciliation checks passed. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "LLM semantic match: The report's verdict explicitly denies and blocks the access grant rather than proceeding with it. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "LLM semantic match: The report reaches a verdict of NOT SATISFIED and decides to withhold grant execution rather than verifying the permit and initiating runbook handoff. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r08" }, { "message": "LLM semantic match: The report references transaction ID TXN-APP-WRITE-9912 as the decisive finding instead of TXN-APP-WRITE-9908. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r04" } ], "content_msg": [ { "message": "LLM semantic match: The message requested re-approval for WRITE scope on billing-prod from Ravi in #iam-approvals-sec, but cited code AUDIT-APP-2026-ARCH instead of AUDIT-SEC-2026-READ. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: No message was sent to Sana_Security_Reviewer informing her of the Application Owner's TARGET-MISMATCH (billing-archive) for REQ-2026-0842, as the message to her only discussed her own narrower-scope mismatch. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" }, { "message": "LLM semantic match: The message sent to Ravi in #iam-approvals-sec explicitly stated that the corrected WRITE approval was NOT yet registered rather than confirming its registration. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r04" }, { "message": "LLM semantic match: The message sent to Sana_Security_Reviewer explicitly refused to confirm that the approval was registered, stating its status was RECEIVED rather than VERIFIED. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r04" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r01': False; normalized fallback disabled ('iam_console_access_scope_matrix_r01' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20241024_085500': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20241024_085500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v02_r07" }, { "message": "assets_accessed.id contains 'audit_dashboard_reconciliation_verdict_20250218_110500': False; normalized fallback disabled ('audit_dashboard_reconciliation_verdict_20250218_110500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r10" }, { "message": "assets_accessed.id contains 'ops_console_canary_cutover_telemetry_20260330_141500': False; normalized fallback disabled ('ops_console_canary_cutover_telemetry_20260330_141500' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r06" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r05" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 48 }, "content_file": { "diagnostic": 0, "required": 224 }, "content_msg": { "diagnostic": 0, "required": 597 }, "grounding": { "diagnostic": 85, "required": 39 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 325 }, "window": { "diagnostic": 0, "required": 24 } }, "failed_total": 1342, "gating": { "content_involved": 378, "grounding_only": 2, "grounding_only_ids": [ "conflict_vlm_multi_device_pairing_v01_r04", "conflict_vlm_multi_tab_slot_monitoring_v01_r07" ], "n_fail": 448, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [], "loop_verbs": {}, "loopy_capped": 0, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 264, "ok_false": 0, "pass_pct": 25.3, "passed": 152, "rank": { "avg_score": 1, "capped": 1, "failed_total": 2, "loopy_capped": 1, "near_miss": 4, "pass_pct": 1, "step_mean": 2, "step_p50": 2, "zero_open_tasks": 2, "zero_score": 2, "zero_write_tasks": 5 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.6392, "n": 10, "passed": 1 }, "access_request_approval_v02": { "mean_score": 0.8442, "n": 10, "passed": 3 }, "access_request_approval_v03": { "mean_score": 0.5314, "n": 10, "passed": 1 }, "barrier_sync_deployment_v01": { "mean_score": 0.6605, "n": 10, "passed": 2 }, "barrier_sync_deployment_v02": { "mean_score": 0.6871, "n": 10, "passed": 2 }, "barrier_sync_deployment_v03": { "mean_score": 0.5167, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.5267, "n": 10, "passed": 2 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.3976, "n": 10, "passed": 0 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.3924, "n": 10, "passed": 1 }, "collaborative_contract_review_v01": { "mean_score": 0.5825, "n": 10, "passed": 4 }, "collaborative_contract_review_v02": { "mean_score": 0.8283, "n": 10, "passed": 6 }, "collaborative_contract_review_v03": { "mean_score": 0.8112, "n": 10, "passed": 5 }, "coordinated_containment_v01": { "mean_score": 0.5395, "n": 10, "passed": 1 }, "coordinated_containment_v02": { "mean_score": 0.6213, "n": 10, "passed": 3 }, "coordinated_containment_v03": { "mean_score": 0.6405, "n": 10, "passed": 1 }, "deadlock_multi_approval_v01": { "mean_score": 0.7175, "n": 10, "passed": 3 }, "deadlock_multi_approval_v02": { "mean_score": 0.8483, "n": 10, "passed": 5 }, "deadlock_multi_approval_v03": { "mean_score": 0.6779, "n": 10, "passed": 2 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.8024, "n": 10, "passed": 4 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.3974, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.422, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v01": { "mean_score": 0.5831, "n": 10, "passed": 3 }, "employee_onboarding_workflow_v02": { "mean_score": 0.594, "n": 10, "passed": 2 }, "employee_onboarding_workflow_v03": { "mean_score": 0.5085, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.7217, "n": 10, "passed": 3 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.595, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.7464, "n": 10, "passed": 4 }, "expiring_resource_hold_v01": { "mean_score": 0.7171, "n": 10, "passed": 4 }, "expiring_resource_hold_v02": { "mean_score": 0.5821, "n": 10, "passed": 2 }, "expiring_resource_hold_v03": { "mean_score": 0.7714, "n": 10, "passed": 2 }, "incident_commander_selection_v01": { "mean_score": 0.6333, "n": 10, "passed": 3 }, "incident_commander_selection_v02": { "mean_score": 0.71, "n": 10, "passed": 4 }, "incident_commander_selection_v03": { "mean_score": 0.685, "n": 10, "passed": 3 }, "it_deployment_gate_v01": { "mean_score": 0.6232, "n": 10, "passed": 4 }, "it_deployment_gate_v02": { "mean_score": 0.4515, "n": 10, "passed": 0 }, "it_deployment_gate_v03": { "mean_score": 0.5974, "n": 10, "passed": 2 }, "multi_device_pairing_v01": { "mean_score": 0.6189, "n": 10, "passed": 1 }, "multi_device_pairing_v02": { "mean_score": 0.6659, "n": 10, "passed": 3 }, "multi_device_pairing_v03": { "mean_score": 0.7187, "n": 10, "passed": 4 }, "multi_party_access_permit_v01": { "mean_score": 0.6942, "n": 10, "passed": 2 }, "multi_party_access_permit_v02": { "mean_score": 0.8595, "n": 10, "passed": 6 }, "multi_party_access_permit_v03": { "mean_score": 0.7229, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.645, "n": 10, "passed": 1 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.471, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.5871, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.4, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.585, "n": 10, "passed": 2 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.53, "n": 10, "passed": 3 }, "payment_chargeback_dispute_v01": { "mean_score": 0.8667, "n": 10, "passed": 8 }, "payment_chargeback_dispute_v02": { "mean_score": 0.5, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v03": { "mean_score": 0.6, "n": 10, "passed": 3 }, "satellite_ground_station_handover_v01": { "mean_score": 0.7583, "n": 10, "passed": 5 }, "satellite_ground_station_handover_v02": { "mean_score": 0.7333, "n": 10, "passed": 4 }, "satellite_ground_station_handover_v03": { "mean_score": 0.7083, "n": 10, "passed": 6 }, "sla_support_response_v01": { "mean_score": 0.66, "n": 10, "passed": 1 }, "sla_support_response_v02": { "mean_score": 0.6958, "n": 10, "passed": 1 }, "sla_support_response_v03": { "mean_score": 0.4635, "n": 10, "passed": 2 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.7167, "n": 10, "passed": 4 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.8, "n": 10, "passed": 5 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.365, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 20, "fail_closed_pct": 0.9, "n": 2137 }, "side_effect_fields": { "approval_escalated": 5, "approval_granted": 3, "approval_recorded": 20, "approval_requested": 23, "assets_accessed": 34, "booking_extended": 4, "booking_held": 24, "booking_released": 11, "containment_action": 21, "deployment_approved": 9, "deployment_cutover": 2, "deployment_gate_set": 9, "deployment_requested": 31, "files_created": 3, "lock_acquired": 14, "messages_sent": 3, "pairing_code_entered": 2, "pairing_code_shown": 1, "pairing_completed": 1, "pairing_started": 10, "review_assigned": 4, "review_submitted": 8 }, "step_mean": 9.65, "step_p50": 9, "stops": { "generation_stall_empty_reply": 3, "incomplete_required_submission": 203, "missing_required_side_effect": 242, "none": 152 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 120": 8, "wait 180": 4, "wait 30": 7, "wait 300": 5, "wait 45": 1, "wait 60": 47, "wait 90": 1 }, "max": 300.0, "mean": 86.9, "median": 60.0, "n": 73, "sub_30": 0, "total": 6345.0 }, "zero_open_tasks": 1, "zero_score": 21, "zero_write_tasks": 3 }, "pro": { "avg_score": 0.4793, "bucket_names": { "anchor": { "effect:files_created field_contains": 255 }, "content_file": { "effect:files_created field_semantic_match": 942 }, "content_msg": { "effect:messages_sent field_semantic_match": 1193 }, "grounding": { "effect:assets_accessed field_contains": 820 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 22, "effect:lock_released any_field_equals": 22, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 61, "checks_total": 3753, "display": "gemini-3.1-pro", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'SEC-TELEMETRY-9914-BP': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'FULLY_RECONCILED_AND_GRANTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r10" }, { "message": "files_created.content contains 'ERR-DATA-BACKFILL-409': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r03" } ], "content_file": [ { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "LLM semantic match: The report concludes that the status is BLOCKED rather than stating RECONCILIATION IN PROGRESS (2/3 roles verified WRITE). [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r06" } ], "content_msg": [ { "message": "LLM semantic match: A message was sent to Quinn explaining the scope mismatches, but it failed to explicitly state the overall status 'UNRECONCILED (2/3 approver scopes mismatched)' and reject the premature grant demand. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The message authorizing Sana_Security_Reviewer to release the migration runbook with permit PERMIT-2026-BP-9982 and report path /reports/access_grant_decision.md was not sent. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The message in #iam-approvals-sec cited reference code AUDIT-SEC-2026-READ and requested updated WRITE authorization on billing-prod, but failed to include the passcode SEC-PASS-8842-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: A message was sent in #iam-approvals-sec regarding the billing-archive mismatch, but it omitted the explicit request ID REQ-2026-0842 and status text TARGET-MISMATCH (billing-archive). [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r07': False; normalized fallback disabled ('iam_console_access_scope_matrix_r07' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20241024_085500': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20241024_085500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v02_r07" }, { "message": "assets_accessed.id contains 'audit_dashboard_passcode_telemetry_20250218_103000': False; normalized fallback disabled ('audit_dashboard_passcode_telemetry_20250218_103000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r06" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20250218_103000': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20250218_103000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r07" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r04" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 87 }, "content_file": { "diagnostic": 0, "required": 369 }, "content_msg": { "diagnostic": 0, "required": 845 }, "grounding": { "diagnostic": 121, "required": 54 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 319 }, "window": { "diagnostic": 0, "required": 25 } }, "failed_total": 1820, "gating": { "content_involved": 511, "grounding_only": 1, "grounding_only_ids": [ "conflict_vlm_access_request_approval_v01_r07" ], "n_fail": 541, "no_required_check_failed": 1, "no_required_check_failed_ids": [ "conflict_vlm_distributed_lock_shared_resource_v02_r02" ] }, "loop_examples": [ { "count": 7, "score": 0.25, "task_id": "conflict_vlm_access_request_approval_v01_r01", "verb": "asset open" }, { "count": 7, "score": 0.2857142857142857, "task_id": "conflict_vlm_access_request_approval_v01_r05", "verb": "message search" }, { "count": 9, "score": 0.25, "task_id": "conflict_vlm_access_request_approval_v02_r01", "verb": "message search" }, { "count": 6, "score": 0.6, "task_id": "conflict_vlm_access_request_approval_v02_r03", "verb": "message search" } ], "loop_verbs": { "agent status": 4, "asset open": 10, "file list": 2, "file read": 1, "message search": 22, "message send": 1 }, "loopy_capped": 41, "missing_check_result_ids": [ "conflict_vlm_distributed_lock_shared_resource_v02_r02" ], "missing_check_results": 1, "n": 600, "near_miss": 250, "ok_false": 0, "pass_pct": 9.8, "passed": 59, "rank": { "avg_score": 9, "capped": 7, "failed_total": 9, "loopy_capped": 7, "near_miss": 2, "pass_pct": 9, "step_mean": 5, "step_p50": 4, "zero_open_tasks": 7, "zero_score": 9, "zero_write_tasks": 8 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.5664, "n": 10, "passed": 1 }, "access_request_approval_v02": { "mean_score": 0.52, "n": 10, "passed": 0 }, "access_request_approval_v03": { "mean_score": 0.5838, "n": 10, "passed": 1 }, "barrier_sync_deployment_v01": { "mean_score": 0.5862, "n": 10, "passed": 2 }, "barrier_sync_deployment_v02": { "mean_score": 0.5621, "n": 10, "passed": 2 }, "barrier_sync_deployment_v03": { "mean_score": 0.6133, "n": 10, "passed": 2 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.2267, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.2571, "n": 10, "passed": 0 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.1277, "n": 10, "passed": 0 }, "collaborative_contract_review_v01": { "mean_score": 0.4736, "n": 10, "passed": 1 }, "collaborative_contract_review_v02": { "mean_score": 0.6217, "n": 10, "passed": 3 }, "collaborative_contract_review_v03": { "mean_score": 0.7373, "n": 10, "passed": 1 }, "coordinated_containment_v01": { "mean_score": 0.2683, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.4871, "n": 10, "passed": 0 }, "coordinated_containment_v03": { "mean_score": 0.6129, "n": 10, "passed": 1 }, "deadlock_multi_approval_v01": { "mean_score": 0.6267, "n": 10, "passed": 2 }, "deadlock_multi_approval_v02": { "mean_score": 0.7283, "n": 10, "passed": 3 }, "deadlock_multi_approval_v03": { "mean_score": 0.5286, "n": 10, "passed": 1 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.5438, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.2664, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.256, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v01": { "mean_score": 0.5254, "n": 10, "passed": 2 }, "employee_onboarding_workflow_v02": { "mean_score": 0.4287, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v03": { "mean_score": 0.3624, "n": 10, "passed": 0 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.5283, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.6, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.4798, "n": 10, "passed": 0 }, "expiring_resource_hold_v01": { "mean_score": 0.6179, "n": 10, "passed": 1 }, "expiring_resource_hold_v02": { "mean_score": 0.5036, "n": 10, "passed": 0 }, "expiring_resource_hold_v03": { "mean_score": 0.6455, "n": 10, "passed": 0 }, "incident_commander_selection_v01": { "mean_score": 0.2417, "n": 10, "passed": 0 }, "incident_commander_selection_v02": { "mean_score": 0.44, "n": 10, "passed": 0 }, "incident_commander_selection_v03": { "mean_score": 0.3983, "n": 10, "passed": 1 }, "it_deployment_gate_v01": { "mean_score": 0.5451, "n": 10, "passed": 1 }, "it_deployment_gate_v02": { "mean_score": 0.6132, "n": 10, "passed": 3 }, "it_deployment_gate_v03": { "mean_score": 0.6667, "n": 10, "passed": 2 }, "multi_device_pairing_v01": { "mean_score": 0.4156, "n": 10, "passed": 0 }, "multi_device_pairing_v02": { "mean_score": 0.5747, "n": 10, "passed": 1 }, "multi_device_pairing_v03": { "mean_score": 0.6193, "n": 10, "passed": 3 }, "multi_party_access_permit_v01": { "mean_score": 0.4483, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.6512, "n": 10, "passed": 1 }, "multi_party_access_permit_v03": { "mean_score": 0.6012, "n": 10, "passed": 1 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.3743, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.3865, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.4286, "n": 10, "passed": 0 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.3033, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.3617, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.34, "n": 10, "passed": 0 }, "payment_chargeback_dispute_v01": { "mean_score": 0.8417, "n": 10, "passed": 5 }, "payment_chargeback_dispute_v02": { "mean_score": 0.3833, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v03": { "mean_score": 0.4667, "n": 10, "passed": 1 }, "satellite_ground_station_handover_v01": { "mean_score": 0.45, "n": 10, "passed": 3 }, "satellite_ground_station_handover_v02": { "mean_score": 0.6, "n": 10, "passed": 1 }, "satellite_ground_station_handover_v03": { "mean_score": 0.4667, "n": 10, "passed": 2 }, "sla_support_response_v01": { "mean_score": 0.4248, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.5226, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.4301, "n": 10, "passed": 1 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.3617, "n": 10, "passed": 1 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.425, "n": 10, "passed": 2 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.0917, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 65, "fail_closed_pct": 3.0, "n": 2135 }, "side_effect_fields": { "approval_escalated": 5, "approval_granted": 3, "approval_recorded": 18, "approval_requested": 24, "assets_accessed": 43, "booking_extended": 4, "booking_held": 26, "booking_released": 10, "containment_action": 21, "deployment_approved": 9, "deployment_cutover": 1, "deployment_gate_set": 10, "deployment_requested": 26, "files_created": 5, "lock_acquired": 16, "lock_released": 1, "messages_sent": 2, "pairing_code_entered": 2, "pairing_code_shown": 1, "pairing_completed": 1, "pairing_started": 10, "review_assigned": 6, "review_submitted": 9 }, "step_mean": 11.01, "step_p50": 10, "stops": { "incomplete_required_submission": 226, "missing_required_side_effect": 253, "none": 60, "reached max_steps=20": 61 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 255 }, "content_file": { "diagnostic": 0, "required": 942 }, "content_msg": { "diagnostic": 0, "required": 1193 }, "grounding": { "diagnostic": 475, "required": 345 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 513 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 10": 176, "wait 120": 3, "wait 15": 1, "wait 20": 13, "wait 30": 26, "wait 300": 7, "wait 5": 45, "wait 60": 6 }, "max": 7200.0, "mean": 65.7, "median": 10.0, "n": 281, "sub_30": 235, "total": 18460.0 }, "zero_open_tasks": 11, "zero_score": 62, "zero_write_tasks": 31 }, "sol": { "avg_score": 0.6276, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 12, "checks_total": 3760, "display": "gpt-5.6-sol", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'SEC-TELEMETRY-9914-BP': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'FULLY_RECONCILED_AND_GRANTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r10" }, { "message": "files_created.content contains 'ERR-DATA-BACKFILL-409': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r03" } ], "content_file": [ { "message": "LLM semantic match: The report concludes that access was granted after successful reconciliation, rather than stating the decisive finding as UNRECONCILED. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The report states that reconciliation is in progress and holds the permit, but indicates only 1 of 3 roles is fully verified rather than 2 of 3. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r06" }, { "message": "LLM semantic match: The report references transaction TXN-APP-WRITE-9912 instead of TXN-APP-WRITE-9908. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r04" }, { "message": "LLM semantic match: The report captures that runbook release requires passcode verification, but states the formal verdict as 'SECURITY SCOPE RECONCILED \u2014 WRITE APPROVED; FINAL RELEASE CONDITIONALLY HELD' rather than 'PENDING RUNBOOK RELEASE (Passcode Verifica", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r05" } ], "content_msg": [ { "message": "LLM semantic match: The message to Sana_Security_Reviewer in #iam-approvals-sec cited AUDIT-SEC-2026-READ and requested updated WRITE authorization on billing-prod, but omitted the required passcode SEC-PASS-8842-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: A message was sent in #iam-approvals-sec to Ravi_Application_Owner requesting re-approval for billing-prod WRITE scope, but reference code AUDIT-SEC-2026-READ was cited for Sana rather than Ravi. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: The request to upgrade Sana's scope to WRITE on billing-prod was sent in #iam-approvals-sec, but it provided the wrong passcode (SEC-PASS-8842-BP instead of SEC-TELEMETRY-9914-BP). [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "LLM semantic match: No message was sent to Quinn_Engineering_Manager in #proj-billing-migration confirming submission of passcode SEC-TELEMETRY-9914-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20250218_103000': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20250218_103000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r07" }, { "message": "assets_accessed.id contains 'audit_dashboard_reconciliation_verdict_20250218_110500': False; normalized fallback disabled ('audit_dashboard_reconciliation_verdict_20250218_110500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r10" }, { "message": "assets_accessed.id contains 'telemetry_dashboard_worker_batch_rate_20231104_081115': False; normalized fallback disabled ('telemetry_dashboard_worker_batch_rate_20231104_081115' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v03_r04" }, { "message": "assets_accessed.id contains 'telemetry_dashboard_staging_cutover_execution_20231104_081800': False; normalized fallback disabled ('telemetry_dashboard_staging_cutover_execution_20231104_081800' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v03_r08" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r04" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 66 }, "content_file": { "diagnostic": 0, "required": 191 }, "content_msg": { "diagnostic": 0, "required": 630 }, "grounding": { "diagnostic": 71, "required": 31 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 311 }, "window": { "diagnostic": 0, "required": 27 } }, "failed_total": 1327, "gating": { "content_involved": 402, "grounding_only": 3, "grounding_only_ids": [ "conflict_vlm_distributed_lock_shared_resource_v03_r06", "conflict_vlm_employee_onboarding_workflow_v02_r08", "conflict_vlm_sla_support_response_v03_r07" ], "n_fail": 462, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 6, "score": 0.2857142857142857, "task_id": "conflict_vlm_access_request_approval_v01_r05", "verb": "message send" }, { "count": 10, "score": 0.75, "task_id": "conflict_vlm_access_request_approval_v02_r01", "verb": "message search" }, { "count": 6, "score": 0.8571428571428571, "task_id": "conflict_vlm_coordinated_containment_v02_r06", "verb": "asset open" }, { "count": 8, "score": 0.8, "task_id": "conflict_vlm_deadlock_multi_approval_v03_r04", "verb": "message search" } ], "loop_verbs": { "asset open": 2, "file read": 1, "message search": 4, "message send": 1 }, "loopy_capped": 8, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 278, "ok_false": 0, "pass_pct": 23.0, "passed": 138, "rank": { "avg_score": 2, "capped": 4, "failed_total": 1, "loopy_capped": 4, "near_miss": 9, "pass_pct": 3, "step_mean": 7, "step_p50": 7, "zero_open_tasks": 2, "zero_score": 1, "zero_write_tasks": 1 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.7139, "n": 10, "passed": 2 }, "access_request_approval_v02": { "mean_score": 0.7267, "n": 10, "passed": 1 }, "access_request_approval_v03": { "mean_score": 0.6028, "n": 10, "passed": 2 }, "barrier_sync_deployment_v01": { "mean_score": 0.6511, "n": 10, "passed": 2 }, "barrier_sync_deployment_v02": { "mean_score": 0.7638, "n": 10, "passed": 4 }, "barrier_sync_deployment_v03": { "mean_score": 0.6467, "n": 10, "passed": 2 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.2767, "n": 10, "passed": 0 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.5357, "n": 10, "passed": 2 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.4832, "n": 10, "passed": 1 }, "collaborative_contract_review_v01": { "mean_score": 0.6646, "n": 10, "passed": 2 }, "collaborative_contract_review_v02": { "mean_score": 0.735, "n": 10, "passed": 5 }, "collaborative_contract_review_v03": { "mean_score": 0.7731, "n": 10, "passed": 3 }, "coordinated_containment_v01": { "mean_score": 0.4895, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.7879, "n": 10, "passed": 4 }, "coordinated_containment_v03": { "mean_score": 0.7105, "n": 10, "passed": 2 }, "deadlock_multi_approval_v01": { "mean_score": 0.6675, "n": 10, "passed": 3 }, "deadlock_multi_approval_v02": { "mean_score": 0.8675, "n": 10, "passed": 5 }, "deadlock_multi_approval_v03": { "mean_score": 0.775, "n": 10, "passed": 3 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.6964, "n": 10, "passed": 4 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.395, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.4095, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v01": { "mean_score": 0.5837, "n": 10, "passed": 2 }, "employee_onboarding_workflow_v02": { "mean_score": 0.6617, "n": 10, "passed": 2 }, "employee_onboarding_workflow_v03": { "mean_score": 0.5326, "n": 10, "passed": 0 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.7917, "n": 10, "passed": 5 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.6, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.6574, "n": 10, "passed": 1 }, "expiring_resource_hold_v01": { "mean_score": 0.6355, "n": 10, "passed": 2 }, "expiring_resource_hold_v02": { "mean_score": 0.4821, "n": 10, "passed": 1 }, "expiring_resource_hold_v03": { "mean_score": 0.775, "n": 10, "passed": 2 }, "incident_commander_selection_v01": { "mean_score": 0.5083, "n": 10, "passed": 1 }, "incident_commander_selection_v02": { "mean_score": 0.7233, "n": 10, "passed": 5 }, "incident_commander_selection_v03": { "mean_score": 0.7017, "n": 10, "passed": 3 }, "it_deployment_gate_v01": { "mean_score": 0.6686, "n": 10, "passed": 4 }, "it_deployment_gate_v02": { "mean_score": 0.6374, "n": 10, "passed": 4 }, "it_deployment_gate_v03": { "mean_score": 0.7356, "n": 10, "passed": 3 }, "multi_device_pairing_v01": { "mean_score": 0.5072, "n": 10, "passed": 0 }, "multi_device_pairing_v02": { "mean_score": 0.6847, "n": 10, "passed": 3 }, "multi_device_pairing_v03": { "mean_score": 0.6219, "n": 10, "passed": 3 }, "multi_party_access_permit_v01": { "mean_score": 0.525, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.7679, "n": 10, "passed": 4 }, "multi_party_access_permit_v03": { "mean_score": 0.7457, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.5611, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.451, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.5346, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.46, "n": 10, "passed": 2 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.5567, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.5683, "n": 10, "passed": 3 }, "payment_chargeback_dispute_v01": { "mean_score": 0.9333, "n": 10, "passed": 8 }, "payment_chargeback_dispute_v02": { "mean_score": 0.4, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v03": { "mean_score": 0.7, "n": 10, "passed": 3 }, "satellite_ground_station_handover_v01": { "mean_score": 0.6917, "n": 10, "passed": 5 }, "satellite_ground_station_handover_v02": { "mean_score": 0.9, "n": 10, "passed": 8 }, "satellite_ground_station_handover_v03": { "mean_score": 0.75, "n": 10, "passed": 5 }, "sla_support_response_v01": { "mean_score": 0.5671, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.644, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.4435, "n": 10, "passed": 0 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.6717, "n": 10, "passed": 3 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.6167, "n": 10, "passed": 3 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.285, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 14, "fail_closed_pct": 0.7, "n": 2137 }, "side_effect_fields": { "approval_escalated": 7, "approval_granted": 3, "approval_recorded": 18, "approval_requested": 22, "assets_accessed": 27, "booking_extended": 4, "booking_held": 25, "booking_released": 12, "containment_action": 20, "deployment_approved": 8, "deployment_cutover": 1, "deployment_gate_set": 11, "deployment_requested": 34, "files_created": 2, "lock_acquired": 17, "lock_released": 1, "messages_sent": 3, "pairing_code_entered": 3, "pairing_started": 12, "review_assigned": 5, "review_submitted": 8 }, "step_mean": 12.41, "step_p50": 12, "stops": { "generation_stall_empty_reply": 1, "incomplete_required_submission": 206, "missing_required_side_effect": 243, "none": 138, "reached max_steps=20": 12 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 1": 9, "wait 10": 36, "wait 15": 10, "wait 2": 30, "wait 20": 5, "wait 3": 31, "wait 30": 11, "wait 5": 88 }, "max": 300.0, "mean": 15.5, "median": 5.0, "n": 232, "sub_30": 210, "total": 3590.0 }, "zero_open_tasks": 1, "zero_score": 20, "zero_write_tasks": 0 }, "sonnet": { "avg_score": 0.5701, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 6, "checks_total": 3760, "display": "claude-sonnet-5", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'FULLY_RECONCILED_AND_GRANTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r10" }, { "message": "files_created.content contains 'ERR-DATA-BACKFILL-409': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r03" }, { "message": "files_created.content contains 'AUTH-GATE-9942-SYNC': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v02_r03" } ], "content_file": [ { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: no 'content' content in files_created to evaluate (empty transcript)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The report finds the status to be UNRECONCILED / BLOCKED with 2 of 3 sign-offs failing to match, rather than stating 'RECONCILIATION IN PROGRESS (2/3 roles verified WRITE)'. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r06" }, { "message": "LLM semantic match: The report concludes that reconciliation failed and is not ready for permit issuance, contradicting the criterion that all checks passed. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" } ], "content_msg": [ { "message": "LLM semantic match: No message was sent to Sana_Security_Reviewer citing permit PERMIT-2026-BP-9982 or report path /reports/access_grant_decision.md to authorize the runbook release. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The message was sent to Ravi_Application_Owner requesting re-approval for WRITE scope on billing-prod in #iam-approvals-sec, but cited reference code AUDIT-APP-2026-ARCH instead of AUDIT-SEC-2026-READ. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: The message confirming the tracking code was sent to Ravi_Application_Owner instead of Sana_Security_Reviewer in #iam-approvals-sec. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r04" }, { "message": "LLM semantic match: A message was sent to Sana_Security_Reviewer in #iam-approvals-sec requesting the scope upgrade, but it provided the wrong passcode (SEC-PASS-8842-BP instead of SEC-TELEMETRY-9914-BP). [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_console_reconciliation_audit_r06': False; normalized fallback disabled ('iam_console_reconciliation_audit_r06' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r06" }, { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r07': False; normalized fallback disabled ('iam_console_access_scope_matrix_r07' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20241024_085500': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20241024_085500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v02_r07" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20250218_101000': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20250218_101000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r05" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r04" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 63 }, "content_file": { "diagnostic": 0, "required": 247 }, "content_msg": { "diagnostic": 0, "required": 641 }, "grounding": { "diagnostic": 219, "required": 122 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 356 }, "window": { "diagnostic": 0, "required": 30 } }, "failed_total": 1678, "gating": { "content_involved": 408, "grounding_only": 18, "grounding_only_ids": [ "conflict_vlm_barrier_sync_deployment_v02_r04", "conflict_vlm_collaborative_contract_review_v02_r06", "conflict_vlm_deadlock_multi_approval_v01_r10", "conflict_vlm_deadlock_multi_approval_v02_r07", "conflict_vlm_deadlock_multi_approval_v02_r10", "conflict_vlm_distributed_lock_shared_resource_v01_r03", "conflict_vlm_distributed_lock_shared_resource_v01_r09", "conflict_vlm_distributed_lock_shared_resource_v03_r06", "conflict_vlm_employee_onboarding_workflow_v01_r08", "conflict_vlm_employee_onboarding_workflow_v01_r10", "conflict_vlm_employee_onboarding_workflow_v02_r06", "conflict_vlm_employee_onboarding_workflow_v03_r07" ], "n_fail": 498, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 7, "score": 0.375, "task_id": "conflict_vlm_access_request_approval_v01_r01", "verb": "asset open" }, { "count": 6, "score": 0.5714285714285714, "task_id": "conflict_vlm_coordinated_containment_v03_r02", "verb": "file read" }, { "count": 8, "score": 0.3333333333333333, "task_id": "conflict_vlm_incident_commander_selection_v01_r06", "verb": "message search" }, { "count": 8, "score": 0.0, "task_id": "conflict_vlm_payment_chargeback_dispute_v03_r01", "verb": "message search" } ], "loop_verbs": { "asset open": 1, "file read": 1, "message search": 2 }, "loopy_capped": 4, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 269, "ok_false": 0, "pass_pct": 17.0, "passed": 102, "rank": { "avg_score": 8, "capped": 2, "failed_total": 8, "loopy_capped": 2, "near_miss": 6, "pass_pct": 8, "step_mean": 1, "step_p50": 1, "zero_open_tasks": 10, "zero_score": 8, "zero_write_tasks": 3 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.6008, "n": 10, "passed": 1 }, "access_request_approval_v02": { "mean_score": 0.7667, "n": 10, "passed": 2 }, "access_request_approval_v03": { "mean_score": 0.4119, "n": 10, "passed": 1 }, "barrier_sync_deployment_v01": { "mean_score": 0.645, "n": 10, "passed": 3 }, "barrier_sync_deployment_v02": { "mean_score": 0.7388, "n": 10, "passed": 4 }, "barrier_sync_deployment_v03": { "mean_score": 0.5067, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.4067, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.3869, "n": 10, "passed": 0 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.4513, "n": 10, "passed": 1 }, "collaborative_contract_review_v01": { "mean_score": 0.6075, "n": 10, "passed": 4 }, "collaborative_contract_review_v02": { "mean_score": 0.755, "n": 10, "passed": 4 }, "collaborative_contract_review_v03": { "mean_score": 0.8369, "n": 10, "passed": 6 }, "coordinated_containment_v01": { "mean_score": 0.3423, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.5779, "n": 10, "passed": 0 }, "coordinated_containment_v03": { "mean_score": 0.6905, "n": 10, "passed": 1 }, "deadlock_multi_approval_v01": { "mean_score": 0.6883, "n": 10, "passed": 2 }, "deadlock_multi_approval_v02": { "mean_score": 0.6599, "n": 10, "passed": 1 }, "deadlock_multi_approval_v03": { "mean_score": 0.69, "n": 10, "passed": 3 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.6948, "n": 10, "passed": 3 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.2702, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.4286, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v01": { "mean_score": 0.455, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v02": { "mean_score": 0.5506, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v03": { "mean_score": 0.4517, "n": 10, "passed": 0 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.6217, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.63, "n": 10, "passed": 2 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.5186, "n": 10, "passed": 0 }, "expiring_resource_hold_v01": { "mean_score": 0.6638, "n": 10, "passed": 2 }, "expiring_resource_hold_v02": { "mean_score": 0.5673, "n": 10, "passed": 1 }, "expiring_resource_hold_v03": { "mean_score": 0.5295, "n": 10, "passed": 0 }, "incident_commander_selection_v01": { "mean_score": 0.5583, "n": 10, "passed": 2 }, "incident_commander_selection_v02": { "mean_score": 0.6483, "n": 10, "passed": 2 }, "incident_commander_selection_v03": { "mean_score": 0.6167, "n": 10, "passed": 2 }, "it_deployment_gate_v01": { "mean_score": 0.6534, "n": 10, "passed": 2 }, "it_deployment_gate_v02": { "mean_score": 0.5188, "n": 10, "passed": 2 }, "it_deployment_gate_v03": { "mean_score": 0.6016, "n": 10, "passed": 1 }, "multi_device_pairing_v01": { "mean_score": 0.46, "n": 10, "passed": 1 }, "multi_device_pairing_v02": { "mean_score": 0.7492, "n": 10, "passed": 3 }, "multi_device_pairing_v03": { "mean_score": 0.6398, "n": 10, "passed": 1 }, "multi_party_access_permit_v01": { "mean_score": 0.5183, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.6869, "n": 10, "passed": 2 }, "multi_party_access_permit_v03": { "mean_score": 0.5529, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.5515, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.4042, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.5746, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.4367, "n": 10, "passed": 2 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.49, "n": 10, "passed": 2 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.5667, "n": 10, "passed": 3 }, "payment_chargeback_dispute_v01": { "mean_score": 0.8333, "n": 10, "passed": 7 }, "payment_chargeback_dispute_v02": { "mean_score": 0.5, "n": 10, "passed": 2 }, "payment_chargeback_dispute_v03": { "mean_score": 0.5, "n": 10, "passed": 3 }, "satellite_ground_station_handover_v01": { "mean_score": 0.6083, "n": 10, "passed": 4 }, "satellite_ground_station_handover_v02": { "mean_score": 0.7667, "n": 10, "passed": 5 }, "satellite_ground_station_handover_v03": { "mean_score": 0.65, "n": 10, "passed": 4 }, "sla_support_response_v01": { "mean_score": 0.5838, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.5623, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.3857, "n": 10, "passed": 0 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.625, "n": 10, "passed": 2 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.5833, "n": 10, "passed": 3 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.2367, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 18, "fail_closed_pct": 0.8, "n": 2137 }, "side_effect_fields": { "approval_escalated": 7, "approval_granted": 4, "approval_recorded": 17, "approval_requested": 21, "assets_accessed": 110, "booking_extended": 3, "booking_held": 21, "booking_released": 7, "containment_action": 20, "deployment_approved": 8, "deployment_gate_set": 10, "deployment_requested": 28, "files_created": 3, "lock_acquired": 10, "messages_sent": 2, "pairing_code_entered": 1, "pairing_code_shown": 1, "pairing_completed": 1, "pairing_started": 9, "review_assigned": 3, "review_submitted": 7 }, "step_mean": 8.76, "step_p50": 8, "stops": { "incomplete_required_submission": 199, "missing_required_side_effect": 293, "none": 102, "reached max_steps=20": 6 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 10": 4, "wait 120": 3, "wait 15": 3, "wait 180": 1, "wait 30": 121, "wait 45": 2, "wait 5": 11, "wait 60": 12 }, "max": 300.0, "mean": 34.2, "median": 30.0, "n": 159, "sub_30": 19, "total": 5440.0 }, "zero_open_tasks": 185, "zero_score": 41, "zero_write_tasks": 2 }, "terra": { "avg_score": 0.5891, "bucket_names": { "anchor": { "effect:files_created field_contains": 256 }, "content_file": { "effect:files_created field_semantic_match": 943 }, "content_msg": { "effect:messages_sent field_semantic_match": 1194 }, "grounding": { "effect:assets_accessed field_contains": 822 }, "other": {}, "side_effect": { "effect:approval_escalated any_field_equals": 11, "effect:approval_granted any_field_equals": 15, "effect:approval_recorded any_field_equals": 33, "effect:approval_requested any_field_equals": 28, "effect:booking_confirmed any_field_equals": 27, "effect:booking_extended any_field_equals": 5, "effect:booking_held any_field_equals": 38, "effect:booking_released any_field_equals": 18, "effect:containment_action any_field_equals": 58, "effect:deployment_approved any_field_equals": 10, "effect:deployment_cutover any_field_equals": 11, "effect:deployment_executed any_field_equals": 2, "effect:deployment_gate_set any_field_equals": 82, "effect:deployment_requested any_field_equals": 39, "effect:document_signed any_field_equals": 4, "effect:lock_acquired any_field_equals": 23, "effect:lock_released any_field_equals": 23, "effect:pairing_code_entered any_field_equals": 18, "effect:pairing_code_shown any_field_equals": 14, "effect:pairing_completed any_field_equals": 12, "effect:pairing_started any_field_equals": 14, "effect:review_assigned any_field_equals": 17, "effect:review_submitted any_field_equals": 13 }, "window": { "effect:messages_sent within_window": 30 } }, "capped": 8, "checks_total": 3760, "display": "gpt-5.6-terra", "dupe_rows": 0, "examples": { "anchor": [ { "message": "files_created.content contains 'READ_ONLY_RESTRICTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r01" }, { "message": "files_created.content contains 'FULLY_RECONCILED_AND_GRANTED': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r10" }, { "message": "files_created.content contains 'ERR-DATA-BACKFILL-409': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v01_r03" }, { "message": "files_created.content contains 'AUTH-GATE-9942-SYNC': False", "role": "required", "task_id": "conflict_vlm_barrier_sync_deployment_v02_r03" } ], "content_file": [ { "message": "LLM semantic match: The report states the final decision is GRANTED after reconciliation, rather than UNRECONCILED. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The report states the verdict as 'APPROVE FOR PERMIT ISSUANCE' rather than 'verify passcode and upgrade security scope'. [self-consistency 1/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r05" }, { "message": "LLM semantic match: The report references transaction ID TXN-APP-WRITE-9912 instead of TXN-APP-WRITE-9908. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v02_r04" }, { "message": "LLM semantic match: The report explicitly denies and holds dataset provisioning rather than authorizing dataset access. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v03_r10" } ], "content_msg": [ { "message": "LLM semantic match: The security passcode was transmitted to Sana in #iam-approvals-sec, but the message incorrectly stated access was already granted instead of requesting an updated WRITE approval. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "LLM semantic match: The message to Quinn in #proj-billing-migration addresses the unverified portal checkmarks, scope matrix mismatches, and delayed access, but omits the required reference code AUDIT-SEC-2026-READ. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: The message was sent to Sana_Security_Reviewer in #iam-approvals-sec citing AUDIT-SEC-2026-READ to request updated WRITE authorization on billing-prod, but failed to include passcode SEC-PASS-8842-BP. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "LLM semantic match: The message to Ravi_Application_Owner requested re-approval for WRITE scope on billing-prod, but cited reference code AUDIT-APP-2026-ARCH instead of AUDIT-SEC-2026-READ. [self-consistency 0/3 passed]", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" } ], "grounding": [ { "message": "assets_accessed.id contains 'iam_console_access_scope_matrix_r07': False; normalized fallback disabled ('iam_console_access_scope_matrix_r07' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r07" }, { "message": "assets_accessed.id contains 'security_dashboard_telemetry_20260330_100000': False; normalized fallback disabled ('security_dashboard_telemetry_20260330_100000' is a concrete visible asset id -- exact match required)", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r08" }, { "message": "assets_accessed.id contains 'iam_portal_access_scope_matrix_20241024_085500': False; normalized fallback disabled ('iam_portal_access_scope_matrix_20241024_085500' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v02_r07" }, { "message": "assets_accessed.id contains 'audit_dashboard_passcode_telemetry_20250218_103000': False; normalized fallback disabled ('audit_dashboard_passcode_telemetry_20250218_103000' is a concrete visible asset id -- exact match required)", "role": "diagnostic", "task_id": "conflict_vlm_access_request_approval_v03_r06" } ], "other": [], "side_effect": [ { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_granted emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r01" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r02" }, { "message": "approval_recorded emitted for target 'billing_prod': False", "role": "required", "task_id": "conflict_vlm_access_request_approval_v01_r03" } ], "window": [ { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r01" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r02" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r03" }, { "message": "messages_sent [sender contains 'Iris_SLA_Monitor'] within window (180 <= virtual_time <= 180): False", "role": "required", "task_id": "conflict_vlm_sla_support_response_v01_r04" } ] }, "failed_checks": { "anchor": { "diagnostic": 0, "required": 63 }, "content_file": { "diagnostic": 0, "required": 222 }, "content_msg": { "diagnostic": 0, "required": 658 }, "grounding": { "diagnostic": 141, "required": 61 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 350 }, "window": { "diagnostic": 0, "required": 28 } }, "failed_total": 1523, "gating": { "content_involved": 414, "grounding_only": 5, "grounding_only_ids": [ "conflict_vlm_distributed_lock_shared_resource_v03_r06", "conflict_vlm_expense_reimbursement_conflict_v01_r04", "conflict_vlm_expense_reimbursement_conflict_v01_r06", "conflict_vlm_expense_reimbursement_conflict_v02_r09", "conflict_vlm_multi_device_pairing_v02_r06" ], "n_fail": 480, "no_required_check_failed": 0, "no_required_check_failed_ids": [] }, "loop_examples": [ { "count": 5, "score": 0.42857142857142855, "task_id": "conflict_vlm_coordinated_containment_v03_r02", "verb": "file read" }, { "count": 6, "score": 0.5, "task_id": "conflict_vlm_distributed_lock_shared_resource_v02_r01", "verb": "file read" }, { "count": 8, "score": 0.3333333333333333, "task_id": "conflict_vlm_incident_commander_selection_v01_r06", "verb": "message search" }, { "count": 6, "score": 0.6666666666666666, "task_id": "conflict_vlm_multimodal_temporal_conflict_01_v02_r01", "verb": "message search" } ], "loop_verbs": { "asset open": 1, "file read": 2, "message search": 3 }, "loopy_capped": 6, "missing_check_result_ids": [], "missing_check_results": 0, "n": 600, "near_miss": 267, "ok_false": 0, "pass_pct": 20.0, "passed": 120, "rank": { "avg_score": 7, "capped": 3, "failed_total": 7, "loopy_capped": 3, "near_miss": 5, "pass_pct": 6, "step_mean": 3, "step_p50": 2, "zero_open_tasks": 6, "zero_score": 7, "zero_write_tasks": 2 }, "rows": 600, "scenarios": { "access_request_approval_v01": { "mean_score": 0.7014, "n": 10, "passed": 2 }, "access_request_approval_v02": { "mean_score": 0.7092, "n": 10, "passed": 2 }, "access_request_approval_v03": { "mean_score": 0.6348, "n": 10, "passed": 2 }, "barrier_sync_deployment_v01": { "mean_score": 0.6186, "n": 10, "passed": 2 }, "barrier_sync_deployment_v02": { "mean_score": 0.7488, "n": 10, "passed": 5 }, "barrier_sync_deployment_v03": { "mean_score": 0.6667, "n": 10, "passed": 3 }, "clinical_trial_dosage_conflict_v01": { "mean_score": 0.3167, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v02": { "mean_score": 0.4774, "n": 10, "passed": 1 }, "clinical_trial_dosage_conflict_v03": { "mean_score": 0.4515, "n": 10, "passed": 1 }, "collaborative_contract_review_v01": { "mean_score": 0.5849, "n": 10, "passed": 2 }, "collaborative_contract_review_v02": { "mean_score": 0.735, "n": 10, "passed": 6 }, "collaborative_contract_review_v03": { "mean_score": 0.7906, "n": 10, "passed": 3 }, "coordinated_containment_v01": { "mean_score": 0.3187, "n": 10, "passed": 0 }, "coordinated_containment_v02": { "mean_score": 0.6284, "n": 10, "passed": 2 }, "coordinated_containment_v03": { "mean_score": 0.6795, "n": 10, "passed": 1 }, "deadlock_multi_approval_v01": { "mean_score": 0.6508, "n": 10, "passed": 3 }, "deadlock_multi_approval_v02": { "mean_score": 0.7492, "n": 10, "passed": 3 }, "deadlock_multi_approval_v03": { "mean_score": 0.735, "n": 10, "passed": 3 }, "distributed_lock_shared_resource_v01": { "mean_score": 0.5514, "n": 10, "passed": 2 }, "distributed_lock_shared_resource_v02": { "mean_score": 0.2387, "n": 10, "passed": 0 }, "distributed_lock_shared_resource_v03": { "mean_score": 0.381, "n": 10, "passed": 0 }, "employee_onboarding_workflow_v01": { "mean_score": 0.5498, "n": 10, "passed": 3 }, "employee_onboarding_workflow_v02": { "mean_score": 0.5306, "n": 10, "passed": 1 }, "employee_onboarding_workflow_v03": { "mean_score": 0.546, "n": 10, "passed": 1 }, "expense_reimbursement_conflict_v01": { "mean_score": 0.7667, "n": 10, "passed": 4 }, "expense_reimbursement_conflict_v02": { "mean_score": 0.595, "n": 10, "passed": 0 }, "expense_reimbursement_conflict_v03": { "mean_score": 0.6631, "n": 10, "passed": 1 }, "expiring_resource_hold_v01": { "mean_score": 0.6093, "n": 10, "passed": 1 }, "expiring_resource_hold_v02": { "mean_score": 0.406, "n": 10, "passed": 0 }, "expiring_resource_hold_v03": { "mean_score": 0.7798, "n": 10, "passed": 1 }, "incident_commander_selection_v01": { "mean_score": 0.4583, "n": 10, "passed": 1 }, "incident_commander_selection_v02": { "mean_score": 0.6983, "n": 10, "passed": 4 }, "incident_commander_selection_v03": { "mean_score": 0.7017, "n": 10, "passed": 3 }, "it_deployment_gate_v01": { "mean_score": 0.6559, "n": 10, "passed": 3 }, "it_deployment_gate_v02": { "mean_score": 0.5857, "n": 10, "passed": 2 }, "it_deployment_gate_v03": { "mean_score": 0.6912, "n": 10, "passed": 2 }, "multi_device_pairing_v01": { "mean_score": 0.5344, "n": 10, "passed": 1 }, "multi_device_pairing_v02": { "mean_score": 0.6481, "n": 10, "passed": 1 }, "multi_device_pairing_v03": { "mean_score": 0.6582, "n": 10, "passed": 3 }, "multi_party_access_permit_v01": { "mean_score": 0.5583, "n": 10, "passed": 0 }, "multi_party_access_permit_v02": { "mean_score": 0.7107, "n": 10, "passed": 3 }, "multi_party_access_permit_v03": { "mean_score": 0.6895, "n": 10, "passed": 2 }, "multi_tab_slot_monitoring_v01": { "mean_score": 0.5402, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v02": { "mean_score": 0.4492, "n": 10, "passed": 0 }, "multi_tab_slot_monitoring_v03": { "mean_score": 0.5546, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v01": { "mean_score": 0.3833, "n": 10, "passed": 1 }, "multimodal_temporal_conflict_01_v02": { "mean_score": 0.41, "n": 10, "passed": 0 }, "multimodal_temporal_conflict_01_v03": { "mean_score": 0.4517, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v01": { "mean_score": 0.7667, "n": 10, "passed": 6 }, "payment_chargeback_dispute_v02": { "mean_score": 0.425, "n": 10, "passed": 1 }, "payment_chargeback_dispute_v03": { "mean_score": 0.7333, "n": 10, "passed": 5 }, "satellite_ground_station_handover_v01": { "mean_score": 0.6083, "n": 10, "passed": 4 }, "satellite_ground_station_handover_v02": { "mean_score": 0.9, "n": 10, "passed": 8 }, "satellite_ground_station_handover_v03": { "mean_score": 0.75, "n": 10, "passed": 5 }, "sla_support_response_v01": { "mean_score": 0.5386, "n": 10, "passed": 0 }, "sla_support_response_v02": { "mean_score": 0.6071, "n": 10, "passed": 0 }, "sla_support_response_v03": { "mean_score": 0.4754, "n": 10, "passed": 2 }, "warehouse_robotics_safety_halt_v01": { "mean_score": 0.4633, "n": 10, "passed": 2 }, "warehouse_robotics_safety_halt_v02": { "mean_score": 0.6167, "n": 10, "passed": 3 }, "warehouse_robotics_safety_halt_v03": { "mean_score": 0.27, "n": 10, "passed": 0 } }, "semantic": { "fail_closed": 16, "fail_closed_pct": 0.7, "n": 2137 }, "side_effect_fields": { "approval_escalated": 8, "approval_granted": 3, "approval_recorded": 19, "approval_requested": 20, "assets_accessed": 49, "booking_extended": 4, "booking_held": 27, "booking_released": 11, "containment_action": 21, "deployment_approved": 8, "deployment_gate_set": 10, "deployment_requested": 29, "files_created": 1, "lock_acquired": 15, "messages_sent": 8, "pairing_code_entered": 2, "pairing_code_shown": 1, "pairing_completed": 1, "pairing_started": 11, "review_assigned": 5, "review_submitted": 8 }, "step_mean": 9.98, "step_p50": 9, "stops": { "incomplete_required_submission": 211, "missing_required_side_effect": 261, "none": 120, "reached max_steps=20": 8 }, "total_checks": { "anchor": { "diagnostic": 0, "required": 256 }, "content_file": { "diagnostic": 0, "required": 943 }, "content_msg": { "diagnostic": 0, "required": 1194 }, "grounding": { "diagnostic": 476, "required": 346 }, "other": { "diagnostic": 0, "required": 0 }, "side_effect": { "diagnostic": 0, "required": 515 }, "window": { "diagnostic": 0, "required": 30 } }, "unbucketed_names": {}, "waits": { "hist": { "wait 1": 4, "wait 10": 35, "wait 2": 13, "wait 20": 2, "wait 3": 18, "wait 300": 4, "wait 5": 140, "wait 8": 7 }, "max": 300.0, "mean": 11.9, "median": 5.0, "n": 230, "sub_30": 223, "total": 2737.0 }, "zero_open_tasks": 6, "zero_score": 35, "zero_write_tasks": 1 } } } }, "efforts": [ "low", "medium", "high" ], "fixed_tree": { "dsv4": "eval_results/data_3_6_flash_v8/open_source_high/DeepSeek/v4_pro", "glm": "eval_results/data_3_6_flash_v8/open_source_high/ZhipuAI/glm_5_2", "kimi": "eval_results/data_3_6_flash_v8/open_source_high/Moonshot/kimi_k3" }, "models": { "dsv4": "deepseek-v4-pro", "flash": "gemini-3.5-flash", "flash36": "gemini-3.6-flash", "flash37": "gemini-3.7-flash", "glm": "glm-5.2", "kimi": "kimi-k3", "luna": "gpt-5.6-luna", "opus": "claude-opus-4.8", "opus5": "claude-opus-5", "pro": "gemini-3.1-pro", "sol": "gpt-5.6-sol", "sonnet": "claude-sonnet-5", "terra": "gpt-5.6-terra" }, "primary_effort": "high" }