{
  "agent": {},
  "commit": "0556f836b8e0205f5c29e68721c9cfa44ab3ca00",
  "exposure": {
    "aigf.controls": 25,
    "aigf.crosswalk_refs": 763,
    "aigf.reference_entries": 967,
    "aigf.reference_frameworks": 13,
    "aigf.risk_control_edges": 94,
    "aigf.risks": 24,
    "cdm.choice_types": 16,
    "cdm.enums": 280,
    "cdm.qualify_functions": 35,
    "cdm.root_types": 16,
    "cdm.samples": 40,
    "cdm.schemas": 1143,
    "cdm.schemas_legacy_vintage": 1066,
    "cdm.types": 767,
    "fdc3.context_types": 28,
    "fdc3.intents": 19,
    "fdc3.schemas": 29,
    "resources_total": 14,
    "tools_total": 29
  },
  "generated_at": "2026-10-26T21:47:05+00:00",
  "latency_detail": {
    "aigf.find_by_external_reference": {
      "budget_p95": 10.0,
      "mean": 1.547,
      "n": 100,
      "p50": 1.547,
      "p95": 1.591,
      "p99": 1.693
    },
    "aigf.get_control": {
      "budget_p95": 10.0,
      "mean": 1.333,
      "n": 100,
      "p50": 1.312,
      "p95": 1.378,
      "p99": 1.479
    },
    "aigf.get_risk": {
      "budget_p95": 10.0,
      "mean": 2.091,
      "n": 100,
      "p50": 2.085,
      "p95": 2.171,
      "p99": 2.235
    },
    "aigf.list_risks": {
      "budget_p95": 10.0,
      "mean": 1.961,
      "n": 100,
      "p50": 1.954,
      "p95": 2.019,
      "p99": 2.099
    },
    "aigf.map_risks_to_controls": {
      "budget_p95": 10.0,
      "mean": 1.903,
      "n": 100,
      "p50": 1.897,
      "p95": 1.978,
      "p99": 2.04
    },
    "aigf.search_framework": {
      "budget_p95": 25.0,
      "mean": 2.957,
      "n": 100,
      "p50": 2.94,
      "p95": 3.087,
      "p99": 3.139
    },
    "aigf.search_status": {
      "budget_p95": 5.0,
      "mean": 0.575,
      "n": 100,
      "p50": 0.571,
      "p95": 0.604,
      "p99": 0.698
    },
    "cdm.describe_type": {
      "budget_p95": 5.0,
      "mean": 1.24,
      "n": 100,
      "p50": 1.223,
      "p95": 1.302,
      "p99": 1.35
    },
    "cdm.explain_event": {
      "budget_p95": 10.0,
      "mean": 0.803,
      "n": 100,
      "p50": 0.801,
      "p95": 0.833,
      "p99": 0.904
    },
    "cdm.list_types": {
      "budget_p95": 10.0,
      "mean": 6.294,
      "n": 100,
      "p50": 5.476,
      "p95": 6.484,
      "p99": 6.821
    },
    "cdm.search_types": {
      "budget_p95": 25.0,
      "mean": 3.122,
      "n": 100,
      "p50": 3.057,
      "p95": 3.404,
      "p99": 4.357
    },
    "cdm.validate_object": {
      "budget_p95": 150.0,
      "mean": 36.357,
      "n": 100,
      "p50": 36.248,
      "p95": 37.419,
      "p99": 38.163
    },
    "fdc3.get_context_schema": {
      "budget_p95": 5.0,
      "mean": 1.067,
      "n": 100,
      "p50": 1.058,
      "p95": 1.099,
      "p99": 1.191
    },
    "fdc3.list_intents": {
      "budget_p95": 10.0,
      "mean": 2.078,
      "n": 100,
      "p50": 2.07,
      "p95": 2.159,
      "p99": 2.19
    },
    "fdc3.suggest_intent": {
      "budget_p95": 10.0,
      "mean": 1.571,
      "n": 100,
      "p50": 1.553,
      "p95": 1.639,
      "p99": 1.737
    },
    "fdc3.validate_context": {
      "budget_p95": 10.0,
      "mean": 0.781,
      "n": 100,
      "p50": 0.775,
      "p95": 0.851,
      "p99": 0.943
    }
  },
  "latency_ms_p95": {
    "aigf.find_by_external_reference": 1.591,
    "aigf.get_control": 1.378,
    "aigf.get_risk": 2.171,
    "aigf.list_risks": 2.019,
    "aigf.map_risks_to_controls": 1.978,
    "aigf.search_framework": 3.087,
    "aigf.search_status": 0.604,
    "cdm.describe_type": 1.302,
    "cdm.explain_event": 0.833,
    "cdm.list_types": 6.484,
    "cdm.search_types": 3.404,
    "cdm.validate_object": 37.419,
    "fdc3.get_context_schema": 1.099,
    "fdc3.list_intents": 2.159,
    "fdc3.suggest_intent": 1.639,
    "fdc3.validate_context": 0.851
  },
  "offline_evals": {
    "by_server": {
      "cdm": {
        "count": 8,
        "passed": 8,
        "steps": 25
      },
      "fdc3": {
        "count": 8,
        "passed": 8,
        "steps": 26
      }
    },
    "failed": 0,
    "failures": [],
    "passed": 16,
    "scenarios": 16,
    "success_rate": 1.0,
    "tool_resource_steps": 51
  },
  "retrieval": {
    "misses": [
      {
        "expected": [
          "AIR-DET-004",
          "AIR-PREV-022"
        ],
        "id": "q41",
        "kind": "multihop",
        "mode": "search_only",
        "question": "What controls should be in place both for an agent whose stored memory gets tampered with and for a multi-agent system where one compromised agent's output cascades to the others?",
        "top5": [
          "AIR-PREV-022",
          "AIR-PREV-018",
          "AIR-PREV-023",
          "AIR-DET-021",
          "AIR-PREV-024"
        ]
      },
      {
        "expected": [
          "AIR-DET-004",
          "AIR-PREV-020",
          "AIR-PREV-023"
        ],
        "id": "q42",
        "kind": "multihop",
        "mode": "search_only",
        "question": "Which controls cover both a poisoned MCP server dependency and an agent that harvests credentials while using its granted tool access?",
        "top5": [
          "AIR-PREV-020",
          "AIR-PREV-025",
          "AIR-PREV-018",
          "AIR-PREV-023",
          "AIR-PREV-024"
        ]
      },
      {
        "expected": [
          "AIR-DET-004",
          "AIR-DET-021"
        ],
        "id": "q43",
        "kind": "multihop",
        "mode": "search_only",
        "question": "Which detective controls mitigate the risks around agents calling MCP servers, having their tool chains manipulated, or bypassing their authorization?",
        "top5": [
          "AIR-PREV-020",
          "AIR-PREV-019",
          "AIR-PREV-018",
          "AIR-PREV-023",
          "AIR-PREV-024"
        ]
      },
      {
        "expected": [
          "AIR-PREV-005",
          "AIR-PREV-006"
        ],
        "id": "q44",
        "kind": "multihop",
        "mode": "search_only",
        "question": "Which preventative controls address both an AI system that confidently states inaccurate facts and one whose behaviour has drifted out of alignment with its intended use?",
        "top5": [
          "AIR-DET-016",
          "AIR-PREV-005",
          "AIR-DET-015",
          "AIR-DET-001",
          "AIR-DET-011"
        ]
      },
      {
        "expected": [
          "AIR-DET-004",
          "AIR-DET-011",
          "AIR-DET-015"
        ],
        "id": "q45",
        "kind": "multihop",
        "mode": "search_only",
        "question": "Which detective controls apply to both a model vendor silently swapping versions and a system that gives different answers to the identical query each time?",
        "top5": [
          "AIR-PREV-010",
          "AIR-DET-001",
          "AIR-PREV-003",
          "AIR-DET-016",
          "AIR-DET-021"
        ]
      },
      {
        "expected": [
          "AIR-PREV-002",
          "AIR-PREV-006",
          "AIR-PREV-007",
          "AIR-PREV-012"
        ],
        "id": "q46",
        "kind": "multihop",
        "mode": "search_only",
        "question": "Which preventative controls are relevant to both training data being poisoned and AI-generated outputs infringing on someone else's intellectual property?",
        "top5": [
          "AIR-PREV-017",
          "AIR-DET-001",
          "AIR-PREV-002",
          "AIR-DET-011",
          "AIR-DET-013"
        ]
      },
      {
        "expected": [
          "AIR-PREV-003",
          "AIR-PREV-007",
          "AIR-PREV-008",
          "AIR-PREV-017"
        ],
        "id": "q47",
        "kind": "multihop",
        "mode": "search_only",
        "question": "Which preventative controls help with both the model host going down unexpectedly and damage to the firm's reputation from AI mistakes?",
        "top5": [
          "AIR-DET-009",
          "AIR-PREV-010",
          "AIR-DET-001",
          "AIR-PREV-017",
          "AIR-PREV-008"
        ]
      },
      {
        "expected": [
          "AIR-PREV-003",
          "AIR-PREV-007",
          "AIR-PREV-008",
          "AIR-PREV-017"
        ],
        "id": "q47",
        "kind": "multihop",
        "mode": "tool_assisted",
        "question": "Which preventative controls help with both the model host going down unexpectedly and damage to the firm's reputation from AI mistakes?",
        "top5": [
          "AIR-PREV-003",
          "AIR-PREV-017",
          "AIR-PREV-007",
          "AIR-PREV-018",
          "AIR-PREV-024"
        ]
      },
      {
        "expected": [
          "AIR-PREV-002",
          "AIR-PREV-006",
          "AIR-PREV-012"
        ],
        "id": "q48",
        "kind": "multihop",
        "mode": "search_only",
        "question": "Which preventative controls are common to both sensitive data leaking to a hosted model and an AI system's training data being poisoned?",
        "top5": [
          "AIR-DET-001",
          "AIR-PREV-017",
          "AIR-PREV-003",
          "AIR-DET-009",
          "AIR-PREV-006"
        ]
      },
      {
        "expected": [
          "AIR-PREV-018",
          "AIR-PREV-019",
          "AIR-PREV-022",
          "AIR-PREV-023"
        ],
        "id": "q49",
        "kind": "multihop",
        "mode": "search_only",
        "question": "Which preventative controls guard against both an agent exceeding its authorization limits and an agent being used to harvest credentials?",
        "top5": [
          "AIR-PREV-023",
          "AIR-PREV-018",
          "AIR-PREV-024",
          "AIR-PREV-022",
          "AIR-PREV-025"
        ]
      },
      {
        "expected": [
          "AIR-DET-013",
          "AIR-DET-016",
          "AIR-DET-021"
        ],
        "id": "q50",
        "kind": "multihop",
        "mode": "search_only",
        "question": "Which detective controls address both an AI system nobody can explain and gaps in meeting regulatory obligations?",
        "top5": [
          "AIR-DET-021",
          "AIR-DET-001",
          "AIR-PREV-007",
          "AIR-DET-004",
          "AIR-DET-016"
        ]
      }
    ],
    "mrr": 0.9066666666666666,
    "questions": 50,
    "recall_at_5": 0.995,
    "search_mode": "hybrid",
    "search_only": {
      "mrr": 0.7978888888888889,
      "recall_at_1": 0.6033333333333333,
      "recall_at_3": 0.8033333333333333,
      "recall_at_5": 0.8833333333333333
    },
    "search_only_non_multihop": {
      "mrr": 0.8583333333333333,
      "recall_at_1": 0.71875,
      "recall_at_3": 0.94375,
      "recall_at_5": 1.0
    },
    "tool_assisted": {
      "mrr": 0.9066666666666666,
      "recall_at_1": 0.6866666666666666,
      "recall_at_3": 0.9116666666666666,
      "recall_at_5": 0.995
    }
  },
  "servers": {
    "aigf": {
      "resource_count": 6,
      "standard_version": "v2",
      "tool_count": 11,
      "tools": [
        "find_by_external_reference",
        "get_control",
        "get_risk",
        "list_controls",
        "list_reference_frameworks",
        "list_risks",
        "map_control_to_external",
        "map_risks_to_controls",
        "search_framework",
        "search_status",
        "server_info"
      ],
      "upstream_commit": "8b0e80132f05a46e3e64cc27a6a3e56d59b5479f",
      "upstream_ref": "main",
      "version": "0.3.2"
    },
    "cdm": {
      "resource_count": 4,
      "standard_version": "7.8.0",
      "tool_count": 9,
      "tools": [
        "describe_type",
        "explain_event",
        "get_sample",
        "list_products",
        "list_samples",
        "list_types",
        "search_types",
        "server_info",
        "validate_object"
      ],
      "upstream_commit": "5b852ed61573467c8f8e6bb5cc02779a70df032d",
      "upstream_ref": "7.8.0",
      "version": "0.3.2"
    },
    "fdc3": {
      "resource_count": 4,
      "standard_version": "2.2.3",
      "tool_count": 9,
      "tools": [
        "find_intents_by_context",
        "get_context_schema",
        "get_intent",
        "list_context_types",
        "list_intents",
        "search_intents",
        "server_info",
        "suggest_intent",
        "validate_context"
      ],
      "upstream_commit": "0d71e7fb1b7a0f660c3efa7ba3ed20562673de40",
      "upstream_ref": "v2.2.3",
      "version": "0.3.2"
    }
  },
  "tests": {
    "count": 282,
    "coverage_pct": 94.92,
    "failed": 0,
    "passed": 281,
    "skipped": 1
  }
}