{
  "model_name": "gpt-realtime-2",
  "model_organization": "OpenAI",
  "submitting_organization": "Sierra",
  "submission_date": "2026-06-29",
  "submission_type": "custom",
  "modality": "voice",
  "contact_info": {
    "email": "soham@sierra.ai",
    "name": "Sierra Research Team"
  },
  "results": {
    "retail": {
      "pass_1": 50.0
    },
    "airline": {
      "pass_1": 66.0
    },
    "telecom": {
      "pass_1": 37.719298245614034
    }
  },
  "reasoning_effort": "xhigh",
  "is_new": true,
  "trajectories_available": true,
  "trajectory_files": {
    "airline": "gpt-realtime-2_voice_xhigh_usersim-gpt55xhigh_airline",
    "retail": "gpt-realtime-2_voice_xhigh_usersim-gpt55xhigh_retail",
    "telecom": "gpt-realtime-2_voice_xhigh_usersim-gpt55_telecom"
  },
  "methodology": {
    "evaluation_date": "2026-06-29",
    "tau2_bench_version": "1.0.0",
    "user_simulator": "gpt-5.5-2026-04-23 (xhigh)",
    "notes": "Custom submission: identical to the standard gpt-realtime-2 voice evaluation (audio-native, regular speech complexity, ElevenLabs eleven_v3 TTS, agent reasoning_effort=xhigh) EXCEPT the user simulator LLM is gpt-5.5-2026-04-23 run at reasoning_effort=xhigh instead of the leaderboard-standard v1.0 (gpt-4.1). Because telecom's user simulator uses function tools, gpt-5.5 rejects reasoning_effort=none with tools, so all domains use xhigh for consistency. Not comparable to standard-user-simulator leaderboard entries.",
    "verification": {
      "modified_prompts": false,
      "omitted_questions": false
    }
  },
  "voice_config": {
    "provider": "OpenAI",
    "model": "gpt-realtime-2",
    "tick_duration_seconds": 0.2,
    "max_steps_seconds": 1200.0,
    "user_tts_provider": "elevenlabs/eleven_v3"
  },
  "model_release": {
    "release_date": "2026-05-07",
    "announcement_url": "https://openai.com/index/advancing-voice-intelligence-with-new-models-in-the-api/",
    "announcement_title": "Advancing voice intelligence with new models in the API"
  },
  "interaction_metrics": {
    "version": "1.0",
    "config": {
      "tick_duration_sec": 0.2,
      "no_yield_window_sec": 2.0,
      "backchannel_yield_window_sec": 1.0,
      "vocal_tic_yield_window_sec": 1.0,
      "non_directed_yield_window_sec": 1.0,
      "vocal_tic_response_window_sec": 2.0,
      "non_directed_response_window_sec": 2.0
    },
    "domains": {
      "airline": {
        "response_latency_mean": 2.1115973741794303,
        "yield_latency_mean": 0.6211267605633803,
        "response_rate": 0.9481327800829875,
        "yield_rate": 1.0,
        "agent_interruption_rate": 0.21991701244813278,
        "selectivity_backchannel": 0.0,
        "selectivity_vocal_tic": 0.13043478260869568,
        "selectivity_non_directed": 0.31999999999999995,
        "counts": {
          "n_simulations": 50,
          "response_total": 482,
          "yield_total": 284,
          "backchannel_total": 29,
          "vocal_tic_total": 46,
          "non_directed_total": 50,
          "agent_interrupts_count": 106
        }
      },
      "retail": {
        "response_latency_mean": 1.8020682523267835,
        "yield_latency_mean": 0.5992673992673994,
        "response_rate": 0.9757820383451059,
        "yield_rate": 0.9981718464351006,
        "agent_interruption_rate": 0.2401614530776993,
        "selectivity_backchannel": 0.0,
        "selectivity_vocal_tic": 0.09890109890109888,
        "selectivity_non_directed": 0.16891891891891897,
        "counts": {
          "n_simulations": 114,
          "response_total": 991,
          "yield_total": 547,
          "backchannel_total": 55,
          "vocal_tic_total": 91,
          "non_directed_total": 148,
          "agent_interrupts_count": 238
        }
      },
      "telecom": {
        "response_latency_mean": 1.9037001441614605,
        "yield_latency_mean": 0.5997118155619595,
        "response_rate": 0.9848556554661618,
        "yield_rate": 0.9971264367816092,
        "agent_interruption_rate": 0.17132039753904402,
        "selectivity_backchannel": 0.03389830508474578,
        "selectivity_vocal_tic": 0.11055276381909551,
        "selectivity_non_directed": 0.12077294685990336,
        "counts": {
          "n_simulations": 114,
          "response_total": 2113,
          "yield_total": 1392,
          "backchannel_total": 59,
          "vocal_tic_total": 199,
          "non_directed_total": 207,
          "agent_interrupts_count": 362
        }
      }
    },
    "overall": {
      "response_latency_mean": 1.9391219235558914,
      "yield_latency_mean": 0.6067019917975798,
      "response_rate": 0.9695901579647517,
      "yield_rate": 0.9984327610722366,
      "agent_interruption_rate": 0.21046628768829204,
      "selectivity_backchannel": 0.011299435028248594,
      "selectivity_vocal_tic": 0.11329621510963002,
      "selectivity_non_directed": 0.20323062192627409,
      "counts": {
        "agent_interrupts_count": 706,
        "backchannel_total": 143,
        "n_simulations": 278,
        "non_directed_total": 405,
        "response_total": 3586,
        "vocal_tic_total": 336,
        "yield_total": 2223
      }
    }
  }
}