{
  "schema": "chi-bench/submission/v1",
  "submission": {
    "id": "nemotron-3-ultra-openai-agents",
    "team": "Actava",
    "contact": "dark.savi@gmail.com",
    "agent": "openai-agents",
    "model": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16:peft:262144",
    "notes": "NVIDIA Nemotron 3 Ultra 256K via the OpenAI Agents SDK harness on Tinker Chat Completions with an explicit tinker provider route. Scored 0/75 binary pass@1 with mean fractional reward 0.217, i.e. partial task progress that never met a full rubric. Infrastructure was healthy: 75/75 verifier-backed results and scorecards, preflight and canary gates passed, and the single agent exception was a genuine MaxTurnsExceeded outcome rather than a harness fault.\n",
    "submitted_at": "2026-08-12T20:20:00Z"
  },
  "dataset": {
    "name": "chi-bench",
    "version": "chi-bench-v1.0.0",
    "domains": [
      "pa_provider",
      "pa_um",
      "cm"
    ]
  },
  "results": {
    "overall": {
      "n_trials": 75,
      "n_tasks": 75,
      "pass_at_1": 0.0,
      "mean_cost_usd": 0.47223568448,
      "mean_walltime_s": 91.86829674666667
    },
    "per_domain": {
      "pa_provider": {
        "n_trials": 25,
        "n_tasks": 25,
        "pass_at_1": 0.0,
        "mean_cost_usd": 0.39578937776000006,
        "mean_walltime_s": 85.60383688
      },
      "pa_um": {
        "n_trials": 25,
        "n_tasks": 25,
        "pass_at_1": 0.0,
        "mean_cost_usd": 0.45005764624,
        "mean_walltime_s": 62.04124316
      },
      "cm": {
        "n_trials": 25,
        "n_tasks": 25,
        "pass_at_1": 0.0,
        "mean_cost_usd": 0.57086002944,
        "mean_walltime_s": 127.9598102
      }
    },
    "mean_cost_usd": 0.47223568448,
    "mean_walltime_s": 91.86829674666667
  },
  "provenance": {
    "chi_bench_git_sha": "2395e9e2ae7d42cdf2750dc3e67daeb0e672b910",
    "image_digest": null,
    "judge_model": "claude-opus-4-7",
    "harness_version": "0.1.0",
    "code_dirty": false,
    "dataset_version": "chi-bench-v1.0.0",
    "environment": "modal",
    "started_at": "2026-07-22T22:31:29.302007Z",
    "finished_at": "2026-07-22T23:37:41.111758Z",
    "source": "nemotron3_ultra_tinker_full_2026_07 experiment matrix",
    "cost_basis": "74 of 75 trials carry price-table-estimated cost; 1 trial reports no usage. Rate reflects the limited-time Tinker discount verified 2026-07-22.",
    "eval_validity": "Closed per docs/superpowers/plans/2026-07-22-nemotron-tinker-eval.md: 3/3 canary, 75/75 trials and scorecards, no infrastructure exceptions.",
    "trajectory_repair": {
      "count": 1,
      "reason": "1 pa_provider trial (MaxTurnsExceeded) completed verification but emitted no trace.jsonl.",
      "method": "Generated ATIF-v1.2 error-only trajectories with the official openai_agents_harness._build_atif_trajectory(records=[], ...) builder from each preserved agent/run_result.json; original logs were not modified."
    }
  }
}